gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-06 13:27:43 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
1131a00fc25b49f2f9d5d14cb37cd17f332a3cd2
rustfs/crates/scanner/src/scanner_io.rs
T
Zhengchao An 40cd10c1d0 fix(scanner): surface per-tier usage in the data-usage snapshot (#5623) 
SizeSummary::tier_stats was populated for every scanned object but
apply_scanner_size_summary dropped it, so per-tier usage never reached
DataUsageInfo. Wire it through the same merge chain repl_target_stats
already uses, up to DataUsageInfo::tier_stats.

DataUsageEntry used the derived MessagePack encoding, which serialises
structs as arrays: appending a field turns the whole cache into a decode
error for older readers, so mixed-version nodes would invalidate each
other's cache every scan cycle. Give it the same hand-written
map-encoded Serialize DataUsageCacheInfo already carries, and record the
invariant in AGENTS.md.

Widen TierStats counters from i32 to u64 so a tier past 2^31 versions
cannot make checked_merge reject an entire usage snapshot, and drop the
duplicate TierStats/AllTierStats definitions in the scanner crate in
favour of the data-usage ones.
2026-08-02 10:46:36 +00:00

5160 lines
208 KiB
Rust
Raw Blame History

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT;
use crate::scanner_budget::ScannerCycleBudget;
use crate::scanner_folder::{ScannerItem, scan_data_folder};
use crate::sleeper::SCANNER_SLEEPER;
use crate::{
DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageCachePrepareOutcome,
DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, ScannerError, SizeSummary,
TierStats,
};
use futures::future::join_all;
use metrics::counter;
use rand::seq::SliceRandom as _;
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics};
#[cfg(test)]
use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS};
use rustfs_data_usage::{BucketTargetUsageInfo, BucketUsageInfo};
use rustfs_filemeta::FileMeta;
use rustfs_lock::{LockError, NamespaceLockGuard};
use rustfs_utils::path::path_join_buf;
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration};
use sha2::{Digest as _, Sha256};
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::path::Path;
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{LazyLock, Mutex as StdMutex, MutexGuard};
use std::time::{Instant, SystemTime};
use std::{fmt::Debug, sync::Arc};
use time::OffsetDateTime;
use tokio::sync::{Mutex, Notify, Semaphore, mpsc};
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, warn};
use crate::ScannerObjectInfo as ObjectInfo;
use crate::storage_api::scan::NamespaceLocking as _;
use crate::storage_api::scanner_io::{BucketInfo, BucketOptions};
use crate::{
BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result,
RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _,
ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, enqueue_runtime_free_version,
get_lifecycle_config, get_object_lock_config, get_replication_config, list_runtime_tiers, storageclass,
};
pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file";
pub(crate) const SCANNER_METADATA_CORRUPT_ERROR: &str = "scanner metadata corrupt";
pub(crate) const SCANNER_METADATA_TRANSIENT_ERROR: &str = "scanner metadata transient";
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_IO: &str = "io";
const EVENT_SCANNER_DISK_BUCKET_STATE: &str = "scanner_disk_bucket_state";
const EVENT_SCANNER_DATA_USAGE_STREAM: &str = "scanner_data_usage_stream";
const EVENT_SCANNER_CACHE_PERSIST_STATE: &str = "scanner_cache_persist_state";
const EVENT_SCANNER_SET_STATE: &str = "scanner_set_state";
const SCANNER_CACHE_LOCK_SUFFIX: &str = ".scanner-cycle.lock";
const SCANNER_CACHE_LOCK_POLL_INTERVAL: Duration = Duration::from_millis(250);
#[cfg(not(test))]
const SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
#[cfg(test)]
const SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(50);
const METRIC_SCANNER_SET_SCAN_CONCURRENCY_LIMIT: &str = "rustfs_scanner_set_scan_concurrency_limit";
const METRIC_SCANNER_DISK_SCAN_CONCURRENCY_LIMIT: &str = "rustfs_scanner_disk_scan_concurrency_limit";
const METRIC_SCANNER_SET_SCAN_WAIT_SECONDS: &str = "rustfs_scanner_set_scan_wait_seconds";
const METRIC_SCANNER_DISK_SCAN_WAIT_SECONDS: &str = "rustfs_scanner_disk_scan_wait_seconds";
const METRIC_SCANNER_SET_SCANS_ACTIVE: &str = "rustfs_scanner_set_scans_active";
const METRIC_SCANNER_SET_SCANS_QUEUED: &str = "rustfs_scanner_set_scans_queued";
const METRIC_SCANNER_DISK_BUCKET_SCANS_ACTIVE: &str = "rustfs_scanner_disk_bucket_scans_active";
const METRIC_SCANNER_DISK_BUCKET_SCANS_QUEUED: &str = "rustfs_scanner_disk_bucket_scans_queued";
pub type DirtyUsageBuckets = HashMap<String, u64>;
#[derive(Clone, Debug)]
struct DirtyUsageSnapshot {
buckets: Arc<DirtyUsageBuckets>,
generation: u64,
covers_all_pending: bool,
}
pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR))
}
pub(crate) fn is_scanner_metadata_transient_error(err: &StorageError) -> bool {
matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_TRANSIENT_ERROR))
}
fn scanner_metadata_corrupt_error(reason: impl std::fmt::Display, bucket: &str, object_path: &str) -> StorageError {
StorageError::other(format!(
"{SCANNER_METADATA_CORRUPT_ERROR}: {reason}, bucket={bucket}, object_path={object_path}"
))
}
fn scanner_metadata_transient_error(reason: impl std::fmt::Display, bucket: &str, object_path: &str) -> StorageError {
StorageError::other(format!(
"{SCANNER_METADATA_TRANSIENT_ERROR}: {reason}, bucket={bucket}, object_path={object_path}"
))
}
async fn object_lock_config_for_scanner_item(item: &ScannerItem) -> Option<Arc<ObjectLockConfiguration>> {
if let Some(config) = item.object_lock.clone() {
return Some(config);
}
get_object_lock_config(&item.bucket)
.await
.ok()
.map(|(config, _)| Arc::new(config))
}
fn object_lock_config_enabled(config: &ObjectLockConfiguration) -> bool {
config
.object_lock_enabled
.as_ref()
.is_some_and(|enabled| enabled.as_str() == ObjectLockEnabled::ENABLED)
}
pub struct ScannerBucketScanPlan {
buckets: Vec<BucketInfo>,
all_buckets: Arc<Vec<BucketInfo>>,
digest: DataUsageScanPlanDigest,
leader_epoch: u64,
dirty_usage_buckets: Arc<DirtyUsageBuckets>,
bucket_failures: ScannerBucketFailureState,
pending_maintenance_work: Arc<AtomicBool>,
cache_cycle_floor: Arc<AtomicU64>,
}
#[derive(Clone, Default)]
struct ScannerBucketFailureState {
hard: Arc<Mutex<HashSet<String>>>,
partial: Arc<Mutex<HashSet<String>>>,
namespace_not_found: Arc<Mutex<HashSet<String>>>,
}
fn scanner_bucket_plan_digest(buckets: &[BucketInfo], activity_digest: [u8; 32]) -> DataUsageScanPlanDigest {
let mut buckets = buckets.iter().collect::<Vec<_>>();
buckets.sort_unstable_by(|left, right| left.name.cmp(&right.name));
let mut hasher = Sha256::new();
hasher.update(activity_digest);
hasher.update(u64::try_from(buckets.len()).unwrap_or(u64::MAX).to_be_bytes());
for bucket in buckets {
let name = bucket.name.as_bytes();
hasher.update(u64::try_from(name.len()).unwrap_or(u64::MAX).to_be_bytes());
hasher.update(name);
match bucket.created {
Some(created) => {
hasher.update([1]);
hasher.update(created.unix_timestamp_nanos().to_be_bytes());
}
None => hasher.update([0]),
}
}
DataUsageScanPlanDigest(hasher.finalize().into())
}
fn scanner_bucket_cache_digest(
scan_plan_digest: DataUsageScanPlanDigest,
dirty_generation: Option<u64>,
) -> DataUsageScanPlanDigest {
let Some(dirty_generation) = dirty_generation else {
return scan_plan_digest;
};
let mut hasher = Sha256::new();
hasher.update(scan_plan_digest.0);
hasher.update(dirty_generation.to_be_bytes());
DataUsageScanPlanDigest(hasher.finalize().into())
}
static DIRTY_USAGE_BUCKET_GENERATION: AtomicU64 = AtomicU64::new(0);
static DIRTY_USAGE_BUCKETS: LazyLock<StdMutex<DirtyUsageBuckets>> = LazyLock::new(|| StdMutex::new(HashMap::new()));
static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0);
static SCANNER_MAINTENANCE_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ScannerDirtyUsageState {
pub generation: u64,
pub pending: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
pub enum ScannerDirtyUsageAckError {
#[error("scanner process instance changed before dirty usage acknowledgement")]
ProcessChanged,
#[error("scanner dirty usage generation cannot be acknowledged")]
InvalidGeneration,
}
fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
DIRTY_USAGE_BUCKETS.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
}
fn usize_to_u64_saturated(value: usize) -> u64 {
u64::try_from(value).unwrap_or(u64::MAX)
}
fn advance_generation(generation: &AtomicU64) -> u64 {
generation
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)))
.map_or_else(|current| current, |previous| previous.saturating_add(1))
}
pub fn record_dirty_usage_bucket(bucket: &str) {
if bucket.is_empty() {
return;
}
let pending_buckets = {
let mut dirty_buckets = dirty_usage_buckets();
let generation = advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
dirty_buckets.insert(bucket.to_string(), generation);
dirty_buckets.len()
};
global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets));
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
}
pub fn record_scanner_maintenance_change(bucket: &str) {
if bucket.is_empty() {
return;
}
advance_generation(&SCANNER_MAINTENANCE_GENERATION);
SCANNER_MAINTENANCE_NOTIFY.notify_one();
record_dirty_usage_bucket(bucket);
}
pub fn scanner_maintenance_generation() -> u64 {
SCANNER_MAINTENANCE_GENERATION.load(Ordering::Acquire)
}
pub(crate) async fn scanner_maintenance_changed() {
SCANNER_MAINTENANCE_NOTIFY.notified().await;
}
pub fn scanner_activity_epoch() -> &'static str {
SCANNER_ACTIVITY_EPOCH.as_str()
}
pub fn scanner_dirty_usage_state() -> ScannerDirtyUsageState {
let dirty_buckets = dirty_usage_buckets();
ScannerDirtyUsageState {
generation: DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire),
pending: !dirty_buckets.is_empty(),
}
}
pub fn acknowledge_dirty_usage_generation(
instance_id: &str,
generation: u64,
) -> std::result::Result<(), ScannerDirtyUsageAckError> {
if instance_id != scanner_activity_epoch() {
return Err(ScannerDirtyUsageAckError::ProcessChanged);
}
let (cleared_buckets, pending_buckets) = {
let mut dirty_buckets = dirty_usage_buckets();
let current_generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
if generation == 0 || generation == u64::MAX || current_generation == u64::MAX || generation > current_generation {
return Err(ScannerDirtyUsageAckError::InvalidGeneration);
}
let before = dirty_buckets.len();
dirty_buckets.retain(|_, dirty_generation| *dirty_generation > generation);
let cleared_buckets = before.saturating_sub(dirty_buckets.len());
if cleared_buckets > 0 {
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
}
(cleared_buckets, dirty_buckets.len())
};
global_metrics()
.record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared_buckets), usize_to_u64_saturated(pending_buckets));
Ok(())
}
pub(crate) fn dirty_usage_generation() -> u64 {
DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire)
}
pub fn clear_dirty_usage_bucket(bucket: &str) {
if bucket.is_empty() {
return;
}
let pending_buckets = {
let mut dirty_buckets = dirty_usage_buckets();
dirty_buckets.remove(bucket);
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
dirty_buckets.len()
};
global_metrics().record_scanner_dirty_usage_clear(usize_to_u64_saturated(pending_buckets));
}
fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo], absent_generation_cutoff: u64) -> DirtyUsageSnapshot {
let (snapshot, generation, covers_all_pending) = {
let dirty_buckets = dirty_usage_buckets();
let listed_buckets = dirty_buckets
.values()
.any(|generation| *generation > absent_generation_cutoff)
.then(|| buckets.iter().map(|bucket| bucket.name.as_str()).collect::<HashSet<_>>());
let snapshot = dirty_buckets
.iter()
.filter(|(bucket, generation)| {
**generation <= absent_generation_cutoff
|| listed_buckets
.as_ref()
.is_some_and(|listed_buckets| listed_buckets.contains(bucket.as_str()))
})
.map(|(bucket, generation)| (bucket.clone(), *generation))
.collect::<DirtyUsageBuckets>();
let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
let covers_all_pending = generation == absent_generation_cutoff && snapshot.len() == dirty_buckets.len();
(snapshot, generation, covers_all_pending)
};
global_metrics().record_scanner_dirty_usage_cycle_snapshot(usize_to_u64_saturated(snapshot.len()));
DirtyUsageSnapshot {
buckets: Arc::new(snapshot),
generation,
covers_all_pending,
}
}
pub(crate) fn dirty_usage_buckets_pending() -> bool {
!dirty_usage_buckets().is_empty()
}
pub(crate) async fn dirty_usage_bucket_notified() {
DIRTY_USAGE_BUCKET_NOTIFY.notified().await;
}
fn clear_dirty_usage_buckets(snapshot: &DirtyUsageBuckets) {
let (cleared_buckets, pending_buckets) = {
let mut dirty_buckets = dirty_usage_buckets();
let mut cleared_buckets = 0usize;
for (bucket, generation) in snapshot {
if dirty_buckets.get(bucket).is_some_and(|current| current == generation) {
dirty_buckets.remove(bucket);
cleared_buckets += 1;
}
}
if cleared_buckets > 0 {
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
}
(cleared_buckets, dirty_buckets.len())
};
global_metrics()
.record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared_buckets), usize_to_u64_saturated(pending_buckets));
}
fn dirty_usage_buckets_excluding_failed(snapshot: &DirtyUsageBuckets, failed_buckets: &HashSet<String>) -> DirtyUsageBuckets {
snapshot
.iter()
.filter(|(bucket, _)| !failed_buckets.contains(*bucket))
.map(|(bucket, generation)| (bucket.clone(), *generation))
.collect()
}
fn should_clear_dirty_usage_snapshot(
result_ok: bool,
completed_all_sets: bool,
budget_elapsed: bool,
activity_and_generation_current: bool,
dirty_buckets: &DirtyUsageBuckets,
failed_buckets: &HashSet<String>,
) -> Option<DirtyUsageBuckets> {
if result_ok && completed_all_sets && !budget_elapsed && activity_and_generation_current {
return Some(dirty_usage_buckets_excluding_failed(dirty_buckets, failed_buckets));
}
None
}
async fn record_failed_dirty_bucket(failed_buckets: &Arc<Mutex<HashSet<String>>>, bucket: &str) {
failed_buckets.lock().await.insert(bucket.to_string());
}
async fn record_partial_dirty_bucket(partial_buckets: &Arc<Mutex<HashSet<String>>>, bucket: &str) {
partial_buckets.lock().await.insert(bucket.to_string());
}
async fn requeue_bucket_work(
bucket_tx: &mpsc::Sender<BucketInfo>,
bucket: &BucketInfo,
work_guard: &mut BucketWorkGuard,
) -> bool {
if bucket_tx.send(bucket.clone()).await.is_err() {
return false;
}
work_guard.mark_requeued();
true
}
async fn mark_unprocessed_bucket_work_failed(
bucket_rx: &Mutex<mpsc::Receiver<BucketInfo>>,
remaining: &Arc<AtomicUsize>,
complete: &CancellationToken,
failed_buckets: &Arc<Mutex<HashSet<String>>>,
) -> usize {
let mut failed_count = 0;
let mut receiver = bucket_rx.lock().await;
while let Some(bucket) = receiver.recv().await {
record_failed_dirty_bucket(failed_buckets, &bucket.name).await;
drop(BucketWorkGuard::new(remaining.clone(), complete.clone()));
failed_count += 1;
}
failed_count
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum DirtyUsageSnapshotStatus {
Current,
Changed,
Unverified,
}
fn dirty_usage_snapshot_status(snapshot: &DirtyUsageSnapshot) -> DirtyUsageSnapshotStatus {
let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
if generation == u64::MAX {
DirtyUsageSnapshotStatus::Unverified
} else if snapshot.covers_all_pending && generation == snapshot.generation {
DirtyUsageSnapshotStatus::Current
} else {
DirtyUsageSnapshotStatus::Changed
}
}
#[cfg(test)]
fn dirty_usage_bucket_count() -> usize {
dirty_usage_buckets().len()
}
#[cfg(test)]
pub(crate) fn clear_dirty_usage_buckets_for_tests() {
dirty_usage_buckets().clear();
}
#[cfg(test)]
pub(crate) fn dirty_usage_buckets_for_tests() -> DirtyUsageBuckets {
dirty_usage_buckets().clone()
}
fn bucket_usage_scan_order(
buckets: &[BucketInfo],
old_cache: &DataUsageCache,
dirty_buckets: &DirtyUsageBuckets,
) -> Vec<BucketInfo> {
let mut ordered = Vec::with_capacity(buckets.len());
for bucket in buckets {
if dirty_buckets.contains_key(&bucket.name) {
ordered.push(bucket.clone());
}
}
for bucket in buckets {
if !dirty_buckets.contains_key(&bucket.name) && old_cache.find(&bucket.name).is_none() {
ordered.push(bucket.clone());
}
}
for bucket in buckets {
if !dirty_buckets.contains_key(&bucket.name) && old_cache.find(&bucket.name).is_some() {
ordered.push(bucket.clone());
}
}
ordered
}
fn record_set_scan_concurrency_limit(limit: usize) {
metrics::gauge!(METRIC_SCANNER_SET_SCAN_CONCURRENCY_LIMIT).set(limit as f64);
global_metrics().record_scanner_set_scan_state(Some(limit), None, None);
}
fn record_set_scans_queued(count: usize) {
metrics::gauge!(METRIC_SCANNER_SET_SCANS_QUEUED).set(count as f64);
global_metrics().record_scanner_set_scan_state(None, Some(count), None);
}
fn record_set_scans_active(count: usize) {
metrics::gauge!(METRIC_SCANNER_SET_SCANS_ACTIVE).set(count as f64);
global_metrics().record_scanner_set_scan_state(None, None, Some(count));
}
fn record_disk_scan_concurrency_limit(pool: &str, set: &str, limit: usize) {
metrics::gauge!(
METRIC_SCANNER_DISK_SCAN_CONCURRENCY_LIMIT,
"pool" => pool.to_owned(),
"set" => set.to_owned()
)
.set(limit as f64);
global_metrics().record_scanner_disk_bucket_scan_state(pool, set, Some(limit), None, None);
}
fn record_disk_bucket_scans_active(count: usize, pool: &str, set: &str) {
metrics::gauge!(
METRIC_SCANNER_DISK_BUCKET_SCANS_ACTIVE,
"pool" => pool.to_owned(),
"set" => set.to_owned()
)
.set(count as f64);
global_metrics().record_scanner_disk_bucket_scan_state(pool, set, None, None, Some(count));
}
struct SetScanActiveGuard {
active: Arc<AtomicUsize>,
}
impl SetScanActiveGuard {
fn new(active: Arc<AtomicUsize>) -> Self {
let active_count = active.fetch_add(1, Ordering::Relaxed) + 1;
record_set_scans_active(active_count);
Self { active }
}
}
impl Drop for SetScanActiveGuard {
fn drop(&mut self) {
let active_count = decrement_atomic_usize(&self.active);
record_set_scans_active(active_count);
}
}
struct DiskBucketScanActiveGuard {
active: Arc<AtomicUsize>,
pool: String,
set: String,
}
struct BucketWorkGuard {
remaining: Arc<AtomicUsize>,
complete: CancellationToken,
requeued: bool,
}
impl BucketWorkGuard {
fn new(remaining: Arc<AtomicUsize>, complete: CancellationToken) -> Self {
Self {
remaining,
complete,
requeued: false,
}
}
fn mark_requeued(&mut self) {
self.requeued = true;
}
}
impl Drop for BucketWorkGuard {
fn drop(&mut self) {
if !self.requeued && self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 {
self.complete.cancel();
}
}
}
impl DiskBucketScanActiveGuard {
fn new(active: Arc<AtomicUsize>, pool: String, set: String) -> Self {
let active_count = active.fetch_add(1, Ordering::Relaxed) + 1;
record_disk_bucket_scans_active(active_count, &pool, &set);
Self { active, pool, set }
}
}
impl Drop for DiskBucketScanActiveGuard {
fn drop(&mut self) {
let active_count = decrement_atomic_usize(&self.active);
record_disk_bucket_scans_active(active_count, &self.pool, &self.set);
}
}
struct BucketDriveFailureGuard {
failed: bool,
}
impl BucketDriveFailureGuard {
fn new() -> Self {
Self { failed: true }
}
fn mark_not_failed(&mut self) {
self.failed = false;
}
}
impl Drop for BucketDriveFailureGuard {
fn drop(&mut self) {
if self.failed {
global_metrics().record_scan_bucket_drive_failure();
}
}
}
struct DiskBucketScanGaugeReset {
pool: String,
set: String,
}
impl DiskBucketScanGaugeReset {
fn new(pool: String, set: String) -> Self {
Self { pool, set }
}
}
impl Drop for DiskBucketScanGaugeReset {
fn drop(&mut self) {
reset_disk_bucket_scan_gauges(&self.pool, &self.set);
}
}
fn decrement_atomic_usize(counter: &AtomicUsize) -> usize {
counter
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)))
.map(|previous| previous.saturating_sub(1))
.unwrap_or_else(|current| current)
}
fn increment_atomic_usize(counter: &AtomicUsize) -> usize {
counter
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(1)))
.map(|previous| previous.saturating_add(1))
.unwrap_or_else(|current| current)
}
fn record_disk_bucket_scans_queued(count: usize, pool: &str, set: &str) {
metrics::gauge!(
METRIC_SCANNER_DISK_BUCKET_SCANS_QUEUED,
"pool" => pool.to_owned(),
"set" => set.to_owned()
)
.set(count as f64);
global_metrics().record_scanner_disk_bucket_scan_state(pool, set, None, Some(count), None);
}
fn decrement_disk_bucket_scans_queued(counter: &AtomicUsize, pool: &str, set: &str) {
let queued_count = decrement_atomic_usize(counter);
record_disk_bucket_scans_queued(queued_count, pool, set);
}
fn increment_disk_bucket_scans_queued(counter: &AtomicUsize, pool: &str, set: &str) {
let queued_count = increment_atomic_usize(counter);
record_disk_bucket_scans_queued(queued_count, pool, set);
}
fn reset_set_scan_gauges() {
record_set_scan_concurrency_limit(0);
record_set_scans_queued(0);
record_set_scans_active(0);
global_metrics().reset_scanner_set_scan_state();
}
fn reset_disk_bucket_scan_gauges(pool: &str, set: &str) {
record_disk_scan_concurrency_limit(pool, set, 0);
record_disk_bucket_scans_queued(0, pool, set);
record_disk_bucket_scans_active(0, pool, set);
}
fn scanner_concurrency_limit(configured: usize, available: usize) -> usize {
if available == 0 {
return 0;
}
if crate::current_foreground_read_activity() > 0 {
return 1;
}
if configured == 0 {
available
} else {
configured.min(available).max(1)
}
}
fn scanner_max_concurrent_set_scans(available: usize) -> usize {
scanner_concurrency_limit(crate::runtime_config::scanner_max_concurrent_set_scans_configured(), available)
}
fn scanner_max_concurrent_disk_scans(available: usize) -> usize {
scanner_concurrency_limit(crate::runtime_config::scanner_max_concurrent_disk_scans_configured(), available)
}
fn scanner_budgeted_concurrency_limit(configured_limit: usize, requires_serial_progress_accounting: bool) -> usize {
if requires_serial_progress_accounting {
1
} else {
configured_limit
}
}
fn record_set_scan_failure(first_err: &mut Option<Error>, err: Error) {
if first_err.is_none() {
*first_err = Some(err);
}
}
fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError) -> Error {
Error::other(format!("{stage} task join failed: {err}"))
}
fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error>) -> Result<()> {
if results.iter().any(|result| result.info.last_update.is_some()) {
return Ok(());
}
if let Some(err) = first_err {
return Err(err);
}
Ok(())
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ScannerBucketScanStatus {
Complete,
Failed,
Partial,
NamespaceNotFound,
}
fn scanner_bucket_scan_status(has_failed: bool, has_partial: bool, has_namespace_not_found: bool) -> ScannerBucketScanStatus {
if has_failed {
ScannerBucketScanStatus::Failed
} else if has_partial {
ScannerBucketScanStatus::Partial
} else if has_namespace_not_found {
ScannerBucketScanStatus::NamespaceNotFound
} else {
ScannerBucketScanStatus::Complete
}
}
fn classify_nsscanner_cycle(
completed_all_sets: bool,
budget_elapsed: bool,
cancelled: bool,
bucket_scan_status: ScannerBucketScanStatus,
dirty_usage_status: DirtyUsageSnapshotStatus,
activity_status: ScannerCycleActivityStatus,
) -> ScannerCycleStatus {
if budget_elapsed
|| cancelled
|| !matches!(bucket_scan_status, ScannerBucketScanStatus::Complete)
|| dirty_usage_status == DirtyUsageSnapshotStatus::Unverified
{
return ScannerCycleStatus::Incomplete;
}
if !completed_all_sets {
return ScannerCycleStatus::Incomplete;
}
match (activity_status, dirty_usage_status) {
(ScannerCycleActivityStatus::Unchanged, DirtyUsageSnapshotStatus::Current) => ScannerCycleStatus::Complete,
(ScannerCycleActivityStatus::Unverified, _) => ScannerCycleStatus::Incomplete,
_ => ScannerCycleStatus::Superseded,
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ScannerCycleActivityStatus {
Unchanged,
Changed,
Unverified,
}
async fn scanner_cycle_activity_status(
store: &ECStore,
distributed: bool,
before: &crate::scanner::ScannerActivitySnapshot,
) -> ScannerCycleActivityStatus {
match crate::scanner::probe_scanner_activity(store, distributed).await {
Ok(after) if after == *before => ScannerCycleActivityStatus::Unchanged,
Ok(_) => ScannerCycleActivityStatus::Changed,
Err(err) => {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "cycle_activity_probe_failed",
error = %err,
"Scanner cycle activity verification failed"
);
ScannerCycleActivityStatus::Unverified
}
}
}
fn scanner_results_have_pending_maintenance_work(results: &[DataUsageCache]) -> bool {
results.iter().any(|result| !result.info.pending_heals.is_empty())
}
fn pending_maintenance_work_for_cycle(pending: &AtomicBool, results: &[DataUsageCache]) -> bool {
pending.load(Ordering::Acquire) || scanner_results_have_pending_maintenance_work(results)
}
fn record_bucket_pending_maintenance_work(cache: &DataUsageCache, pending: &AtomicBool) {
if !cache.info.pending_heals.is_empty() {
pending.store(true, Ordering::Release);
}
}
fn is_xl_meta_path(path: &str) -> bool {
Path::new(path)
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name == STORAGE_FORMAT_FILE)
}
pub(crate) fn cache_root_entry_info(cache: &DataUsageCache) -> std::result::Result<DataUsageEntryInfo, ScannerError> {
if cache.info.name.is_empty() {
return Err(ScannerError::Other("scanner cache root name is empty".to_string()));
}
let entry = cache
.checked_flatten_complete_scope(&cache.info.name)
.ok_or_else(|| ScannerError::Other(format!("scanner cache root is missing or corrupt: {}", cache.info.name)))?;
Ok(DataUsageEntryInfo {
name: cache.info.name.clone(),
parent: DATA_USAGE_ROOT.to_string(),
entry,
})
}
fn apply_bucket_result_to_cache(cache: &mut DataUsageCache, result: DataUsageEntryInfo, update_time: SystemTime) {
cache.replace(&result.name, &result.parent, result.entry);
cache.info.last_update = Some(update_time);
}
fn should_publish_completed_snapshot(completed_count: usize, total_count: usize, budget_elapsed: bool, cancelled: bool) -> bool {
completed_count == total_count && !budget_elapsed && !cancelled
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum NamespaceScannerWorkerMode {
Coordinator,
RemoteV4(uuid::Uuid),
}
fn namespace_scanner_workers<T>(
coordinator_disks: Vec<T>,
remote_disks: Vec<(T, uuid::Uuid)>,
) -> Vec<(T, NamespaceScannerWorkerMode)> {
let mut workers = Vec::with_capacity(coordinator_disks.len() + remote_disks.len());
workers.extend(
coordinator_disks
.into_iter()
.map(|disk| (disk, NamespaceScannerWorkerMode::Coordinator)),
);
workers.extend(
remote_disks
.into_iter()
.map(|(disk, server_epoch)| (disk, NamespaceScannerWorkerMode::RemoteV4(server_epoch))),
);
workers
}
fn group_remote_disks_by_peer<T>(disks: Vec<T>, peer_key: impl Fn(&T) -> String) -> Vec<Vec<T>> {
let mut groups = HashMap::<String, Vec<T>>::new();
for disk in disks {
groups.entry(peer_key(&disk)).or_default().push(disk);
}
groups.into_values().collect()
}
fn scanner_results_match_scan_scope(results: &[DataUsageCache], expected_sources: &HashSet<DataUsageCacheSource>) -> bool {
if results.is_empty() {
return false;
}
let sources_match_topology = results
.iter()
.map(|result| result.info.source)
.collect::<Option<HashSet<_>>>()
.is_some_and(|sources| sources.len() == results.len() && sources == *expected_sources);
let plan_digests = results
.iter()
.map(|result| result.info.scan_plan_digest)
.collect::<Option<HashSet<_>>>();
let cycles = results.iter().map(|result| result.info.next_cycle).collect::<HashSet<_>>();
let leader_epochs = results.iter().map(|result| result.info.leader_epoch).collect::<HashSet<_>>();
sources_match_topology
&& plan_digests.is_some_and(|digests| digests.len() == 1)
&& cycles.len() == 1
&& leader_epochs.len() == 1
}
fn scanner_results_form_complete_snapshot(results: &[DataUsageCache], expected_sources: &HashSet<DataUsageCacheSource>) -> bool {
results
.iter()
.all(|result| result.info.last_update.is_some() && result.info.snapshot_complete)
&& scanner_results_match_scan_scope(results, expected_sources)
}
fn checked_bucket_usage_info(entry: &DataUsageEntry) -> Option<BucketUsageInfo> {
let mut usage = BucketUsageInfo {
size: u64::try_from(entry.size).ok()?,
versions_count: u64::try_from(entry.versions).ok()?,
objects_count: u64::try_from(entry.objects).ok()?,
delete_markers_count: u64::try_from(entry.delete_markers).ok()?,
object_size_histogram: entry.obj_sizes.to_map(),
object_versions_histogram: entry.obj_versions.to_map(),
..Default::default()
};
if let Some(replication) = &entry.replication_stats {
usage.replica_size = replication.replica_size;
usage.replica_count = replication.replica_count;
for (target, stats) in &replication.targets {
usage.replication_info.insert(
target.clone(),
BucketTargetUsageInfo {
replication_pending_size: stats.pending_size,
replicated_size: stats.replicated_size,
replication_failed_size: stats.failed_size,
replication_pending_count: stats.pending_count,
replication_failed_count: stats.failed_count,
replicated_count: stats.replicated_count,
..Default::default()
},
);
}
}
Some(usage)
}
pub(crate) fn scanner_cache_lock_resource(cache_name: &str, source: DataUsageCacheSource) -> String {
let lock_name = format!("{SCANNER_CACHE_LOCK_SUFFIX}.pool-{}.set-{}", source.pool_index, source.set_index);
path_join_buf(&[crate::BUCKET_META_PREFIX, cache_name, &lock_name])
}
pub(crate) fn scanner_cache_lock_timeout() -> Duration {
Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
}
#[derive(Debug)]
pub(crate) struct ScannerCacheLockGuards {
scoped: NamespaceLockGuard,
}
impl ScannerCacheLockGuards {
pub(crate) fn is_lock_lost(&self) -> bool {
self.scoped.is_lock_lost()
}
}
#[derive(Debug)]
pub(crate) enum ScannerCacheLockError {
Create { resource: String, source: Error },
Acquire { resource: String, source: LockError },
}
impl ScannerCacheLockError {
pub(crate) fn state(&self) -> &'static str {
match self {
Self::Create { .. } => "lock_create_failed",
Self::Acquire { .. } => "lock_acquire_failed",
}
}
pub(crate) fn is_contention(&self) -> bool {
matches!(
self,
Self::Acquire {
source: LockError::Timeout { .. } | LockError::AlreadyLocked { .. },
..
}
)
}
}
impl std::fmt::Display for ScannerCacheLockError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Create { resource, source } => write!(formatter, "create scanner cache lock {resource}: {source}"),
Self::Acquire { resource, source } => write!(formatter, "acquire scanner cache lock {resource}: {source}"),
}
}
}
pub(crate) async fn acquire_scanner_cache_locks(
store: &SetDisks,
cache_name: &str,
source: DataUsageCacheSource,
) -> std::result::Result<ScannerCacheLockGuards, ScannerCacheLockError> {
let timeout = scanner_cache_lock_timeout();
let scoped_resource = scanner_cache_lock_resource(cache_name, source);
let scoped_lock = store
.new_ns_lock(RUSTFS_META_BUCKET, &scoped_resource)
.await
.map_err(|source| ScannerCacheLockError::Create {
resource: scoped_resource.clone(),
source,
})?;
let scoped = scoped_lock
.get_write_lock_quiet(timeout)
.await
.map_err(|source| ScannerCacheLockError::Acquire {
resource: scoped_resource,
source,
})?;
Ok(ScannerCacheLockGuards { scoped })
}
async fn await_scanner_disk_shutdown<F>(scan: Pin<&mut F>)
where
F: Future,
{
let _ = tokio::time::timeout(SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan).await;
}
pub(crate) fn current_cache_root_entry(
cache: &DataUsageCache,
name: &str,
source: DataUsageCacheSource,
next_cycle: u64,
leader_epoch: u64,
scan_plan_digest: DataUsageScanPlanDigest,
) -> std::result::Result<Option<DataUsageEntryInfo>, ScannerError> {
let metadata_is_current = cache.info.name == name
&& cache.info.source == Some(source)
&& cache.info.snapshot_complete
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
&& cache.info.last_update.is_some()
&& cache.info.next_cycle == next_cycle
&& cache.info.leader_epoch == leader_epoch
&& cache.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT;
if !metadata_is_current {
return Ok(None);
}
cache_root_entry_info(cache).map(Some)
}
pub(crate) enum DataUsageCacheScanState {
Current(Box<DataUsageEntryInfo>),
Prepared {
outcome: DataUsageCachePrepareOutcome,
invalid_current: Option<ScannerError>,
},
}
pub(crate) fn current_cache_root_or_prepare(
cache: &mut DataUsageCache,
name: &str,
source: DataUsageCacheSource,
next_cycle: u64,
leader_epoch: u64,
scan_plan_digest: DataUsageScanPlanDigest,
require_source: bool,
) -> DataUsageCacheScanState {
match current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest) {
Ok(Some(root)) => DataUsageCacheScanState::Current(Box::new(root)),
current => DataUsageCacheScanState::Prepared {
invalid_current: current.err(),
outcome: cache.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, require_source),
},
}
}
#[cfg(test)]
fn cache_snapshot_is_current(
cache: &DataUsageCache,
name: &str,
source: DataUsageCacheSource,
next_cycle: u64,
leader_epoch: u64,
scan_plan_digest: DataUsageScanPlanDigest,
) -> bool {
matches!(
current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest),
Ok(Some(_))
)
}
fn completed_data_usage_info(
results: &[DataUsageCache],
expected_sources: &HashSet<DataUsageCacheSource>,
all_buckets: &[String],
bucket_plan_complete: bool,
budget_elapsed: bool,
cancelled: bool,
) -> Option<(DataUsageInfo, SystemTime)> {
if !bucket_plan_complete {
return None;
}
let completed_set_count = results.iter().filter(|result| result.info.last_update.is_some()).count();
if !should_publish_completed_snapshot(completed_set_count, results.len(), budget_elapsed, cancelled) {
return None;
}
if !scanner_results_form_complete_snapshot(results, expected_sources) {
return None;
}
if results.iter().any(|result| result.root().is_none()) {
return None;
}
let mut total = DataUsageEntry::default();
let mut buckets_usage = HashMap::with_capacity(all_buckets.len());
for bucket in all_buckets {
let mut merged = DataUsageEntry::default();
for result in results {
let entry = result.checked_flatten(bucket)?;
if !merged.checked_merge(&entry) {
return None;
}
}
if !total.checked_merge(&merged) {
return None;
}
buckets_usage.insert(bucket.clone(), checked_bucket_usage_info(&merged)?);
}
let merged_last_update = results.iter().filter_map(|result| result.info.last_update).max()?;
let bucket_sizes = buckets_usage
.iter()
.map(|(bucket, usage)| (bucket.clone(), usage.size))
.collect();
let data_usage_info = DataUsageInfo {
last_update: Some(merged_last_update),
scanner_cycle: Some(results.first()?.info.next_cycle),
objects_total_count: u64::try_from(total.objects).ok()?,
versions_total_count: u64::try_from(total.versions).ok()?,
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
objects_total_size: u64::try_from(total.size).ok()?,
tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()),
buckets_count: u64::try_from(all_buckets.len()).ok()?,
bucket_sizes,
buckets_usage,
usage_snapshot_complete: true,
..Default::default()
};
Some((data_usage_info, merged_last_update))
}
#[cfg(test)]
mod publish_gate_tests {
use super::*;
use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]);
#[test]
fn should_publish_completed_snapshot_requires_full_clean_cycle() {
assert!(should_publish_completed_snapshot(3, 3, false, false));
assert!(!should_publish_completed_snapshot(2, 3, false, false));
assert!(!should_publish_completed_snapshot(3, 3, true, false));
assert!(!should_publish_completed_snapshot(3, 3, false, true));
assert!(
should_publish_completed_snapshot(0, 0, false, false),
"a completed empty namespace is an authoritative zero snapshot"
);
}
fn incomplete_scope_cache(source: DataUsageCacheSource) -> DataUsageCache {
DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
}
}
#[test]
fn incomplete_scan_scope_requires_every_expected_set_marker() {
let first_source = DataUsageCacheSource::new(0, 0);
let second_source = DataUsageCacheSource::new(1, 0);
let expected_sources = HashSet::from([first_source, second_source]);
let first = incomplete_scope_cache(first_source);
let second = incomplete_scope_cache(second_source);
assert!(scanner_results_match_scan_scope(&[first.clone(), second], &expected_sources));
assert!(!scanner_results_form_complete_snapshot(
&[first.clone(), incomplete_scope_cache(second_source)],
&expected_sources
));
assert!(!scanner_results_match_scan_scope(
&[first.clone(), DataUsageCache::default()],
&expected_sources
));
assert!(!scanner_results_match_scan_scope(
&[first.clone(), incomplete_scope_cache(first_source)],
&expected_sources
));
let mut mismatched_plan = incomplete_scope_cache(second_source);
mismatched_plan.info.scan_plan_digest = Some(DataUsageScanPlanDigest([8; 32]));
assert!(!scanner_results_match_scan_scope(&[first, mismatched_plan], &expected_sources));
}
fn completed_root_cache(bucket: &str, objects: usize, update_secs: u64, source: DataUsageCacheSource) -> DataUsageCache {
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(update_secs)),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.replace(
bucket,
DATA_USAGE_ROOT,
DataUsageEntry {
objects,
size: objects.saturating_mul(10),
..Default::default()
},
);
cache
}
fn completed_data_usage_info_for_test(
results: &[DataUsageCache],
all_buckets: &[String],
budget_elapsed: bool,
cancelled: bool,
) -> Option<(DataUsageInfo, SystemTime)> {
let expected_sources = results.iter().filter_map(|result| result.info.source).collect::<HashSet<_>>();
completed_data_usage_info(results, &expected_sources, all_buckets, true, budget_elapsed, cancelled)
}
#[test]
fn completed_data_usage_info_publishes_tier_stats_across_sets() {
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()];
let warm = |total_size, num_versions, num_objects| {
HashMap::from([(
"WARM".to_string(),
TierStats {
total_size,
num_versions,
num_objects,
},
)])
};
let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
let mut tiered = DataUsageEntry::default();
tiered.add_tier_sizes(&warm(100, 2, 1));
first_set.replace("bucket-b", DATA_USAGE_ROOT, tiered);
let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0));
let mut tiered = DataUsageEntry::default();
tiered.add_tier_sizes(&warm(50, 1, 1));
second_set.replace("bucket-a", DATA_USAGE_ROOT, tiered);
let (data_usage_info, _) = completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false)
.expect("completed sets should publish a snapshot");
assert_eq!(
data_usage_info
.tier_stats
.expect("tier usage should reach the snapshot")
.tiers["WARM"],
TierStats {
total_size: 150,
num_versions: 3,
num_objects: 2,
}
);
}
#[test]
fn completed_data_usage_info_omits_tier_stats_without_tiered_objects() {
let all_buckets = vec!["bucket-a".to_string()];
let set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
let (data_usage_info, _) = completed_data_usage_info_for_test(&[set], &all_buckets, false, false)
.expect("completed set should publish a snapshot");
assert!(data_usage_info.tier_stats.is_none());
}
#[test]
fn completed_data_usage_info_requires_every_set_before_publish() {
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string(), "bucket-empty".to_string()];
let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
first_set.replace("bucket-b", DATA_USAGE_ROOT, DataUsageEntry::default());
first_set.replace("bucket-empty", DATA_USAGE_ROOT, DataUsageEntry::default());
let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0));
second_set.replace("bucket-a", DATA_USAGE_ROOT, DataUsageEntry::default());
second_set.replace("bucket-empty", DATA_USAGE_ROOT, DataUsageEntry::default());
assert!(
completed_data_usage_info_for_test(&[first_set.clone(), DataUsageCache::default()], &all_buckets, false, false)
.is_none()
);
assert!(
completed_data_usage_info_for_test(&[first_set.clone(), second_set.clone()], &all_buckets, true, false).is_none()
);
assert!(
completed_data_usage_info_for_test(&[first_set.clone(), second_set.clone()], &all_buckets, false, true).is_none()
);
let (data_usage_info, last_update) =
completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false)
.expect("all completed sets should produce a publishable data usage snapshot");
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
assert_eq!(data_usage_info.scanner_cycle, Some(0));
assert_eq!(data_usage_info.objects_total_count, 3);
assert_eq!(data_usage_info.buckets_usage.len(), 3);
assert!(data_usage_info.usage_snapshot_complete);
assert_eq!(
data_usage_info
.buckets_usage
.get("bucket-empty")
.map(|usage| (usage.objects_count, usage.size)),
Some((0, 0))
);
}
#[test]
fn completed_data_usage_info_publishes_confirmed_empty_namespace() {
let mut completed_set = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
source: Some(DataUsageCacheSource::new(0, 0)),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
..Default::default()
},
..Default::default()
};
completed_set.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
let (data_usage_info, _) = completed_data_usage_info_for_test(&[completed_set], &[], false, false)
.expect("a completed empty namespace should produce an authoritative snapshot");
assert!(data_usage_info.is_complete_bucket_usage_snapshot());
assert_eq!(data_usage_info.buckets_count, 0);
assert!(data_usage_info.buckets_usage.is_empty());
}
#[test]
fn complete_usage_candidate_with_changed_generation_is_superseded() {
let all_buckets = vec!["bucket".to_string()];
let completed_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
let completed_usage = completed_data_usage_info_for_test(&[completed_set], &all_buckets, false, false);
assert!(completed_usage.is_some());
assert_eq!(
classify_nsscanner_cycle(
completed_usage.is_some(),
false,
false,
ScannerBucketScanStatus::Complete,
DirtyUsageSnapshotStatus::Changed,
ScannerCycleActivityStatus::Unchanged,
),
ScannerCycleStatus::Superseded
);
}
#[test]
fn completed_data_usage_info_adds_same_bucket_across_unique_sets() {
let all_buckets = vec!["bucket".to_string()];
let first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
let second_entry = second_set.find("bucket").cloned().expect("second set bucket entry");
second_set.replace(
"bucket",
DATA_USAGE_ROOT,
DataUsageEntry {
versions: 4,
delete_markers: 1,
..second_entry
},
);
let (data_usage_info, last_update) =
completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false)
.expect("unique completed set snapshots should be aggregated");
let bucket = data_usage_info.buckets_usage.get("bucket").expect("merged bucket usage");
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
assert_eq!(data_usage_info.objects_total_count, 5);
assert_eq!(data_usage_info.objects_total_size, 50);
assert_eq!(bucket.objects_count, 5);
assert_eq!(bucket.size, 50);
assert_eq!(bucket.versions_count, 4);
assert_eq!(bucket.delete_markers_count, 1);
}
#[test]
fn completed_data_usage_info_flattens_nested_bucket_entries() {
let all_buckets = vec!["bucket".to_string()];
let mut first_set = completed_root_cache("bucket", 1, 10, DataUsageCacheSource::new(0, 0));
let mut nested = DataUsageEntry {
objects: 2,
versions: 3,
size: 2048,
replication_stats: Some(ReplicationAllStats {
targets: HashMap::from([(
"arn:target".to_string(),
ReplicationStats {
replicated_size: 2048,
replicated_count: 2,
..Default::default()
},
)]),
replica_size: 2048,
replica_count: 2,
}),
..Default::default()
};
nested.obj_sizes.add(2048);
nested.obj_versions.add(3);
first_set.replace("bucket/prefix", "bucket", nested);
let second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
let (data_usage_info, _) = completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false)
.expect("nested bucket entries should be flattened before aggregation");
let bucket = data_usage_info.buckets_usage.get("bucket").expect("merged bucket usage");
assert_eq!(data_usage_info.objects_total_count, 6);
assert_eq!(data_usage_info.objects_total_size, 2088);
assert_eq!(bucket.objects_count, 6);
assert_eq!(bucket.versions_count, 3);
assert_eq!(bucket.object_size_histogram["BETWEEN_1024_B_AND_64_KB"], 1);
assert_eq!(bucket.object_versions_histogram["BETWEEN_2_AND_10"], 1);
assert_eq!(bucket.replica_size, 2048);
assert_eq!(bucket.replica_count, 2);
assert_eq!(bucket.replication_info["arn:target"].replicated_size, 2048);
assert_eq!(bucket.replication_info["arn:target"].replicated_count, 2);
}
#[test]
fn completed_data_usage_info_rejects_cyclic_bucket_entries() {
let all_buckets = vec!["bucket".to_string()];
let mut cache = completed_root_cache("bucket", 1, 10, DataUsageCacheSource::new(0, 0));
cache.replace(
"bucket/prefix",
"bucket",
DataUsageEntry {
objects: 1,
size: 10,
..Default::default()
},
);
cache
.cache
.get_mut(&crate::hash_path("bucket/prefix").key())
.expect("nested entry")
.add_child(&crate::hash_path("bucket"));
assert!(completed_data_usage_info_for_test(&[cache], &all_buckets, false, false).is_none());
}
#[test]
fn completed_data_usage_info_rejects_duplicate_or_incomplete_set_sources() {
let all_buckets = vec!["bucket".to_string()];
let source = DataUsageCacheSource::new(0, 0);
let first_set = completed_root_cache("bucket", 2, 10, source);
let duplicate_set = completed_root_cache("bucket", 3, 20, source);
assert!(completed_data_usage_info_for_test(&[first_set.clone(), duplicate_set], &all_buckets, false, false).is_none());
let mut incomplete_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
incomplete_set.info.snapshot_complete = false;
assert!(completed_data_usage_info_for_test(&[first_set, incomplete_set], &all_buckets, false, false).is_none());
}
#[test]
fn completed_data_usage_info_requires_exact_topology_sources() {
let all_buckets = vec!["bucket".to_string()];
let first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
let unexpected_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(99, 99));
let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0), DataUsageCacheSource::new(1, 0)]);
assert!(
completed_data_usage_info(&[first_set, unexpected_set], &expected_sources, &all_buckets, true, false, false)
.is_none()
);
}
#[test]
fn completed_data_usage_info_rejects_incomplete_bucket_plan() {
let all_buckets = vec!["bucket".to_string()];
let set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0)]);
assert!(completed_data_usage_info(&[set], &expected_sources, &all_buckets, false, false, false).is_none());
}
#[test]
fn completed_data_usage_info_rejects_missing_bucket_from_any_set() {
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()];
let mut complete_set = completed_root_cache("bucket-a", 2, 10, DataUsageCacheSource::new(0, 0));
complete_set.replace("bucket-b", DATA_USAGE_ROOT, DataUsageEntry::default());
let missing_bucket_set = completed_root_cache("bucket-a", 3, 20, DataUsageCacheSource::new(1, 0));
assert!(completed_data_usage_info_for_test(&[complete_set, missing_bucket_set], &all_buckets, false, false).is_none());
}
#[test]
fn completed_data_usage_info_rejects_mixed_scan_plans() {
let all_buckets = vec!["bucket".to_string()];
let first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
second_set.info.scan_plan_digest = Some(DataUsageScanPlanDigest([8; 32]));
assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none());
}
#[test]
fn completed_data_usage_info_rejects_mixed_scanner_cycles() {
let all_buckets = vec!["bucket".to_string()];
let mut first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
first_set.info.next_cycle = 12;
second_set.info.next_cycle = 13;
assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none());
}
#[test]
fn completed_data_usage_info_rejects_mixed_leader_epochs() {
let all_buckets = vec!["bucket".to_string()];
let mut first_set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
let mut second_set = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
first_set.info.leader_epoch = 11;
second_set.info.leader_epoch = 12;
assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none());
}
#[test]
fn completed_data_usage_info_rejects_counter_overflow() {
let all_buckets = vec!["bucket".to_string()];
let first_set = completed_root_cache("bucket", usize::MAX, 10, DataUsageCacheSource::new(0, 0));
let second_set = completed_root_cache("bucket", 1, 20, DataUsageCacheSource::new(1, 0));
assert!(completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false).is_none());
}
#[test]
fn current_cache_snapshot_requires_matching_complete_source_and_cycle() {
let source = DataUsageCacheSource::new(1, 2);
let mut cache = completed_root_cache("bucket", 1, 10, source);
cache.info.next_cycle = 10;
assert!(cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 0, TEST_PLAN_DIGEST));
assert!(!cache_snapshot_is_current(&cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST));
assert!(!cache_snapshot_is_current(
&cache,
DATA_USAGE_ROOT,
DataUsageCacheSource::new(2, 1),
10,
0,
TEST_PLAN_DIGEST
));
assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 11, 0, TEST_PLAN_DIGEST));
assert!(!cache_snapshot_is_current(
&cache,
DATA_USAGE_ROOT,
source,
10,
0,
DataUsageScanPlanDigest([8; 32])
));
cache.info.leader_epoch = 2;
assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 1, TEST_PLAN_DIGEST));
assert!(cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 2, TEST_PLAN_DIGEST));
cache.info.next_cycle = 11;
assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 2, TEST_PLAN_DIGEST));
}
#[test]
fn current_cache_snapshot_rejects_persisted_windows_key_mismatch() {
let source = DataUsageCacheSource::new(1, 2);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 10,
last_update: Some(SystemTime::UNIX_EPOCH),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.cache.insert(
"bucket".to_string(),
DataUsageEntry {
children: HashSet::from(["bucket/prefix".to_string()]),
..Default::default()
},
);
cache.cache.insert(
"bucket\\prefix".to_string(),
DataUsageEntry {
objects: 3,
..Default::default()
},
);
assert!(!cache_snapshot_is_current(&cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST));
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
DataUsageCacheScanState::Prepared {
outcome: DataUsageCachePrepareOutcome::Reset,
invalid_current: Some(_),
} => {}
_ => panic!("an invalid current cache must enter the rebuild path"),
}
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn current_cache_snapshot_rejects_structurally_valid_legacy_key_format() {
let source = DataUsageCacheSource::new(1, 2);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 10,
last_update: Some(SystemTime::UNIX_EPOCH),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
..Default::default()
},
..Default::default()
};
cache.cache.insert(
"bucket".to_string(),
DataUsageEntry {
children: HashSet::from(["bucket\\prefix".to_string()]),
..Default::default()
},
);
cache.cache.insert(
"bucket\\prefix".to_string(),
DataUsageEntry {
objects: 3,
..Default::default()
},
);
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(3));
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
DataUsageCacheScanState::Prepared {
outcome: DataUsageCachePrepareOutcome::Reset,
invalid_current: None,
} => {}
_ => panic!("a legacy key format must enter the rebuild path"),
}
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn current_cache_snapshot_rejects_current_bucket_cache_with_detached_entry() {
let source = DataUsageCacheSource::new(1, 2);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 10,
last_update: Some(SystemTime::UNIX_EPOCH),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.cache.insert(
"bucket".to_string(),
DataUsageEntry {
objects: 1,
..Default::default()
},
);
cache.cache.insert(
"bucket/detached".to_string(),
DataUsageEntry {
objects: 2,
..Default::default()
},
);
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(1));
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
DataUsageCacheScanState::Prepared {
outcome: DataUsageCachePrepareOutcome::Reset,
invalid_current: Some(_),
} => {}
_ => panic!("a detached complete bucket cache must enter the rebuild path"),
}
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn namespace_scanner_worker_selection_keeps_coordinator_fallback_disks() {
let server_epoch = uuid::Uuid::new_v4();
let workers = namespace_scanner_workers(vec!["local", "legacy-remote"], vec![("v4", server_epoch)]);
assert_eq!(
workers,
vec![
("local", NamespaceScannerWorkerMode::Coordinator),
("legacy-remote", NamespaceScannerWorkerMode::Coordinator),
("v4", NamespaceScannerWorkerMode::RemoteV4(server_epoch)),
]
);
assert!(namespace_scanner_workers::<()>(Vec::new(), Vec::new()).is_empty());
}
#[test]
fn remote_scanner_capability_probes_are_grouped_by_peer() {
let mut groups = group_remote_disks_by_peer(vec![("node-a", 0), ("node-b", 0), ("node-a", 1), ("node-b", 1)], |disk| {
disk.0.to_string()
});
groups.sort_by_key(|group| group[0].0);
assert_eq!(groups.len(), 2);
assert_eq!(groups[0], vec![("node-a", 0), ("node-a", 1)]);
assert_eq!(groups[1], vec![("node-b", 0), ("node-b", 1)]);
}
#[test]
fn scanner_bucket_plan_digest_is_order_independent_and_membership_sensitive() {
let activity_digest = [4; 32];
let buckets = vec![
BucketInfo {
name: "photos".to_string(),
..Default::default()
},
BucketInfo {
name: "videos".to_string(),
..Default::default()
},
];
let reversed = vec![buckets[1].clone(), buckets[0].clone()];
let changed = vec![
buckets[0].clone(),
BucketInfo {
name: "archives".to_string(),
..Default::default()
},
];
let mut regenerated = buckets.clone();
regenerated[0].created = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1));
assert_eq!(
scanner_bucket_plan_digest(&buckets, activity_digest),
scanner_bucket_plan_digest(&reversed, activity_digest)
);
assert_ne!(
scanner_bucket_plan_digest(&buckets, activity_digest),
scanner_bucket_plan_digest(&changed, activity_digest)
);
assert_ne!(
scanner_bucket_plan_digest(&buckets, activity_digest),
scanner_bucket_plan_digest(&regenerated, activity_digest)
);
assert_ne!(
scanner_bucket_plan_digest(&buckets, activity_digest),
scanner_bucket_plan_digest(&buckets, [5; 32])
);
}
#[test]
fn dirty_bucket_cache_digest_changes_with_generation() {
let source = DataUsageCacheSource::new(0, 0);
let plan = DataUsageScanPlanDigest([9; 32]);
let first = scanner_bucket_cache_digest(plan, Some(7));
let second = scanner_bucket_cache_digest(plan, Some(8));
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "photos".to_string(),
next_cycle: 11,
last_update: Some(SystemTime::now()),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(first),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.replace("photos", "", DataUsageEntry::default());
assert_eq!(scanner_bucket_cache_digest(plan, None), plan);
assert!(cache_snapshot_is_current(&cache, "photos", source, 11, 0, first));
assert!(!cache_snapshot_is_current(&cache, "photos", source, 11, 0, second));
}
#[test]
fn scanner_cache_lock_resource_is_scoped_to_cache_source() {
let cache_name = "photos/.usage-cache.bin";
let first_source = DataUsageCacheSource::new(0, 1);
let same_source = DataUsageCacheSource::new(0, 1);
let other_source = DataUsageCacheSource::new(1, 0);
let first = scanner_cache_lock_resource(cache_name, first_source);
assert_eq!(first, scanner_cache_lock_resource(cache_name, same_source));
assert_ne!(first, scanner_cache_lock_resource(cache_name, other_source));
assert!(first.ends_with(".scanner-cycle.lock.pool-0.set-1"));
}
#[test]
fn count_budget_serializes_set_and_disk_work() {
assert_eq!(scanner_budgeted_concurrency_limit(8, true), 1);
assert_eq!(scanner_budgeted_concurrency_limit(8, false), 8);
}
#[test]
fn requeued_bucket_work_is_only_completed_after_retry() {
let remaining = Arc::new(AtomicUsize::new(1));
let complete = CancellationToken::new();
let mut first = BucketWorkGuard::new(remaining.clone(), complete.clone());
first.mark_requeued();
drop(first);
assert_eq!(remaining.load(Ordering::Acquire), 1);
assert!(!complete.is_cancelled());
drop(BucketWorkGuard::new(remaining.clone(), complete.clone()));
assert_eq!(remaining.load(Ordering::Acquire), 0);
assert!(complete.is_cancelled());
}
#[tokio::test]
async fn exhausted_workers_mark_all_queued_bucket_work_failed() {
let (tx, rx) = mpsc::channel(2);
tx.send(BucketInfo {
name: "photos".to_string(),
..Default::default()
})
.await
.expect("queue photos");
tx.send(BucketInfo {
name: "videos".to_string(),
..Default::default()
})
.await
.expect("queue videos");
drop(tx);
let receiver = Mutex::new(rx);
let remaining = Arc::new(AtomicUsize::new(2));
let complete = CancellationToken::new();
let failed = Arc::new(Mutex::new(HashSet::new()));
let failed_count = mark_unprocessed_bucket_work_failed(&receiver, &remaining, &complete, &failed).await;
assert_eq!(failed_count, 2);
assert_eq!(remaining.load(Ordering::Acquire), 0);
assert!(complete.is_cancelled());
assert_eq!(*failed.lock().await, HashSet::from(["photos".to_string(), "videos".to_string()]));
}
#[tokio::test]
async fn requeued_bucket_remains_pending_for_another_worker() {
let (tx, mut rx) = mpsc::channel(1);
let remaining = Arc::new(AtomicUsize::new(1));
let complete = CancellationToken::new();
let mut guard = BucketWorkGuard::new(remaining.clone(), complete.clone());
let bucket = BucketInfo {
name: "photos".to_string(),
..Default::default()
};
assert!(requeue_bucket_work(&tx, &bucket, &mut guard).await);
drop(guard);
assert_eq!(remaining.load(Ordering::Acquire), 1);
assert!(!complete.is_cancelled());
assert_eq!(rx.recv().await.expect("requeued bucket").name, "photos");
}
}
async fn send_cache_root_entry_info(
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
cache: &DataUsageCache,
pending_maintenance_work: &AtomicBool,
) -> std::result::Result<(), ScannerError> {
let root = cache_root_entry_info(cache)?;
send_cache_root_entry(bucket_result_tx, root, cache, pending_maintenance_work).await
}
async fn send_cache_root_entry(
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
root: DataUsageEntryInfo,
cache: &DataUsageCache,
pending_maintenance_work: &AtomicBool,
) -> std::result::Result<(), ScannerError> {
record_bucket_pending_maintenance_work(cache, pending_maintenance_work);
bucket_result_tx
.send(root)
.await
.map_err(|err| ScannerError::Other(format!("scanner cache root channel closed: {err}")))
}
async fn persist_and_publish_cache_snapshot(
store: Arc<SetDisks>,
updates: &mpsc::Sender<DataUsageCache>,
mut cache_snapshot: DataUsageCache,
cache_cycle_floor: &AtomicU64,
) -> Option<SystemTime> {
let source = cache_snapshot.info.source?;
let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await {
Ok(guard) => guard,
Err(err) => {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = err.state(),
error = %err,
"Scanner cache snapshot lock acquisition failed"
);
return None;
}
};
let mut persisted = DataUsageCache::default();
let revisions = match persisted.load_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME).await {
Ok(revisions) => revisions,
Err(err) => {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "load_or_revision_lookup_failed",
error = %err,
"Scanner cache snapshot load or revision lookup failed"
);
return None;
}
};
let scan_plan_digest = cache_snapshot.info.scan_plan_digest?;
if persisted.info.next_cycle > cache_snapshot.info.next_cycle {
cache_cycle_floor.fetch_max(persisted.info.next_cycle, Ordering::AcqRel);
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
requested_cycle = cache_snapshot.info.next_cycle,
persisted_cycle = persisted.info.next_cycle,
state = "stale_cycle_rejected",
"Scanner rejected a set cache cycle regression"
);
return None;
}
if persisted.info.leader_epoch > cache_snapshot.info.leader_epoch {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
requested_epoch = cache_snapshot.info.leader_epoch,
persisted_epoch = persisted.info.leader_epoch,
state = "stale_leader_rejected",
"Scanner rejected a set cache snapshot from an older leader epoch"
);
return None;
}
if matches!(
current_cache_root_entry(
&persisted,
DATA_USAGE_ROOT,
source,
cache_snapshot.info.next_cycle,
cache_snapshot.info.leader_epoch,
scan_plan_digest,
),
Ok(Some(_))
) {
cache_snapshot = persisted;
} else {
if guard.is_lock_lost() {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "lock_lost",
"Scanner cache snapshot save skipped after lock loss"
);
return None;
}
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache_snapshot
.save_with_revisions(store, DATA_USAGE_CACHE_NAME, &revisions)
.await
{
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "save_failed",
error = %e,
"Scanner cache snapshot persistence failed"
);
done_save();
return None;
}
done_save();
}
if guard.is_lock_lost() {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "lock_lost_after_save",
"Scanner cache snapshot publish skipped after lock loss"
);
return None;
}
drop(guard);
let last_update = cache_snapshot.info.last_update;
if let Err(e) = updates.send(cache_snapshot).await {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "publish_failed",
error = %e,
"Scanner cache snapshot publish failed"
);
}
last_update
}
async fn send_data_usage_update(updates: &mpsc::Sender<DataUsageInfo>, data_usage_info: DataUsageInfo) -> Result<()> {
updates.send(data_usage_info).await.map_err(|e| {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "send_failed",
error = %e,
"Scanner data usage publish failed"
);
StorageError::other("scanner data usage receiver closed before update delivery")
})
}
#[async_trait::async_trait]
pub trait ScannerIO: Send + Sync + Debug + 'static {
async fn nsscanner(
&self,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
updates: mpsc::Sender<DataUsageInfo>,
want_cycle: u64,
scan_mode: HealScanMode,
) -> Result<()>;
}
#[async_trait::async_trait]
pub(crate) trait ScannerIOCycle: Send + Sync + Debug + 'static {
async fn nsscanner_with_status(
&self,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
updates: mpsc::Sender<DataUsageInfo>,
want_cycle: u64,
leader_epoch: u64,
scan_mode: HealScanMode,
) -> Result<ScannerCycleResult>;
}
#[async_trait::async_trait]
pub trait ScannerIOCache: Send + Sync + Debug + 'static {
async fn nsscanner_cache(
self: Arc<Self>,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
scan_plan: ScannerBucketScanPlan,
updates: mpsc::Sender<DataUsageCache>,
want_cycle: u64,
scan_mode: HealScanMode,
) -> Result<()>;
}
#[async_trait::async_trait]
pub trait ScannerIODisk: Send + Sync + Debug + 'static {
async fn nsscanner_disk(
self: Arc<Self>,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
set_disks: Vec<Arc<Disk>>,
cache: DataUsageCache,
updates: Option<mpsc::Sender<DataUsageEntry>>,
scan_mode: HealScanMode,
) -> Result<ScannerDiskScanOutcome>;
async fn get_size(&self, item: ScannerItem) -> Result<SizeSummary>;
}
#[derive(Debug)]
pub enum ScannerDiskScanOutcome {
Complete(DataUsageCache),
Partial(DataUsageCache),
NamespaceNotFound(DataUsageCache),
}
pub(crate) async fn scanner_set_disk_inventory(set: &SetDisks) -> Vec<Arc<Disk>> {
let membership = set.drive_membership_snapshot().await;
let capacity = membership
.online
.len()
.saturating_add(membership.suspect.len())
.saturating_add(membership.returning.len())
.saturating_add(membership.offline.len());
let mut disks = Vec::with_capacity(capacity);
disks.extend(membership.online);
disks.extend(membership.suspect);
disks.extend(membership.returning);
disks.extend(membership.offline);
disks
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ScannerCycleStatus {
Complete,
Incomplete,
Superseded,
}
#[derive(Debug)]
pub(crate) struct ScannerCycleResult {
pub(crate) status: ScannerCycleStatus,
dirty_usage_clear: Option<DirtyUsageBuckets>,
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
failed_dirty_usage: bool,
pending_maintenance_work: bool,
required_cycle_floor: Option<u64>,
}
impl ScannerCycleResult {
pub(crate) fn new(status: ScannerCycleStatus, dirty_usage_clear: Option<DirtyUsageBuckets>) -> Self {
Self {
status,
dirty_usage_clear,
remote_dirty_usage_acknowledgements: Vec::new(),
failed_dirty_usage: false,
pending_maintenance_work: false,
required_cycle_floor: None,
}
}
fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self {
self.failed_dirty_usage = failed_dirty_usage;
self
}
fn with_pending_maintenance_work(mut self, pending_maintenance_work: bool) -> Self {
self.pending_maintenance_work = pending_maintenance_work;
self
}
fn with_required_cycle_floor(mut self, required_cycle_floor: Option<u64>) -> Self {
self.required_cycle_floor = required_cycle_floor;
self
}
pub(crate) fn with_remote_dirty_usage_acknowledgements(
mut self,
acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
) -> Self {
self.remote_dirty_usage_acknowledgements = acknowledgements;
self
}
pub(crate) fn acknowledge_durable_usage(self) -> Vec<crate::scanner::ScannerDirtyUsageAcknowledgement> {
if let Some(snapshot) = self.dirty_usage_clear {
clear_dirty_usage_buckets(&snapshot);
}
self.remote_dirty_usage_acknowledgements
}
pub(crate) fn has_dirty_usage_to_acknowledge(&self) -> bool {
self.dirty_usage_clear.as_ref().is_some_and(|snapshot| !snapshot.is_empty())
|| !self.remote_dirty_usage_acknowledgements.is_empty()
}
pub(crate) fn has_failed_dirty_usage(&self) -> bool {
self.failed_dirty_usage
}
pub(crate) fn has_pending_maintenance_work(&self) -> bool {
self.pending_maintenance_work
}
pub(crate) fn required_cycle_floor(&self) -> Option<u64> {
self.required_cycle_floor
}
}
#[async_trait::async_trait]
impl ScannerIO for ECStore {
async fn nsscanner(
&self,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
updates: mpsc::Sender<DataUsageInfo>,
want_cycle: u64,
scan_mode: HealScanMode,
) -> Result<()> {
// This public path can prove delivery to the receiver, but not that
// the receiver persisted the update. Keep dirty usage pending unless
// the main scanner confirms durability through nsscanner_with_status.
let leader_epoch = crate::scanner::current_scanner_leader_epoch()
.await
.map_err(|err| StorageError::other(format!("failed to resolve scanner leader epoch: {err}")))?;
ScannerIOCycle::nsscanner_with_status(self, ctx, budget, updates, want_cycle, leader_epoch, scan_mode).await?;
Ok(())
}
}
#[async_trait::async_trait]
impl ScannerIOCycle for ECStore {
#[tracing::instrument(skip(self, budget, updates))]
async fn nsscanner_with_status(
&self,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
updates: mpsc::Sender<DataUsageInfo>,
want_cycle: u64,
leader_epoch: u64,
scan_mode: HealScanMode,
) -> Result<ScannerCycleResult> {
let child_token = ctx.child_token();
let distributed = self.setup_is_dist_erasure().await;
let activity_before = match crate::scanner::probe_scanner_activity(self, distributed).await {
Ok(snapshot) => snapshot,
Err(err) => {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "cycle_activity_baseline_failed",
error = %err,
"Scanner cycle skipped because cluster activity could not be baselined"
);
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
}
};
if !crate::scanner::scanner_activity_allows_usage_publication(&activity_before) {
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "cycle_data_movement_active",
"Scanner cycle deferred while rebalance or decommission data movement is active"
);
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
}
let dirty_generation_before_bucket_list = dirty_usage_generation();
let bucket_listing = self.list_bucket_for_scanner(&BucketOptions::default()).await?;
let mut bucket_plan_complete = bucket_listing.topology_complete;
let all_buckets = Arc::new(bucket_listing.buckets);
let expected_sources = Arc::new(
self.pools
.iter()
.flat_map(|pool| {
pool.disk_set
.iter()
.map(|set| DataUsageCacheSource::new(set.pool_index, set.set_index))
})
.collect::<HashSet<_>>(),
);
let mut buckets_by_source = HashMap::with_capacity(bucket_listing.set_buckets.len());
for scope in bucket_listing.set_buckets {
let source = DataUsageCacheSource::new(scope.pool_index, scope.set_index);
if buckets_by_source.insert(source, scope.buckets).is_some() {
bucket_plan_complete = false;
}
}
bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources;
let scan_plan_digest =
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_snapshot_digest(&activity_before));
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle));
if all_buckets.is_empty() {
reset_set_scan_gauges();
if !bucket_plan_complete {
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
}
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
let status = classify_nsscanner_cycle(
true,
false,
ctx.is_cancelled(),
ScannerBucketScanStatus::Complete,
dirty_usage_status,
activity_status,
);
if status != ScannerCycleStatus::Complete {
return Ok(ScannerCycleResult::new(status, None));
}
let empty_usage = DataUsageInfo {
last_update: Some(SystemTime::now()),
scanner_cycle: Some(want_cycle),
usage_snapshot_complete: true,
..Default::default()
};
send_data_usage_update(&updates, empty_usage).await?;
let dirty_usage_clear = Some(dirty_usage_snapshot.buckets.as_ref().clone());
return Ok(
ScannerCycleResult::new(status, dirty_usage_clear).with_remote_dirty_usage_acknowledgements(
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before),
),
);
}
let total_results = expected_sources.len();
if total_results == 0 {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket_count = all_buckets.len(),
state = "no_disk_sets",
"Scanner set state update detected missing disk sets"
);
reset_set_scan_gauges();
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
}
let set_scan_limit = scanner_budgeted_concurrency_limit(
scanner_max_concurrent_set_scans(total_results),
budget.requires_serial_progress_accounting(),
);
let bucket_failures = ScannerBucketFailureState::default();
let pending_maintenance_work = Arc::new(AtomicBool::new(false));
record_set_scan_concurrency_limit(set_scan_limit);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
total_sets = total_results,
concurrency_limit = set_scan_limit,
state = "concurrency_budget",
"Scanner set concurrency budget resolved"
);
let set_scan_semaphore = Arc::new(Semaphore::new(set_scan_limit));
let queued_set_scans = Arc::new(AtomicUsize::new(total_results));
let active_set_scans = Arc::new(AtomicUsize::new(0));
record_set_scans_queued(total_results);
record_set_scans_active(0);
let results = vec![DataUsageCache::default(); total_results];
let results_mutex: Arc<Mutex<Vec<DataUsageCache>>> = Arc::new(Mutex::new(results));
let first_err_mutex: Arc<Mutex<Option<Error>>> = Arc::new(Mutex::new(None));
let mut results_index = 0usize;
let mut wait_futs = Vec::new();
for pool in self.pools.iter() {
for set in pool.disk_set.iter() {
let results_index_clone = results_index;
results_index += 1;
// Clone the Arc to move it into the spawned task
let set_clone: Arc<SetDisks> = Arc::clone(set);
let source = DataUsageCacheSource::new(set.pool_index, set.set_index);
let set_buckets = buckets_by_source.remove(&source).unwrap_or_default();
let pool_label = set.pool_index.to_string();
let set_label = set.set_index.to_string();
let child_token_clone = child_token.clone();
let budget_clone = budget.clone();
let want_cycle_clone = want_cycle;
let scan_mode_clone = scan_mode;
let results_mutex_clone = results_mutex.clone();
let first_err_mutex_clone = first_err_mutex.clone();
let set_scan_semaphore_clone = set_scan_semaphore.clone();
let queued_set_scans_clone = queued_set_scans.clone();
let active_set_scans_clone = active_set_scans.clone();
let (tx, mut rx) = mpsc::channel::<DataUsageCache>(1);
// Spawn task to receive and store results
let receiver_fut = tokio::spawn(async move {
while let Some(result) = rx.recv().await {
let mut results = results_mutex_clone.lock().await;
results[results_index_clone] = result;
}
});
wait_futs.push(receiver_fut);
let scan_plan = ScannerBucketScanPlan {
buckets: set_buckets,
all_buckets: Arc::clone(&all_buckets),
digest: scan_plan_digest,
leader_epoch,
dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(),
bucket_failures: bucket_failures.clone(),
pending_maintenance_work: pending_maintenance_work.clone(),
cache_cycle_floor: cache_cycle_floor.clone(),
};
// Spawn task to run the scanner
let scanner_fut = tokio::spawn(async move {
let permit_wait = child_token_clone.clone();
let permit_wait_start = Instant::now();
let _permit = tokio::select! {
permit = set_scan_semaphore_clone.acquire_owned() => match permit {
Ok(permit) => permit,
Err(_) => return,
},
_ = permit_wait.cancelled() => return,
};
metrics::histogram!(
METRIC_SCANNER_SET_SCAN_WAIT_SECONDS,
"pool" => pool_label.clone(),
"set" => set_label.clone()
)
.record(permit_wait_start.elapsed().as_secs_f64());
let queued_count = decrement_atomic_usize(&queued_set_scans_clone);
record_set_scans_queued(queued_count);
let _active_guard = SetScanActiveGuard::new(active_set_scans_clone);
if let Err(e) = set_clone
.nsscanner_cache(
child_token_clone.clone(),
budget_clone,
scan_plan,
tx,
want_cycle_clone,
scan_mode_clone,
)
.await
{
if child_token_clone.is_cancelled() {
debug!(
pool = %pool_label,
set = %set_label,
error = %e,
"Scanner set scan stopped after cancellation"
);
return;
}
counter!(
"rustfs_scanner_set_failure_total",
"pool" => pool_label.clone(),
"set" => set_label.clone(),
"stage" => "nsscanner_cache".to_string()
)
.increment(1);
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = %pool_label,
set = %set_label,
error = %e,
state = "set_scan_failed",
"Scanner set scan failed; continuing cycle"
);
let mut first_err = first_err_mutex_clone.lock().await;
record_set_scan_failure(&mut first_err, e);
}
});
wait_futs.push(scanner_fut);
}
}
for join_result in join_all(wait_futs).await {
if let Err(err) = join_result {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "set_task_join_failed",
error = %err,
"Scanner set task join failed"
);
let mut first_err = first_err_mutex.lock().await;
record_set_scan_failure(&mut first_err, scanner_task_join_error("scanner set", err));
}
}
record_set_scan_concurrency_limit(0);
record_set_scans_queued(0);
record_set_scans_active(0);
let first_err = first_err_mutex.lock().await.take();
let results = results_mutex.lock().await.clone();
let completed_all_sets = bucket_plan_complete && scanner_results_form_complete_snapshot(&results, &expected_sources);
let result = finalize_nsscanner_result(&results, first_err);
let failed_buckets = bucket_failures.hard.lock().await.clone();
let partial_buckets = bucket_failures.partial.lock().await.clone();
let namespace_not_found_buckets = bucket_failures.namespace_not_found.lock().await.clone();
let scan_scope_matches = scanner_results_match_scan_scope(&results, &expected_sources);
let bucket_scan_status = scanner_bucket_scan_status(
!failed_buckets.is_empty(),
scan_scope_matches && !partial_buckets.is_empty(),
scan_scope_matches && !namespace_not_found_buckets.is_empty(),
);
let pending_maintenance_work = pending_maintenance_work_for_cycle(&pending_maintenance_work, &results);
let observed_cycle_floor = cache_cycle_floor.load(Ordering::Acquire);
let required_cycle_floor = (observed_cycle_floor > want_cycle).then_some(observed_cycle_floor);
let budget_elapsed = budget.budget_elapsed();
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
let dirty_usage_current = dirty_usage_status == DirtyUsageSnapshotStatus::Current;
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::<Vec<_>>();
let completed_usage = completed_data_usage_info(
&results,
&expected_sources,
&all_bucket_names,
bucket_plan_complete,
budget_elapsed,
ctx.is_cancelled(),
);
let structurally_complete_snapshot = result.is_ok() && completed_all_sets && completed_usage.is_some();
let cycle_status = classify_nsscanner_cycle(
structurally_complete_snapshot,
budget_elapsed,
ctx.is_cancelled(),
bucket_scan_status,
dirty_usage_status,
activity_status,
);
if cycle_status == ScannerCycleStatus::Complete
&& let Some((data_usage_info, _)) = completed_usage
{
send_data_usage_update(&updates, data_usage_info).await?;
}
let dirty_usage_clear = should_clear_dirty_usage_snapshot(
result.is_ok(),
structurally_complete_snapshot,
budget_elapsed,
activity_status == ScannerCycleActivityStatus::Unchanged && dirty_usage_current,
&dirty_usage_snapshot.buckets,
&failed_buckets,
);
result?;
let remote_dirty_usage_acknowledgements = if cycle_status == ScannerCycleStatus::Complete {
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before)
} else {
Vec::new()
};
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
.with_failed_dirty_usage(!failed_buckets.is_empty())
.with_pending_maintenance_work(pending_maintenance_work)
.with_required_cycle_floor(required_cycle_floor))
}
}
#[async_trait::async_trait]
impl ScannerIOCache for SetDisks {
#[tracing::instrument(skip(self, budget, scan_plan, updates))]
async fn nsscanner_cache(
self: Arc<Self>,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
scan_plan: ScannerBucketScanPlan,
updates: mpsc::Sender<DataUsageCache>,
want_cycle: u64,
scan_mode: HealScanMode,
) -> Result<()> {
let ScannerBucketScanPlan {
buckets,
all_buckets,
digest: scan_plan_digest,
leader_epoch,
dirty_usage_buckets,
bucket_failures,
pending_maintenance_work,
cache_cycle_floor,
} = scan_plan;
let pool_label = self.pool_index.to_string();
let set_label = self.set_index.to_string();
let source = DataUsageCacheSource::new(self.pool_index, self.set_index);
if buckets.is_empty() {
let now = SystemTime::now();
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: want_cycle,
last_update: Some(now),
leader_epoch,
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for bucket in all_buckets.iter() {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
return persist_and_publish_cache_snapshot(self, &updates, cache, cache_cycle_floor.as_ref())
.await
.map(|_| ())
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
}
let (disks, healing) = self.get_online_disks_with_healing(false).await;
if disks.is_empty() {
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
state = "no_online_disks",
"Scanner set state found no online disks"
);
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
return Ok(());
}
// Preserve the original set topology across capability filtering. During
// rolling upgrades, an old remote peer must not make a distributed set
// look local and allow an unscoped legacy cache to be adopted.
let require_cache_source = disks.iter().any(|disk| !disk.is_local());
let mut coordinator_disks = Vec::new();
let mut remote_candidates = Vec::new();
for disk in disks {
if disk.is_local() {
coordinator_disks.push(disk);
} else {
remote_candidates.push(disk);
}
}
let remote_groups = group_remote_disks_by_peer(remote_candidates, |disk| disk.host_name());
let capability_results = join_all(remote_groups.into_iter().map(|disks| async move {
let server_epoch = match disks.first() {
Some(disk) => disk.ns_scanner_server_epoch().await,
None => Ok(None),
};
(disks, server_epoch)
}))
.await;
let mut remote_disks = Vec::new();
let mut unsupported_remote_disks = 0_usize;
for (disks, server_epoch) in capability_results {
match server_epoch {
Ok(Some(server_epoch)) => {
remote_disks.extend(disks.into_iter().map(|disk| (disk, server_epoch)));
}
Ok(None) => {
let disk_count = disks.len();
let peer = disks.first().map(|disk| disk.host_name()).unwrap_or_default();
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
peer = %peer,
disk_count,
state = "remote_scanner_unsupported",
"Scanner found a peer without remote namespace scanner support"
);
unsupported_remote_disks = unsupported_remote_disks.saturating_add(disk_count);
coordinator_disks.extend(disks);
}
Err(err) => {
let peer = disks.first().map(|disk| disk.host_name()).unwrap_or_default();
let disk_count = disks.len();
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
peer = %peer,
disk_count,
state = "remote_scanner_probe_failed",
error = %err,
"Scanner skipped a peer whose remote namespace scanner capability could not be confirmed"
);
}
}
}
let remote_disk_count = remote_disks.len();
let workers = namespace_scanner_workers(coordinator_disks, remote_disks);
if workers.is_empty() {
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
state = "no_compatible_disks",
"Scanner set state found no usable namespace scanner disks"
);
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
return Ok(());
}
let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await);
if unsupported_remote_disks > 0 {
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
v4_disks = remote_disk_count,
unsupported_remote_disks,
state = "unsupported_remote_disks_using_coordinator",
"Scanner set assigned remote disks without namespace scanner support to coordinator-driven workers"
);
}
let disk_scan_limit = scanner_budgeted_concurrency_limit(
scanner_max_concurrent_disk_scans(workers.len()),
budget.requires_serial_progress_accounting(),
);
record_disk_scan_concurrency_limit(&pool_label, &set_label, disk_scan_limit);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
online_disks = workers.len(),
concurrency_limit = disk_scan_limit,
state = "disk_concurrency_budget",
"Scanner disk concurrency budget resolved"
);
let disk_scan_semaphore = Arc::new(Semaphore::new(disk_scan_limit));
let queued_disk_bucket_scans = Arc::new(AtomicUsize::new(buckets.len()));
let active_disk_bucket_scans = Arc::new(AtomicUsize::new(0));
record_disk_bucket_scans_queued(buckets.len(), &pool_label, &set_label);
record_disk_bucket_scans_active(0, &pool_label, &set_label);
let _reset_disk_bucket_scan_gauges = DiskBucketScanGaugeReset::new(pool_label.clone(), set_label.clone());
let mut old_cache = DataUsageCache::default();
if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
state = "old_cache_load_failed",
error = %e,
"Scanner old data usage cache load failed; rebuilding from bucket caches"
);
}
match old_cache.prepare_for_scan(
DATA_USAGE_ROOT,
want_cycle,
leader_epoch,
source,
scan_plan_digest,
require_cache_source,
) {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
cache_cycle_floor.fetch_max(old_cache.info.next_cycle, Ordering::AcqRel);
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
requested_cycle = want_cycle,
cached_cycle = old_cache.info.next_cycle,
state = "stale_cycle_rejected",
"Scanner rejected a set cache cycle regression"
);
return Ok(());
}
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
requested_epoch = leader_epoch,
cached_epoch = old_cache.info.leader_epoch,
state = "stale_leader_rejected",
"Scanner rejected work from an older leader epoch"
);
return Ok(());
}
DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {}
}
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: want_cycle,
leader_epoch,
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for bucket in all_buckets.iter() {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
let (bucket_tx, bucket_rx) = mpsc::channel::<BucketInfo>(buckets.len());
let mut permutes = buckets.clone();
permutes.shuffle(&mut rand::rng());
let scan_order = bucket_usage_scan_order(&permutes, &old_cache, &dirty_usage_buckets);
for bucket in scan_order.iter() {
if let Some(c) = old_cache.find(&bucket.name) {
cache.replace(&bucket.name, DATA_USAGE_ROOT, c.clone());
}
if let Err(e) = bucket_tx.send(bucket.clone()).await {
record_failed_dirty_bucket(&bucket_failures.hard, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_bucket_failed",
error = %e,
"Scanner bucket dispatch failed"
);
}
}
let cache_mutex: Arc<Mutex<DataUsageCache>> = Arc::new(Mutex::new(cache));
let (bucket_result_tx, mut bucket_result_rx) = mpsc::channel::<DataUsageEntryInfo>(workers.len());
let cache_mutex_clone = cache_mutex.clone();
let ctx_clone = ctx.clone();
let completed_bucket_count = Arc::new(AtomicUsize::new(0));
let completed_bucket_count_clone = completed_bucket_count.clone();
let collect_bucket_results_fut = tokio::spawn(async move {
let mut cancelled = false;
loop {
tokio::select! {
_ = ctx_clone.cancelled(), if !cancelled => {
cancelled = true;
}
result = bucket_result_rx.recv() => {
let Some(result) = result else {
return;
};
let mut cache = cache_mutex_clone.lock().await;
apply_bucket_result_to_cache(&mut cache, result, SystemTime::now());
completed_bucket_count_clone.fetch_add(1, Ordering::Relaxed);
}
}
}
});
let mut futs = Vec::new();
let bucket_rx_mutex: Arc<Mutex<mpsc::Receiver<BucketInfo>>> = Arc::new(Mutex::new(bucket_rx));
let remaining_bucket_work = Arc::new(AtomicUsize::new(buckets.len()));
let bucket_work_complete = CancellationToken::new();
for (disk, worker_mode) in workers {
let bucket_rx_mutex_clone = bucket_rx_mutex.clone();
let bucket_tx_clone = bucket_tx.clone();
let remaining_bucket_work_clone = remaining_bucket_work.clone();
let bucket_work_complete_clone = bucket_work_complete.clone();
let ctx_clone = ctx.clone();
let budget_clone = budget.clone();
let store_clone_clone = self.clone();
let bucket_result_tx_clone = bucket_result_tx.clone();
let disk_clone = disk.clone();
let set_disk_inventory_clone = set_disk_inventory.clone();
let disk_scan_semaphore_clone = disk_scan_semaphore.clone();
let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone();
let active_disk_bucket_scans_clone = active_disk_bucket_scans.clone();
let pool_label_clone = pool_label.clone();
let set_label_clone = set_label.clone();
let failed_dirty_buckets_clone = bucket_failures.hard.clone();
let partial_dirty_buckets_clone = bucket_failures.partial.clone();
let pending_maintenance_work_clone = pending_maintenance_work.clone();
let dirty_usage_buckets_clone = dirty_usage_buckets.clone();
let cache_cycle_floor_clone = cache_cycle_floor.clone();
let remote_server_epoch = match worker_mode {
NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
NamespaceScannerWorkerMode::Coordinator => None,
};
futs.push(tokio::spawn(async move {
let remote_session_id = uuid::Uuid::new_v4();
let mut remote_session_sequence = 0_u64;
loop {
let bucket = tokio::select! {
_ = bucket_work_complete_clone.cancelled() => break,
_ = ctx_clone.cancelled() => break,
bucket = async { bucket_rx_mutex_clone.lock().await.recv().await } => {
let Some(bucket) = bucket else {
break;
};
bucket
}
};
let mut work_guard =
BucketWorkGuard::new(remaining_bucket_work_clone.clone(), bucket_work_complete_clone.clone());
let permit_wait = ctx_clone.clone();
let permit_wait_start = Instant::now();
let _permit = tokio::select! {
permit = disk_scan_semaphore_clone.clone().acquire_owned() => match permit {
Ok(permit) => permit,
Err(_) => {
decrement_disk_bucket_scans_queued(
&queued_disk_bucket_scans_clone,
&pool_label_clone,
&set_label_clone,
);
break;
},
},
_ = permit_wait.cancelled() => {
decrement_disk_bucket_scans_queued(
&queued_disk_bucket_scans_clone,
&pool_label_clone,
&set_label_clone,
);
break;
},
};
metrics::histogram!(
METRIC_SCANNER_DISK_SCAN_WAIT_SECONDS,
"pool" => pool_label_clone.clone(),
"set" => set_label_clone.clone()
)
.record(permit_wait_start.elapsed().as_secs_f64());
decrement_disk_bucket_scans_queued(&queued_disk_bucket_scans_clone, &pool_label_clone, &set_label_clone);
let _active_guard = DiskBucketScanActiveGuard::new(
active_disk_bucket_scans_clone.clone(),
pool_label_clone.clone(),
set_label_clone.clone(),
);
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "scan_started",
"Scanner disk bucket scan started"
);
let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]);
let bucket_scan_plan_digest =
scanner_bucket_cache_digest(scan_plan_digest, dirty_usage_buckets_clone.get(&bucket.name).copied());
if let Some(server_epoch) = remote_server_epoch {
let request_sequence = remote_session_sequence;
let Some(next_sequence) = remote_session_sequence.checked_add(1) else {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "remote_session_sequence_exhausted",
"Remote scanner session sequence exhausted"
);
break;
};
remote_session_sequence = next_sequence;
let remote_outcome = crate::remote_scanner::scan_remote_bucket(
&disk_clone,
ctx_clone.clone(),
budget_clone.clone(),
crate::remote_scanner::RemoteScannerScanSpec {
bucket: &bucket.name,
next_cycle: want_cycle,
leader_epoch,
server_epoch,
session_id: remote_session_id,
session_sequence: request_sequence,
scan_plan_digest: bucket_scan_plan_digest,
skip_healing: healing,
scan_mode,
},
)
.await;
match remote_outcome {
Ok(crate::remote_scanner::RemoteScannerOutcome::Complete {
usage,
pending_maintenance_work,
}) => {
if pending_maintenance_work {
pending_maintenance_work_clone.store(true, Ordering::Release);
}
if let Err(e) = bucket_result_tx_clone.send(*usage).await {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_remote_root_failed",
error = %e,
"Remote scanner root entry publish failed"
);
}
}
Ok(crate::remote_scanner::RemoteScannerOutcome::Partial) => {
record_partial_dirty_bucket(&partial_dirty_buckets_clone, &bucket.name).await;
}
Ok(crate::remote_scanner::RemoteScannerOutcome::NamespaceNotFound) => {
if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await {
increment_disk_bucket_scans_queued(
&queued_disk_bucket_scans_clone,
&pool_label_clone,
&set_label_clone,
);
} else {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "remote_namespace_missing_requeued",
"Remote scanner requeued a bucket missing from this disk"
);
break;
}
Ok(crate::remote_scanner::RemoteScannerOutcome::CycleAhead(required_cycle)) => {
cache_cycle_floor_clone.fetch_max(required_cycle, Ordering::AcqRel);
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
requested_cycle = want_cycle,
cached_cycle = required_cycle,
state = "remote_stale_cycle_rejected",
"Remote scanner rejected a bucket cache cycle regression"
);
}
Err(e) => {
if e.retry_bucket_work() {
if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await {
increment_disk_bucket_scans_queued(
&queued_disk_bucket_scans_clone,
&pool_label_clone,
&set_label_clone,
);
} else {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "remote_zero_progress_requeued",
error = %e,
"Remote scanner requeued bucket rejected before execution"
);
break;
}
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
if ctx_clone.is_cancelled() {
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "remote_cancelled",
error = %e,
"Remote scanner bucket scan cancelled"
);
} else {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "remote_scan_failed",
error = %e,
"Remote scanner bucket scan failed"
);
}
if e.retire_worker() {
break;
}
}
}
continue;
}
let _local_admission = if disk_clone.is_local() {
match crate::remote_scanner::try_admit_remote_scanner(&disk_clone) {
Ok(admission) => Some(admission),
Err(e) => {
if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await {
increment_disk_bucket_scans_queued(
&queued_disk_bucket_scans_clone,
&pool_label_clone,
&set_label_clone,
);
} else {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "local_disk_busy",
error = %e,
"Scanner local disk is already serving namespace scanner work"
);
break;
}
}
} else {
None
};
// Lock order: scanner leader fence -> set-scoped per-bucket cache lock ->
// cache object read/write.
let cache_guard = match acquire_scanner_cache_locks(store_clone_clone.as_ref(), &cache_name, source).await {
Ok(guard) => guard,
Err(e) => {
if e.is_contention() {
if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await {
increment_disk_bucket_scans_queued(
&queued_disk_bucket_scans_clone,
&pool_label_clone,
&set_label_clone,
);
} else {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "lock_contention_requeued",
error = %e,
"Scanner bucket cache lock contention requeued bucket work"
);
break;
}
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = e.state(),
error = %e,
"Scanner bucket cache lock acquisition failed"
);
continue;
}
};
let mut cache = DataUsageCache::default();
let revisions = match cache.load_with_revisions(store_clone_clone.clone(), &cache_name).await {
Ok(revisions) => revisions,
Err(e) => {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "load_or_revision_lookup_failed",
error = %e,
"Scanner bucket cache load or revision lookup failed"
);
continue;
}
};
let scan_state = current_cache_root_or_prepare(
&mut cache,
&bucket.name,
source,
want_cycle,
leader_epoch,
bucket_scan_plan_digest,
require_cache_source,
);
let outcome = match scan_state {
DataUsageCacheScanState::Current(root) => {
if cache_guard.is_lock_lost() {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "lock_lost_before_reuse",
"Current scanner bucket cache root publish skipped after lock loss"
);
continue;
}
if let Err(e) =
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
.await
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_current_root_failed",
error = %e,
"Current scanner bucket cache root entry publish failed"
);
}
continue;
}
DataUsageCacheScanState::Prepared {
outcome,
invalid_current,
} => {
if let Some(e) = invalid_current {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "current_cache_invalid",
error = %e,
"Current scanner bucket cache is invalid; rebuilding"
);
}
outcome
}
};
match outcome {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
cache_cycle_floor_clone.fetch_max(cache.info.next_cycle, Ordering::AcqRel);
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
requested_cycle = want_cycle,
cached_cycle = cache.info.next_cycle,
state = "stale_cycle_rejected",
"Scanner rejected a bucket cache cycle regression"
);
continue;
}
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
requested_epoch = leader_epoch,
cached_epoch = cache.info.leader_epoch,
state = "stale_leader_rejected",
"Scanner rejected bucket work from an older leader epoch"
);
continue;
}
DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {}
}
cache.info.skip_healing = healing;
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = ?cache.info.name,
state = "cache_ready",
"Scanner disk bucket cache ready"
);
let before = cache.info.last_update;
let scan_ctx = ctx_clone.child_token();
let scan = disk_clone.clone().nsscanner_disk(
scan_ctx.clone(),
budget_clone.clone(),
set_disk_inventory_clone.as_ref().clone(),
cache.clone(),
None,
scan_mode,
);
tokio::pin!(scan);
let mut lock_watch = tokio::time::interval(SCANNER_CACHE_LOCK_POLL_INTERVAL);
lock_watch.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
let scan_result = loop {
tokio::select! {
result = &mut scan => break result,
_ = lock_watch.tick() => {
if cache_guard.is_lock_lost() {
scan_ctx.cancel();
await_scanner_disk_shutdown(scan.as_mut()).await;
break Err(Error::other("scanner bucket cache lock was lost during bucket scan"));
}
}
}
};
let scan_outcome = match scan_result {
Ok(scan_outcome) => scan_outcome,
Err(e) => {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
if ctx_clone.is_cancelled() {
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "cancelled",
error = %e,
"Scanner disk bucket scan cancelled"
);
} else {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "scan_failed",
error = %e,
"Scanner disk bucket scan failed"
);
}
if !cache_guard.is_lock_lost()
&& let (Some(last_update), Some(before_update)) = (cache.info.last_update, before)
&& last_update > before_update
{
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions)
.await
{
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "save_failed",
error = %e,
"Scanner bucket cache save failed"
);
}
done_save();
}
continue;
}
};
let partial = match scan_outcome {
ScannerDiskScanOutcome::Complete(completed_cache) => {
cache = completed_cache;
None
}
ScannerDiskScanOutcome::Partial(partial_cache) => Some((partial_cache, &partial_dirty_buckets_clone)),
ScannerDiskScanOutcome::NamespaceNotFound(_) => {
if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await {
increment_disk_bucket_scans_queued(
&queued_disk_bucket_scans_clone,
&pool_label_clone,
&set_label_clone,
);
} else {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "local_namespace_missing_requeued",
"Scanner requeued a bucket missing from this local disk"
);
break;
}
};
if let Some((partial_cache, failure_buckets)) = partial {
record_partial_dirty_bucket(failure_buckets, &bucket.name).await;
let done_save = Metrics::time(Metric::SaveUsage);
let partial_saved = if cache_guard.is_lock_lost() {
false
} else {
match partial_cache
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions)
.await
{
Ok(()) => true,
Err(e) => {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "partial_save_failed",
error = %e,
"Scanner partial bucket cache save failed"
);
false
}
}
};
done_save();
if !partial_saved {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
}
if partial_saved {
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "partial_saved_not_published",
"Scanner partial bucket cache saved without publishing usage aggregate"
);
}
continue;
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache.info.name,
state = "scan_completed",
"Scanner disk bucket scan completed"
);
if ctx_clone.is_cancelled() {
break;
}
if cache_guard.is_lock_lost() {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "lock_lost",
"Scanner bucket cache save skipped after lock loss"
);
continue;
}
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache
.save_with_revisions(store_clone_clone.clone(), &cache_name, &revisions)
.await
{
done_save();
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "save_failed",
error = %e,
"Scanner bucket cache save failed"
);
continue;
}
done_save();
if cache_guard.is_lock_lost() {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "lock_lost_after_save",
"Scanner bucket cache root publish skipped after lock loss"
);
continue;
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache.info.name,
state = "send_root_entry",
"Scanner root entry publish started"
);
if let Err(e) =
send_cache_root_entry_info(&bucket_result_tx_clone, &cache, &pending_maintenance_work_clone).await
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_root_failed",
error = %e,
"Scanner root entry publish failed"
);
}
}
}));
}
drop(bucket_tx);
drop(bucket_result_tx);
let mut first_join_err = None;
for join_result in join_all(futs).await {
if let Err(err) = join_result {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
state = "disk_bucket_task_join_failed",
error = %err,
"Scanner disk bucket task join failed"
);
record_set_scan_failure(&mut first_join_err, scanner_task_join_error("scanner disk bucket", err));
}
}
let unprocessed_buckets = mark_unprocessed_bucket_work_failed(
bucket_rx_mutex.as_ref(),
&remaining_bucket_work,
&bucket_work_complete,
&bucket_failures.hard,
)
.await;
if unprocessed_buckets > 0 {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
unprocessed_buckets,
state = "workers_exhausted",
"Scanner marked queued bucket work failed after all disk workers exited"
);
}
record_disk_scan_concurrency_limit(&pool_label, &set_label, 0);
record_disk_bucket_scans_queued(0, &pool_label, &set_label);
record_disk_bucket_scans_active(0, &pool_label, &set_label);
if let Err(err) = collect_bucket_results_fut.await {
return Err(scanner_task_join_error("scanner bucket result collector", err));
}
if let Some(err) = first_join_err {
return Err(err);
}
let completed_count = completed_bucket_count.load(Ordering::Relaxed);
if should_publish_completed_snapshot(completed_count, buckets.len(), budget.budget_elapsed(), ctx.is_cancelled()) {
let cache_snapshot = {
let mut cache = cache_mutex.lock().await;
cache.info.next_cycle = want_cycle;
cache.info.last_update.get_or_insert_with(SystemTime::now);
cache.info.snapshot_complete = true;
cache.clone()
};
let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot, cache_cycle_floor.as_ref()).await;
} else {
let incomplete_scope = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: want_cycle,
leader_epoch,
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
};
if let Err(e) = updates.send(incomplete_scope).await {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
state = "incomplete_scope_publish_failed",
error = %e,
"Scanner incomplete set scope publish failed"
);
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
completed_buckets = completed_count,
total_buckets = buckets.len(),
budget_elapsed = budget.budget_elapsed(),
cancelled = ctx.is_cancelled(),
state = "set_cache_publish_skipped",
"Scanner set cache publish skipped because cycle did not complete cleanly"
);
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "set_scan_completed",
"Scanner set scan completed"
);
Ok(())
}
}
#[async_trait::async_trait]
impl ScannerIODisk for Disk {
async fn get_size(&self, mut item: ScannerItem) -> Result<SizeSummary> {
let done_object = Metrics::time(Metric::ScanObject);
if !is_xl_meta_path(&item.path) {
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
}
let data = match self.read_metadata(&item.bucket, &item.object_path()).await {
Ok(data) => data,
Err(e) if DiskError::is_err_object_not_found(&e) || DiskError::is_err_version_not_found(&e) => {
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
}
Err(e) => {
return Err(scanner_metadata_transient_error(
format!("failed to read metadata: {e}"),
&item.bucket,
&item.object_path(),
));
}
};
item.transform_meta_dir();
let meta = FileMeta::load(&data).map_err(|e| {
scanner_metadata_corrupt_error(format!("failed to load metadata: {e}"), &item.bucket, &item.object_path())
})?;
let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) {
Ok(versions) => versions,
Err(e) => {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %item.bucket,
object = %item.object_path(),
state = "file_info_versions_failed",
error = %e,
"Scanner disk bucket failed to resolve file info versions"
);
return Err(scanner_metadata_corrupt_error(
format!("failed to resolve file info versions: {e}"),
&item.bucket,
&item.object_path(),
));
}
};
let versioned = BucketVersioningSys::get(&item.bucket)
.await
.map(|v| v.versioned(&item.object_path()))
.unwrap_or(false);
let object_infos = fivs
.versions
.iter()
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
.collect::<Vec<ObjectInfo>>();
let free_version_infos = fivs
.free_versions
.iter()
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
.collect::<Vec<ObjectInfo>>();
let mut size_summary = SizeSummary::default();
let tiers = list_runtime_tiers().await;
for tier in tiers.iter() {
size_summary.tier_stats.insert(tier.name.clone(), TierStats::default());
}
if !size_summary.tier_stats.is_empty() {
size_summary
.tier_stats
.insert(storageclass::STANDARD.to_string(), TierStats::default());
size_summary
.tier_stats
.insert(storageclass::RRS.to_string(), TierStats::default());
}
let lock_config = object_lock_config_for_scanner_item(&item).await;
// Count every version this object contributes to the scan, independent
// of any lifecycle configuration, so scan-coverage metrics stay honest
// on clusters without ILM rules. Recorded before `apply_actions` moves
// `object_infos`.
global_metrics().record_scanner_versions_scanned(object_infos.len() as u64);
item.apply_actions(object_infos, lock_config, &mut size_summary).await;
if !free_version_infos.is_empty() {
for oi in free_version_infos {
enqueue_runtime_free_version(oi).await;
}
}
done_object();
Ok(size_summary)
}
#[tracing::instrument(skip(self, budget, updates, cache))]
async fn nsscanner_disk(
self: Arc<Self>,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
set_disks: Vec<Arc<Disk>>,
cache: DataUsageCache,
updates: Option<mpsc::Sender<DataUsageEntry>>,
scan_mode: HealScanMode,
) -> Result<ScannerDiskScanOutcome> {
let done_drive = Metrics::time(Metric::ScanBucketDrive);
let drive_start = std::time::Instant::now();
let bucket = cache.info.name.clone();
let disk_path = self.path().to_string_lossy().to_string();
global_metrics().record_scan_bucket_drive_start();
let mut failure_guard = BucketDriveFailureGuard::new();
let _guard = self.start_scan();
let mut cache = cache;
let (lifecycle_config, _) = get_lifecycle_config(&cache.info.name)
.await
.unwrap_or_else(|_| (BucketLifecycleConfiguration::default(), OffsetDateTime::now_utc()));
if lifecycle_config.has_active_rules("") {
cache.info.lifecycle = Some(Arc::new(lifecycle_config));
}
let (replication_config, _) = get_replication_config(&cache.info.name).await.unwrap_or((
ReplicationConfiguration {
role: "".to_string(),
rules: vec![],
},
OffsetDateTime::now_utc(),
));
if replication_config.has_active_rules("", true)
&& let Ok(targets) = BucketTargetSys::get().list_bucket_targets(&cache.info.name).await
{
cache.info.replication = Some(Arc::new(ReplicationConfig::new(Some(replication_config), Some(targets))));
}
if let Ok((object_lock_config, _)) = get_object_lock_config(&cache.info.name).await
&& object_lock_config_enabled(&object_lock_config)
{
cache.info.object_lock = Some(Arc::new(object_lock_config));
}
let result = scan_data_folder(
ctx.clone(),
budget,
set_disks,
self.clone(),
cache,
updates,
scan_mode,
SCANNER_SLEEPER.clone(),
)
.await;
match result {
Ok(mut data_usage_info) => {
done_drive();
emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed());
data_usage_info.info.last_update = Some(SystemTime::now());
failure_guard.mark_not_failed();
Ok(ScannerDiskScanOutcome::Complete(data_usage_info))
}
Err(ScannerError::PartialCache(mut partial_cache)) => {
done_drive();
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
failure_guard.mark_not_failed();
Ok(ScannerDiskScanOutcome::Partial(*partial_cache))
}
Err(ScannerError::NamespaceNotFoundCache(mut partial_cache)) => {
done_drive();
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
failure_guard.mark_not_failed();
Ok(ScannerDiskScanOutcome::NamespaceNotFound(*partial_cache))
}
Err(e) => {
if ctx.is_cancelled() {
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
failure_guard.mark_not_failed();
} else {
done_drive();
emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed());
}
Err(StorageError::other(format!("Failed to scan data folder: {e}")))
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::scanner_budget::ScannerCycleBudgetConfig;
use crate::scanner_folder::ScannerItem;
use crate::storage_api::scan::{BucketOperations as _, MakeBucketOptions, ObjectIO as _};
use crate::{
DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions,
ScannerPutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests,
init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
};
use rustfs_filemeta::FileInfo;
use serial_test::serial;
use temp_env::with_var;
use time::OffsetDateTime;
use uuid::Uuid;
fn bucket_info(name: &str) -> BucketInfo {
BucketInfo {
name: name.to_string(),
created: None,
deleted: None,
versioning: false,
object_locking: false,
}
}
async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc<ECStore>) {
init_ecstore_config_for_scanner_tests();
let temp_dir = tempfile::tempdir().expect("multi-pool scanner test directory should be created");
let mut pools = Vec::new();
for pool_index in 0..2 {
let mut endpoints = Vec::new();
for disk_index in 0..4 {
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
tokio::fs::create_dir_all(&disk_path)
.await
.expect("multi-pool scanner test disk should be created");
let mut endpoint =
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
endpoints.push(endpoint);
}
pools.push(PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("scanner-cycle-pool-{pool_index}"),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
});
}
let endpoint_pools = EndpointServerPools::from(pools);
let instance_ctx = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await
.expect("multi-pool local disks should initialize");
let store = ECStore::new_with_instance_ctx(
"127.0.0.1:0".parse().expect("test address should parse"),
endpoint_pools,
CancellationToken::new(),
instance_ctx,
)
.await
.expect("multi-pool ECStore should initialize");
init_bucket_metadata_sys_for_scanner_tests(store.clone()).await;
(temp_dir, store)
}
#[tokio::test]
#[serial]
async fn scanner_cache_locks_block_same_source_workers() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let set = &store.pools[0].disk_set[0];
let source = DataUsageCacheSource::new(0, 0);
let cache_name = "photos/.usage-cache.bin";
let guards = acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
.await
.expect("scanner cache locks should be acquired");
let scoped_lock = set
.new_ns_lock(RUSTFS_META_BUCKET, &scanner_cache_lock_resource(cache_name, source))
.await
.expect("scoped scanner cache lock should be created");
let scoped_err = scoped_lock
.get_write_lock_quiet(Duration::from_millis(100))
.await
.expect_err("same-source workers must be blocked while scanner cache lock is held");
assert!(matches!(scoped_err, LockError::Timeout { .. } | LockError::AlreadyLocked { .. }));
drop(guards);
acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
.await
.expect("scanner cache locks should be released when guards drop");
}
#[tokio::test]
#[serial]
async fn scanner_cache_locks_allow_cross_source_workers() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let first_set = &store.pools[0].disk_set[0];
let second_set = &store.pools[1].disk_set[0];
let cache_name = "photos/.usage-cache.bin";
let first = acquire_scanner_cache_locks(first_set.as_ref(), cache_name, DataUsageCacheSource::new(0, 0))
.await
.expect("first source scanner cache locks should be acquired");
let second = acquire_scanner_cache_locks(second_set.as_ref(), cache_name, DataUsageCacheSource::new(1, 0))
.await
.expect("different source scanner cache locks should not contend");
assert!(!first.is_lock_lost());
assert!(!second.is_lock_lost());
}
#[tokio::test]
async fn data_usage_publish_fails_when_receiver_is_closed() {
let (updates, receiver) = mpsc::channel(1);
drop(receiver);
let err = send_data_usage_update(&updates, DataUsageInfo::default())
.await
.expect_err("closed usage receiver must reject the scanner update");
assert!(err.to_string().contains("receiver closed"));
}
#[tokio::test]
#[serial]
async fn multi_pool_scanner_cycle_publishes_combined_usage() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let bucket = format!("scanner-union-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created across both pools");
for (pool_index, (object, body)) in [("pool-a", b"first".as_slice()), ("pool-b", b"second".as_slice())]
.into_iter()
.enumerate()
{
let mut reader = ScannerPutObjReader::from_vec(body.to_vec());
store.pools[pool_index].disk_set[0]
.put_object(&bucket, object, &mut reader, &ScannerObjectOptions::default())
.await
.expect("object should be written to its selected pool");
}
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
let (updates, mut receiver) = mpsc::channel(1);
let result = tokio::time::timeout(
Duration::from_secs(30),
ScannerIOCycle::nsscanner_with_status(store.as_ref(), ctx, budget, updates, 1, 1, HealScanMode::Normal),
)
.await
.expect("multi-pool scanner cycle should finish")
.expect("multi-pool scanner cycle should succeed");
assert_eq!(result.status, ScannerCycleStatus::Complete);
let usage = receiver.recv().await.expect("complete scanner cycle should publish usage");
let bucket_usage = usage
.buckets_usage
.get(&bucket)
.expect("combined bucket usage should be present");
assert_eq!(bucket_usage.objects_count, 2);
assert_eq!(bucket_usage.size, 11);
assert_eq!(usage.objects_total_count, 2);
assert_eq!(usage.objects_total_size, 11);
}
#[tokio::test]
#[serial]
async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let bucket = format!("scanner-second-pool-{}", Uuid::new_v4().simple());
store.pools[1].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created only in the second pool");
let body = b"second-only";
let mut reader = ScannerPutObjReader::from_vec(body.to_vec());
store.pools[1].disk_set[0]
.put_object(&bucket, "pool-b", &mut reader, &ScannerObjectOptions::default())
.await
.expect("object should be written only to the second pool");
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
let (updates, mut receiver) = mpsc::channel(1);
let result = tokio::time::timeout(
Duration::from_secs(30),
ScannerIOCycle::nsscanner_with_status(store.as_ref(), ctx, budget, updates, 1, 1, HealScanMode::Normal),
)
.await
.expect("second-pool-only scanner cycle should finish")
.expect("second-pool-only scanner cycle should succeed");
assert_eq!(result.status, ScannerCycleStatus::Complete);
let usage = receiver.recv().await.expect("complete scanner cycle should publish usage");
let bucket_usage = usage
.buckets_usage
.get(&bucket)
.expect("second-pool-only bucket usage should be present");
assert_eq!(bucket_usage.objects_count, 1);
assert_eq!(bucket_usage.size, u64::try_from(body.len()).expect("test body length should fit u64"));
assert_eq!(usage.objects_total_count, 1);
assert_eq!(
usage.objects_total_size,
u64::try_from(body.len()).expect("test body length should fit u64")
);
}
#[tokio::test]
async fn scanner_item_object_lock_uses_cached_config() {
let temp_dir = std::env::temp_dir();
let cached = Arc::new(ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
..Default::default()
});
let item = ScannerItem {
path: temp_dir.join("object").to_string_lossy().to_string(),
bucket: "bucket".to_string(),
prefix: String::new(),
object_name: "object".to_string(),
file_type: std::fs::metadata(&temp_dir)
.expect("temp dir metadata should be readable")
.file_type(),
lifecycle: None,
object_lock: Some(cached.clone()),
replication: None,
heal_enabled: false,
heal_bitrot: false,
debug: false,
};
let resolved = object_lock_config_for_scanner_item(&item)
.await
.expect("cached object-lock config should resolve");
assert!(Arc::ptr_eq(&resolved, &cached));
}
#[test]
fn object_lock_config_enabled_accepts_enabled_only() {
let enabled = ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
..Default::default()
};
assert!(object_lock_config_enabled(&enabled));
assert!(!object_lock_config_enabled(&ObjectLockConfiguration::default()));
}
#[test]
#[serial]
fn dirty_usage_snapshot_clear_preserves_newer_generation() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
let buckets = vec![bucket_info("photos")];
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
record_dirty_usage_bucket("photos");
clear_dirty_usage_buckets(&snapshot.buckets);
assert_eq!(dirty_usage_bucket_count(), 1);
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
let acknowledged_generation = scanner_dirty_usage_state().generation;
record_dirty_usage_bucket("videos");
acknowledge_dirty_usage_generation(scanner_activity_epoch(), acknowledged_generation)
.expect("a matching process and prior generation should be acknowledged");
acknowledge_dirty_usage_generation(scanner_activity_epoch(), acknowledged_generation)
.expect("replaying an acknowledged generation should be idempotent");
let pending = dirty_usage_buckets_for_tests();
assert!(!pending.contains_key("photos"));
assert!(pending.contains_key("videos"));
assert!(scanner_dirty_usage_state().pending);
drop(pending);
let remaining_generation = scanner_dirty_usage_state().generation;
acknowledge_dirty_usage_generation(scanner_activity_epoch(), remaining_generation)
.expect("the remaining generation should be acknowledged");
assert!(!scanner_dirty_usage_state().pending);
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_generation() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
let generation = scanner_dirty_usage_state().generation;
assert_eq!(
acknowledge_dirty_usage_generation("stale-process", generation),
Err(ScannerDirtyUsageAckError::ProcessChanged)
);
assert_eq!(
acknowledge_dirty_usage_generation(scanner_activity_epoch(), 0),
Err(ScannerDirtyUsageAckError::InvalidGeneration)
);
assert_eq!(
acknowledge_dirty_usage_generation(scanner_activity_epoch(), u64::MAX),
Err(ScannerDirtyUsageAckError::InvalidGeneration)
);
assert_eq!(
acknowledge_dirty_usage_generation(
scanner_activity_epoch(),
generation.checked_add(1).expect("test generation should not be exhausted")
),
Err(ScannerDirtyUsageAckError::InvalidGeneration)
);
assert!(dirty_usage_buckets_for_tests().contains_key("photos"));
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn dirty_usage_snapshot_detects_uncovered_generation() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
let buckets = vec![bucket_info("photos")];
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Current);
record_dirty_usage_bucket("photos");
assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Changed);
clear_dirty_usage_buckets_for_tests();
}
#[test]
fn generation_saturates_instead_of_wrapping() {
let generation = AtomicU64::new(u64::MAX - 1);
assert_eq!(advance_generation(&generation), u64::MAX);
assert_eq!(advance_generation(&generation), u64::MAX);
assert_eq!(generation.load(Ordering::Acquire), u64::MAX);
}
#[test]
#[serial]
fn dirty_usage_snapshot_clears_a_stably_absent_bucket_after_durable_save() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
record_dirty_usage_bucket("temporarily-omitted");
let generation_before_bucket_list = dirty_usage_generation();
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], generation_before_bucket_list);
assert!(snapshot.buckets.contains_key("photos"));
assert!(snapshot.buckets.contains_key("temporarily-omitted"));
assert!(dirty_usage_buckets().contains_key("temporarily-omitted"));
assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Current);
let acknowledgements = ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone()))
.acknowledge_durable_usage();
assert!(acknowledgements.is_empty());
assert!(!dirty_usage_buckets().contains_key("temporarily-omitted"));
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn dirty_usage_snapshot_preserves_an_absent_bucket_recorded_after_listing_started() {
clear_dirty_usage_buckets_for_tests();
let generation_before_bucket_list = dirty_usage_generation();
record_dirty_usage_bucket("new-or-racing-bucket");
let snapshot = snapshot_dirty_usage_buckets(&[], generation_before_bucket_list);
assert!(!snapshot.buckets.contains_key("new-or-racing-bucket"));
assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Changed);
assert!(dirty_usage_buckets().contains_key("new-or-racing-bucket"));
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn deleting_a_clean_bucket_invalidates_an_inflight_usage_snapshot() {
clear_dirty_usage_buckets_for_tests();
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation());
assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Current);
record_dirty_usage_bucket("photos");
assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Changed);
assert!(dirty_usage_buckets().contains_key("photos"));
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn deleting_a_bucket_during_listing_invalidates_the_resulting_usage_snapshot() {
clear_dirty_usage_buckets_for_tests();
let generation_before_bucket_list = dirty_usage_generation();
record_dirty_usage_bucket("photos");
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], generation_before_bucket_list);
assert_eq!(dirty_usage_snapshot_status(&snapshot), DirtyUsageSnapshotStatus::Changed);
assert!(dirty_usage_buckets().contains_key("photos"));
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn scanner_maintenance_change_advances_generation_and_marks_usage_dirty() {
clear_dirty_usage_buckets_for_tests();
let generation = scanner_maintenance_generation();
record_scanner_maintenance_change("photos");
assert!(scanner_maintenance_generation() > generation);
assert!(dirty_usage_buckets().contains_key("photos"));
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn dirty_usage_clear_excludes_failed_buckets() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
record_dirty_usage_bucket("videos");
let buckets = vec![bucket_info("photos"), bucket_info("videos")];
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
let failed_buckets = HashSet::from(["videos".to_string()]);
let clear_snapshot = dirty_usage_buckets_excluding_failed(&snapshot.buckets, &failed_buckets);
clear_dirty_usage_buckets(&clear_snapshot);
let dirty_buckets = dirty_usage_buckets();
assert!(!dirty_buckets.contains_key("photos"));
assert!(dirty_buckets.contains_key("videos"));
drop(dirty_buckets);
clear_dirty_usage_buckets_for_tests();
}
#[test]
fn dirty_usage_clear_plan_excludes_cache_save_failures() {
let snapshot = DirtyUsageBuckets::from([("photos".to_string(), 1), ("videos".to_string(), 2)]);
let failed_buckets = HashSet::from(["videos".to_string()]);
let clear_snapshot = should_clear_dirty_usage_snapshot(true, true, false, true, &snapshot, &failed_buckets)
.expect("successful completed cycle should produce a clear snapshot");
assert!(clear_snapshot.contains_key("photos"));
assert!(!clear_snapshot.contains_key("videos"));
}
#[test]
#[serial]
fn dirty_usage_is_acknowledged_only_after_durable_usage_confirmation() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation());
let unconfirmed = ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone()));
drop(unconfirmed);
assert!(dirty_usage_buckets().contains_key("photos"));
let confirmed = ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(snapshot.buckets.as_ref().clone()));
let acknowledgements = confirmed.acknowledge_durable_usage();
assert!(acknowledgements.is_empty());
assert!(!dirty_usage_buckets().contains_key("photos"));
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn clear_dirty_usage_bucket_removes_deleted_bucket_marker() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("photos");
record_dirty_usage_bucket("videos");
clear_dirty_usage_bucket("photos");
let buckets = vec![bucket_info("photos"), bucket_info("videos")];
let snapshot = snapshot_dirty_usage_buckets(&buckets, dirty_usage_generation());
assert!(!snapshot.buckets.contains_key("photos"));
assert!(snapshot.buckets.contains_key("videos"));
assert_eq!(dirty_usage_bucket_count(), 1);
clear_dirty_usage_buckets_for_tests();
}
#[test]
fn bucket_usage_scan_order_prioritizes_dirty_buckets() {
let buckets = vec![bucket_info("missing"), bucket_info("cached"), bucket_info("dirty")];
let mut old_cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
..Default::default()
},
..Default::default()
};
old_cache.replace("cached", DATA_USAGE_ROOT, DataUsageEntry::default());
old_cache.replace("dirty", DATA_USAGE_ROOT, DataUsageEntry::default());
let dirty_buckets = HashMap::from([("dirty".to_string(), 1)]);
let ordered = bucket_usage_scan_order(&buckets, &old_cache, &dirty_buckets);
let names = ordered.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>();
assert_eq!(names, vec!["dirty", "missing", "cached"]);
}
#[test]
fn record_set_scan_failure_preserves_first_error() {
let mut first = None;
record_set_scan_failure(&mut first, Error::other("first"));
record_set_scan_failure(&mut first, Error::other("second"));
let first = first.expect("first error should be recorded");
assert!(first.to_string().contains("first"));
}
#[tokio::test]
async fn scanner_task_join_error_includes_stage() {
let handle = tokio::spawn(async {
tokio::time::sleep(Duration::from_secs(60)).await;
});
handle.abort();
let join_err = handle.await.expect_err("aborted task should return a join error");
let err = scanner_task_join_error("scanner set", join_err);
assert!(err.to_string().contains("scanner set task join failed"));
}
#[test]
fn finalize_nsscanner_result_returns_ok_when_any_set_succeeds() {
let mut results = vec![DataUsageCache::default(), DataUsageCache::default()];
results[1].info.last_update = Some(SystemTime::now());
let result = finalize_nsscanner_result(&results, Some(Error::other("set failed")));
assert!(result.is_ok());
}
#[test]
fn finalize_nsscanner_result_returns_first_error_when_all_sets_fail() {
let results = vec![DataUsageCache::default(), DataUsageCache::default()];
let err = finalize_nsscanner_result(&results, Some(Error::other("set failed")))
.expect_err("all failed sets should bubble first error");
assert!(err.to_string().contains("set failed"));
}
#[test]
fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
assert_eq!(
classify_nsscanner_cycle(
true,
false,
false,
ScannerBucketScanStatus::Complete,
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Unchanged,
),
ScannerCycleStatus::Complete
);
assert_eq!(
classify_nsscanner_cycle(
true,
false,
false,
ScannerBucketScanStatus::Complete,
DirtyUsageSnapshotStatus::Changed,
ScannerCycleActivityStatus::Unchanged,
),
ScannerCycleStatus::Superseded
);
assert_eq!(
classify_nsscanner_cycle(
true,
false,
false,
ScannerBucketScanStatus::Complete,
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Changed,
),
ScannerCycleStatus::Superseded
);
assert_eq!(
classify_nsscanner_cycle(
true,
false,
false,
ScannerBucketScanStatus::Complete,
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Unverified,
),
ScannerCycleStatus::Incomplete
);
for status in [
classify_nsscanner_cycle(
false,
false,
false,
ScannerBucketScanStatus::Complete,
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Unchanged,
),
classify_nsscanner_cycle(
true,
true,
false,
ScannerBucketScanStatus::Complete,
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Unchanged,
),
classify_nsscanner_cycle(
true,
false,
true,
ScannerBucketScanStatus::Complete,
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Unchanged,
),
classify_nsscanner_cycle(
true,
false,
false,
ScannerBucketScanStatus::Failed,
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Changed,
),
classify_nsscanner_cycle(
false,
false,
false,
ScannerBucketScanStatus::Partial,
DirtyUsageSnapshotStatus::Changed,
ScannerCycleActivityStatus::Changed,
),
] {
assert_eq!(status, ScannerCycleStatus::Incomplete);
}
}
#[test]
fn scanner_cycle_fails_closed_for_namespace_disappearance() {
for activity_status in [
ScannerCycleActivityStatus::Changed,
ScannerCycleActivityStatus::Unchanged,
ScannerCycleActivityStatus::Unverified,
] {
assert_eq!(
classify_nsscanner_cycle(
false,
false,
false,
ScannerBucketScanStatus::NamespaceNotFound,
DirtyUsageSnapshotStatus::Changed,
activity_status,
),
ScannerCycleStatus::Incomplete
);
}
assert_eq!(
classify_nsscanner_cycle(
true,
true,
false,
ScannerBucketScanStatus::NamespaceNotFound,
DirtyUsageSnapshotStatus::Changed,
ScannerCycleActivityStatus::Changed,
),
ScannerCycleStatus::Incomplete
);
}
#[test]
fn scanner_cycle_fails_closed_when_dirty_generation_is_unverified() {
assert_eq!(
classify_nsscanner_cycle(
true,
false,
false,
ScannerBucketScanStatus::Complete,
DirtyUsageSnapshotStatus::Unverified,
ScannerCycleActivityStatus::Unchanged,
),
ScannerCycleStatus::Incomplete
);
}
#[test]
fn scanner_bucket_failure_status_preserves_the_strongest_failure() {
assert_eq!(scanner_bucket_scan_status(false, false, false), ScannerBucketScanStatus::Complete);
assert_eq!(scanner_bucket_scan_status(false, false, true), ScannerBucketScanStatus::NamespaceNotFound);
assert_eq!(scanner_bucket_scan_status(false, true, true), ScannerBucketScanStatus::Partial);
assert_eq!(scanner_bucket_scan_status(true, true, true), ScannerBucketScanStatus::Failed);
}
#[test]
fn scanner_cycle_surfaces_persisted_pending_heal_work() {
let clean = DataUsageCache::default();
assert!(!scanner_results_have_pending_maintenance_work(std::slice::from_ref(&clean)));
let mut pending = clean;
pending.info.pending_heals.push(crate::PendingScannerHeal {
kind: crate::PendingScannerHealKind::Object,
bucket: "photos".to_string(),
object: Some("image.jpg".to_string()),
version_id: None,
scan_mode: HealScanMode::Normal,
first_seen: 1,
last_attempt: 1,
attempts: 1,
last_admission_result: "queue_full".to_string(),
last_admission_reason: "capacity".to_string(),
});
assert!(scanner_results_have_pending_maintenance_work(&[pending]));
}
#[tokio::test]
async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() {
let pending_maintenance_work = Arc::new(AtomicBool::new(false));
let mut bucket_cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "photos".to_string(),
..Default::default()
},
..Default::default()
};
bucket_cache.replace("photos", DATA_USAGE_ROOT, DataUsageEntry::default());
bucket_cache.info.pending_heals.push(crate::PendingScannerHeal {
kind: crate::PendingScannerHealKind::Object,
bucket: "photos".to_string(),
object: Some("image.jpg".to_string()),
version_id: None,
scan_mode: HealScanMode::Normal,
first_seen: 1,
last_attempt: 1,
attempts: 1,
last_admission_result: "queue_full".to_string(),
last_admission_reason: "capacity".to_string(),
});
let (sender, mut receiver) = mpsc::channel(1);
send_cache_root_entry_info(&sender, &bucket_cache, &pending_maintenance_work)
.await
.expect("bucket result should send");
let cycle_pending = pending_maintenance_work_for_cycle(&pending_maintenance_work, &[]);
assert!(cycle_pending);
assert_eq!(
crate::scanner::scanner_cycle_outcome_with_pending_maintenance(
crate::scanner::ScannerCycleOutcome::Completed,
cycle_pending,
),
crate::scanner::ScannerCycleOutcome::CompletedWithPendingMaintenance
);
assert!(receiver.recv().await.is_some());
}
#[test]
#[serial]
fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
crate::reset_foreground_read_activity_for_test();
assert_eq!(scanner_concurrency_limit(0, 4), 4);
}
#[test]
#[serial]
fn scanner_concurrency_limit_caps_to_configured_value() {
crate::reset_foreground_read_activity_for_test();
assert_eq!(scanner_concurrency_limit(2, 4), 2);
}
#[test]
#[serial]
fn scanner_concurrency_limit_never_exceeds_available_work() {
crate::reset_foreground_read_activity_for_test();
assert_eq!(scanner_concurrency_limit(8, 4), 4);
}
#[test]
#[serial]
fn scanner_concurrency_limit_handles_no_available_work() {
crate::reset_foreground_read_activity_for_test();
assert_eq!(scanner_concurrency_limit(2, 0), 0);
}
#[test]
#[serial]
fn scanner_concurrency_limit_yields_to_foreground_reads() {
crate::reset_foreground_read_activity_for_test();
crate::set_foreground_read_activity(8);
assert_eq!(scanner_concurrency_limit(0, 4), 1);
assert_eq!(scanner_concurrency_limit(3, 4), 1);
crate::reset_foreground_read_activity_for_test();
}
#[test]
#[serial]
fn scanner_concurrency_limit_yields_to_streaming_reads() {
crate::reset_foreground_read_activity_for_test();
let _guard = crate::ForegroundReadGuard::new();
assert_eq!(scanner_concurrency_limit(0, 4), 1);
assert_eq!(scanner_concurrency_limit(3, 4), 1);
}
#[test]
fn decrement_atomic_usize_saturates_at_zero() {
let counter = AtomicUsize::new(1);
assert_eq!(decrement_atomic_usize(&counter), 0);
assert_eq!(decrement_atomic_usize(&counter), 0);
}
#[test]
fn increment_atomic_usize_saturates_at_max() {
let counter = AtomicUsize::new(usize::MAX);
assert_eq!(increment_atomic_usize(&counter), usize::MAX);
assert_eq!(counter.load(Ordering::Relaxed), usize::MAX);
}
#[test]
#[serial]
fn scanner_max_concurrent_set_scans_uses_env_cap() {
with_var(ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("2"), || {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
assert_eq!(scanner_max_concurrent_set_scans(4), 2);
});
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
}
#[test]
#[serial]
fn scanner_max_concurrent_disk_scans_uses_env_cap() {
with_var(ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, Some("1"), || {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
assert_eq!(scanner_max_concurrent_disk_scans(4), 1);
});
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
}
#[test]
#[cfg(windows)]
fn is_xl_meta_path_accepts_windows_separator() {
assert!(is_xl_meta_path("D:\\data\\bucket\\object\\xl.meta"));
}
#[test]
fn is_xl_meta_path_accepts_forward_separator() {
assert!(is_xl_meta_path("/data/bucket/object/xl.meta"));
}
#[tokio::test]
async fn get_size_treats_missing_metadata_as_skip_file() {
let temp_dir = std::env::temp_dir().join(format!("rustfs-scanner-missing-meta-{}", Uuid::new_v4()));
let bucket = "bucket";
let object = "object";
let object_dir = temp_dir.join(bucket).join(object);
let metadata_path = object_dir.join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(&object_dir)
.await
.expect("failed to create object directory");
tokio::fs::write(&metadata_path, [])
.await
.expect("failed to create metadata placeholder");
let endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("failed to open local disk");
let relative_path = metadata_path.to_string_lossy().to_string();
let (_, scanner_path) = path2_bucket_object_with_base_path(temp_dir.to_string_lossy().as_ref(), relative_path.as_str());
let file_type = tokio::fs::metadata(&metadata_path)
.await
.expect("failed to stat metadata placeholder")
.file_type();
tokio::fs::remove_dir_all(&object_dir)
.await
.expect("failed to remove object directory");
let item = ScannerItem {
path: scanner_path,
bucket: bucket.to_string(),
prefix: object.to_string(),
object_name: STORAGE_FORMAT_FILE.to_string(),
file_type,
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
debug: false,
};
let err = disk
.get_size(item)
.await
.expect_err("missing metadata should be skipped instead of reported as a scanner failure");
assert!(matches!(err, StorageError::Io(ref io) if io.to_string() == SCANNER_SKIP_FILE_ERROR));
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
#[tokio::test]
async fn get_size_marks_corrupt_metadata_for_heal() {
let temp_dir = std::env::temp_dir().join(format!("rustfs-scanner-corrupt-meta-{}", Uuid::new_v4()));
let bucket = "bucket";
let object = "object";
let object_dir = temp_dir.join(bucket).join(object);
let metadata_path = object_dir.join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(&object_dir)
.await
.expect("failed to create object directory");
tokio::fs::write(&metadata_path, b"not-valid-filemeta")
.await
.expect("failed to write corrupt metadata");
let endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("failed to open local disk");
let relative_path = metadata_path.to_string_lossy().to_string();
let (_, scanner_path) = path2_bucket_object_with_base_path(temp_dir.to_string_lossy().as_ref(), relative_path.as_str());
let file_type = tokio::fs::metadata(&metadata_path)
.await
.expect("failed to stat metadata")
.file_type();
let item = ScannerItem {
path: scanner_path,
bucket: bucket.to_string(),
prefix: object.to_string(),
object_name: STORAGE_FORMAT_FILE.to_string(),
file_type,
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
debug: false,
};
let err = disk
.get_size(item)
.await
.expect_err("corrupt metadata should be surfaced as scanner-heal work");
assert!(is_scanner_metadata_corrupt_error(&err));
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
#[tokio::test]
async fn get_size_counts_delete_markers_separately_from_versions() {
let temp_dir = std::env::temp_dir().join(format!("rustfs-scanner-versioned-usage-{}", Uuid::new_v4()));
let bucket = "bucket";
let object = "object";
let object_dir = temp_dir.join(bucket).join(object);
let metadata_path = object_dir.join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(&object_dir)
.await
.expect("failed to create object directory");
let mut meta = FileMeta::new();
for (size, timestamp) in [(10, 10), (20, 20)] {
let mut fi = FileInfo::new(object, 1, 1);
fi.version_id = Some(Uuid::new_v4());
fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(timestamp).expect("timestamp should be valid"));
fi.size = size;
meta.add_version(fi).expect("object version should be added");
}
// A real delete marker carries no erasure geometry (delete paths build it as
// `FileInfo { deleted: true, .. }`). Construct it that way so it classifies as a
// storage delete marker rather than a purge-pending payload object.
let delete_marker = FileInfo {
name: object.to_string(),
version_id: Some(Uuid::new_v4()),
mod_time: Some(OffsetDateTime::from_unix_timestamp(30).expect("timestamp should be valid")),
deleted: true,
..Default::default()
};
meta.add_version(delete_marker).expect("delete marker should be added");
tokio::fs::write(&metadata_path, meta.marshal_msg().expect("metadata should marshal"))
.await
.expect("failed to write metadata");
let endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("failed to open local disk");
let relative_path = metadata_path.to_string_lossy().to_string();
let (_, scanner_path) = path2_bucket_object_with_base_path(temp_dir.to_string_lossy().as_ref(), relative_path.as_str());
let file_type = tokio::fs::metadata(&metadata_path)
.await
.expect("failed to stat metadata")
.file_type();
let item = ScannerItem {
path: scanner_path,
bucket: bucket.to_string(),
prefix: object.to_string(),
object_name: STORAGE_FORMAT_FILE.to_string(),
file_type,
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
debug: false,
};
let summary = disk.get_size(item).await.expect("scanner should read versioned metadata");
assert_eq!(summary.versions, 2);
assert_eq!(summary.delete_markers, 1);
assert_eq!(summary.total_size, 30);
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
#[test]
fn cache_root_entry_info_flattens_bucket_children() {
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
cache.replace(
"bucket",
DATA_USAGE_ROOT,
DataUsageEntry {
size: 10,
objects: 1,
..Default::default()
},
);
cache.replace(
"bucket/prefix",
"bucket",
DataUsageEntry {
size: 20,
objects: 2,
..Default::default()
},
);
let info = cache_root_entry_info(&cache).expect("valid cache should flatten");
assert_eq!(info.name, "bucket");
assert_eq!(info.parent, DATA_USAGE_ROOT);
assert_eq!(info.entry.size, 30);
assert_eq!(info.entry.objects, 3);
assert!(info.entry.children.is_empty());
}
#[test]
fn cache_root_entry_info_rejects_missing_or_dangling_roots() {
let missing_root = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
assert!(cache_root_entry_info(&missing_root).is_err());
let mut dangling = missing_root;
let mut root = DataUsageEntry::default();
root.add_child(&crate::hash_path("bucket/missing"));
dangling.replace("bucket", DATA_USAGE_ROOT, root);
assert!(cache_root_entry_info(&dangling).is_err());
let mut detached = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
..Default::default()
},
..Default::default()
};
detached.replace("bucket", DATA_USAGE_ROOT, DataUsageEntry::default());
detached.replace(
"bucket/detached",
"",
DataUsageEntry {
objects: 1,
..Default::default()
},
);
assert!(cache_root_entry_info(&detached).is_err());
let mut detached_bucket = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
detached_bucket.replace(
"bucket",
DATA_USAGE_ROOT,
DataUsageEntry {
objects: 1,
..Default::default()
},
);
detached_bucket.replace(
"bucket/detached",
"",
DataUsageEntry {
objects: 1,
..Default::default()
},
);
assert!(cache_root_entry_info(&detached_bucket).is_err());
let mut compacted_with_child = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
compacted_with_child.replace(
"bucket",
DATA_USAGE_ROOT,
DataUsageEntry {
compacted: true,
..Default::default()
},
);
compacted_with_child.replace("bucket/prefix", "bucket", DataUsageEntry::default());
assert!(cache_root_entry_info(&compacted_with_child).is_err());
}
#[test]
fn apply_bucket_result_to_cache_updates_bucket_entry() {
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
..Default::default()
},
..Default::default()
};
cache.replace(
"bucket",
DATA_USAGE_ROOT,
DataUsageEntry {
size: 5,
objects: 1,
..Default::default()
},
);
let update_time = SystemTime::now();
apply_bucket_result_to_cache(
&mut cache,
DataUsageEntryInfo {
name: "bucket".to_string(),
parent: DATA_USAGE_ROOT.to_string(),
entry: DataUsageEntry {
size: 10,
objects: 2,
..Default::default()
},
},
update_time,
);
assert_eq!(cache.info.last_update, Some(update_time));
let entry = cache.find("bucket").expect("bucket entry should remain present");
assert_eq!(entry.size, 10);
assert_eq!(entry.objects, 2);
}
}
Reference in New Issue View Git Blame Copy Permalink