gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 11:32:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(scanner): add scanner budget progress controls (#3185)

Browse Source 
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-06-03 22:37:58 +08:00
committed by GitHub
parent f49827fc58
commit ad1a489f75
13 changed files with 1346 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/error.rs
+6
View File
@@ -14,6 +14,8 @@
use thiserror::Error;
use crate::data_usage_define::DataUsageCache;
/// Scanner-related errors
#[derive(Error, Debug)]
#[non_exhaustive]
@@ -33,4 +35,8 @@ pub enum ScannerError {
/// Generic error
#[error("Scanner error: {0}")]
Other(String),
/// Partial data usage cache produced before the scanner stopped.
#[error("Scanner stopped with partial data usage cache")]
PartialCache(Box<DataUsageCache>),
}
crates/scanner/src/lib.rs
+1
View File
@@ -23,6 +23,7 @@
pub mod data_usage_define;
pub mod error;
pub mod scanner;
pub mod scanner_budget;
pub mod scanner_folder;
pub mod scanner_io;
pub mod sleeper;
crates/scanner/src/scanner.rs
+103 -68
View File
@@ -14,21 +14,25 @@
use std::sync::{
Arc,
atomic::{AtomicBool, AtomicU64, Ordering},
atomic::{AtomicU64, Ordering},
};
use crate::data_usage_define::{BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH};
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason};
use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
use crate::scanner_io::ScannerIO;
use crate::sleeper::{SCANNER_SLEEPER, scanner_speed_from_env_or_default, set_scanner_default_speed};
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError};
use chrono::{DateTime, Utc};
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{CurrentCycle, Metric, Metrics, emit_scan_cycle_complete, emit_scan_cycle_partial, global_metrics};
use rustfs_common::metrics::{
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, emit_scan_cycle_complete, emit_scan_cycle_partial, global_metrics,
};
use rustfs_config::ScannerSpeed;
use rustfs_config::{
DEFAULT_SCANNER_BITROT_CYCLE_SECS, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CYCLE,
ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS,
DEFAULT_SCANNER_BITROT_CYCLE_SECS, DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS,
DEFAULT_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DIRECTORIES,
ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS,
};
use rustfs_ecstore::StorageAPI as _;
use rustfs_ecstore::bucket::lifecycle::lifecycle::Lifecycle as _;
@@ -94,6 +98,30 @@ fn scanner_cycle_max_duration() -> Option<Duration> {
}
}
fn scanner_cycle_count_budget(env: &str, default: u64) -> Option<u64> {
match rustfs_utils::get_env_u64(env, default) {
0 => None,
count => Some(count),
}
}
fn scanner_cycle_budget_config() -> ScannerCycleBudgetConfig {
ScannerCycleBudgetConfig {
max_duration: scanner_cycle_max_duration(),
max_objects: scanner_cycle_count_budget(ENV_SCANNER_CYCLE_MAX_OBJECTS, DEFAULT_SCANNER_CYCLE_MAX_OBJECTS),
max_directories: scanner_cycle_count_budget(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES),
}
}
fn scan_cycle_partial_reason(reason: Option<ScannerCycleBudgetReason>) -> ScanCyclePartialReason {
match reason {
Some(ScannerCycleBudgetReason::Runtime) => ScanCyclePartialReason::Runtime,
Some(ScannerCycleBudgetReason::Objects) => ScanCyclePartialReason::Objects,
Some(ScannerCycleBudgetReason::Directories) => ScanCyclePartialReason::Directories,
None => ScanCyclePartialReason::Unknown,
}
}
/// Compute a randomized inter-cycle sleep.
// Delay is scan interval +- 10%, with a floor of 1 second.
fn randomized_cycle_delay() -> Duration {
@@ -439,60 +467,6 @@ fn get_lock_acquire_timeout() -> Duration {
Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
}
struct ScannerCycleBudget {
token: CancellationToken,
elapsed: Arc<AtomicBool>,
max_duration: Option<Duration>,
}
impl ScannerCycleBudget {
fn new(parent: &CancellationToken, max_duration: Option<Duration>) -> Self {
let token = parent.child_token();
let elapsed = Arc::new(AtomicBool::new(false));
if let Some(duration) = max_duration {
let parent = parent.clone();
let token_wait = token.clone();
let token_cancel = token.clone();
let elapsed = elapsed.clone();
tokio::spawn(async move {
tokio::select! {
_ = parent.cancelled() => {}
_ = token_wait.cancelled() => {}
_ = tokio::time::sleep(duration) => {
elapsed.store(true, Ordering::Relaxed);
token_cancel.cancel();
}
}
});
}
Self {
token,
elapsed,
max_duration,
}
}
fn token(&self) -> CancellationToken {
self.token.clone()
}
fn budget_elapsed(&self) -> bool {
self.elapsed.load(Ordering::Relaxed)
}
fn max_duration(&self) -> Option<Duration> {
self.max_duration
}
}
impl Drop for ScannerCycleBudget {
fn drop(&mut self) {
self.token.cancel();
}
}
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle) {
cycle_info.current = 0;
global_metrics().clear_current_scan_mode();
@@ -505,11 +479,13 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
SCANNER_SLEEPER.refresh_from_env();
let configured_cycle_interval = cycle_interval();
let configured_bitrot_cycle = bitrot_scan_cycle();
let configured_cycle_max_duration = scanner_cycle_max_duration();
let cycle_budget_config = scanner_cycle_budget_config();
global_metrics().record_scanner_cycle_config(
configured_cycle_interval,
configured_bitrot_cycle,
configured_cycle_max_duration,
cycle_budget_config.max_duration,
cycle_budget_config.max_objects,
cycle_budget_config.max_directories,
);
info!("Start run data scanner cycle");
cycle_info.current = cycle_info.next;
@@ -548,10 +524,10 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
let done_cycle = Metrics::time(Metric::ScanCycle);
let cycle_start = std::time::Instant::now();
let cycle_work_start = global_metrics().start_scan_cycle_work();
let cycle_budget = ScannerCycleBudget::new(ctx, configured_cycle_max_duration);
let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
if let Err(e) = storeapi
.clone()
.nsscanner(cycle_budget.token(), sender, cycle_info.current, scan_mode)
.nsscanner(cycle_budget.token(), cycle_budget.clone(), sender, cycle_info.current, scan_mode)
.await
{
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
@@ -559,10 +535,13 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
if budget_elapsed {
warn!(
duration = ?now.elapsed(),
reason = ?cycle_budget.reason(),
max_duration = ?cycle_budget.max_duration(),
"Data scanner cycle stopped after reaching its runtime budget"
max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(),
"Data scanner cycle stopped after reaching its cycle budget"
);
emit_scan_cycle_partial(cycle_start.elapsed());
emit_scan_cycle_partial(cycle_start.elapsed(), scan_cycle_partial_reason(cycle_budget.reason()));
mark_scan_cycle_idle(cycle_info).await;
return;
}
@@ -576,11 +555,14 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
if cycle_budget.budget_elapsed() && !ctx.is_cancelled() {
warn!(
duration = ?now.elapsed(),
reason = ?cycle_budget.reason(),
max_duration = ?cycle_budget.max_duration(),
"Data scanner cycle stopped after reaching its runtime budget"
max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(),
"Data scanner cycle stopped after reaching its cycle budget"
);
global_metrics().finish_scan_cycle_work(cycle_work_start);
emit_scan_cycle_partial(cycle_start.elapsed());
emit_scan_cycle_partial(cycle_start.elapsed(), scan_cycle_partial_reason(cycle_budget.reason()));
mark_scan_cycle_idle(cycle_info).await;
return;
}
@@ -833,7 +815,13 @@ mod tests {
#[tokio::test]
async fn test_scanner_cycle_budget_cancels_after_duration() {
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Some(Duration::from_millis(1)));
let budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_duration: Some(Duration::from_millis(1)),
..Default::default()
},
);
tokio::time::timeout(Duration::from_secs(5), budget.token().cancelled())
.await
@@ -846,7 +834,13 @@ mod tests {
#[tokio::test]
async fn test_scanner_cycle_budget_drop_cancels_child_without_elapsed() {
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Some(Duration::from_secs(60)));
let budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_duration: Some(Duration::from_secs(60)),
..Default::default()
},
);
let token = budget.token();
drop(budget);
@@ -854,6 +848,47 @@ mod tests {
assert!(token.is_cancelled());
}
#[test]
#[serial]
fn test_scanner_cycle_budget_config_uses_work_budget_env() {
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("100"), || {
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("25"), || {
let config = scanner_cycle_budget_config();
assert_eq!(config.max_objects, Some(100));
assert_eq!(config.max_directories, Some(25));
});
});
}
#[test]
#[serial]
fn test_scanner_cycle_budget_config_disables_zero_work_budgets() {
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("0"), || {
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("0"), || {
let config = scanner_cycle_budget_config();
assert_eq!(config.max_objects, None);
assert_eq!(config.max_directories, None);
});
});
}
#[test]
fn test_scan_cycle_partial_reason_maps_budget_reason() {
assert_eq!(
scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Runtime)),
ScanCyclePartialReason::Runtime
);
assert_eq!(
scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Objects)),
ScanCyclePartialReason::Objects
);
assert_eq!(
scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Directories)),
ScanCyclePartialReason::Directories
);
assert_eq!(scan_cycle_partial_reason(None), ScanCyclePartialReason::Unknown);
}
#[tokio::test]
#[serial]
async fn test_mark_scan_cycle_idle_clears_published_cycle_state() {
crates/scanner/src/scanner_budget.rs
+231
View File
@@ -0,0 +1,231 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::{
Arc,
atomic::{AtomicU8, AtomicU64, Ordering},
};
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
const BUDGET_REASON_NONE: u8 = 0;
const BUDGET_REASON_RUNTIME: u8 = 1;
const BUDGET_REASON_OBJECTS: u8 = 2;
const BUDGET_REASON_DIRECTORIES: u8 = 3;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) struct ScannerCycleBudgetConfig {
pub max_duration: Option<Duration>,
pub max_objects: Option<u64>,
pub max_directories: Option<u64>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ScannerCycleBudgetReason {
Runtime,
Objects,
Directories,
}
impl ScannerCycleBudgetReason {
fn code(self) -> u8 {
match self {
Self::Runtime => BUDGET_REASON_RUNTIME,
Self::Objects => BUDGET_REASON_OBJECTS,
Self::Directories => BUDGET_REASON_DIRECTORIES,
}
}
fn from_code(code: u8) -> Option<Self> {
match code {
BUDGET_REASON_RUNTIME => Some(Self::Runtime),
BUDGET_REASON_OBJECTS => Some(Self::Objects),
BUDGET_REASON_DIRECTORIES => Some(Self::Directories),
_ => None,
}
}
}
pub struct ScannerCycleBudget {
token: CancellationToken,
reason: Arc<AtomicU8>,
max_duration: Option<Duration>,
max_objects: Option<u64>,
max_directories: Option<u64>,
objects_scanned: AtomicU64,
directories_started: AtomicU64,
}
impl ScannerCycleBudget {
pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
let token = parent.child_token();
let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE));
if let Some(duration) = config.max_duration {
let parent = parent.clone();
let token_wait = token.clone();
let token_cancel = token.clone();
let reason = reason.clone();
tokio::spawn(async move {
tokio::select! {
_ = parent.cancelled() => {}
_ = token_wait.cancelled() => {}
_ = tokio::time::sleep(duration) => {
Self::cancel_for_reason(&reason, &token_cancel, ScannerCycleBudgetReason::Runtime);
}
}
});
}
Arc::new(Self {
token,
reason,
max_duration: config.max_duration,
max_objects: config.max_objects,
max_directories: config.max_directories,
objects_scanned: AtomicU64::new(0),
directories_started: AtomicU64::new(0),
})
}
pub(crate) fn token(&self) -> CancellationToken {
self.token.clone()
}
pub(crate) fn budget_elapsed(&self) -> bool {
self.reason.load(Ordering::Relaxed) != BUDGET_REASON_NONE
}
pub(crate) fn reason(&self) -> Option<ScannerCycleBudgetReason> {
ScannerCycleBudgetReason::from_code(self.reason.load(Ordering::Relaxed))
}
pub(crate) fn max_duration(&self) -> Option<Duration> {
self.max_duration
}
pub(crate) fn max_objects(&self) -> Option<u64> {
self.max_objects
}
pub(crate) fn max_directories(&self) -> Option<u64> {
self.max_directories
}
pub(crate) fn try_start_directory(&self) -> bool {
let Some(max_directories) = self.max_directories else {
return true;
};
let directories = self.directories_started.fetch_add(1, Ordering::Relaxed) + 1;
if directories <= max_directories {
return true;
}
self.cancel_for(ScannerCycleBudgetReason::Directories);
false
}
pub(crate) fn record_object_scanned(&self) {
let Some(max_objects) = self.max_objects else {
return;
};
let objects = self.objects_scanned.fetch_add(1, Ordering::Relaxed) + 1;
if objects >= max_objects {
self.cancel_for(ScannerCycleBudgetReason::Objects);
}
}
fn cancel_for(&self, reason: ScannerCycleBudgetReason) {
Self::cancel_for_reason(&self.reason, &self.token, reason);
}
fn cancel_for_reason(reason: &AtomicU8, token: &CancellationToken, budget_reason: ScannerCycleBudgetReason) {
if reason
.compare_exchange(BUDGET_REASON_NONE, budget_reason.code(), Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
token.cancel();
}
}
}
impl Drop for ScannerCycleBudget {
fn drop(&mut self) {
self.token.cancel();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn runtime_budget_cancels_child_token() {
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_duration: Some(Duration::from_millis(1)),
..Default::default()
},
);
tokio::time::sleep(Duration::from_millis(10)).await;
assert!(budget.budget_elapsed());
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Runtime));
assert!(budget.token().is_cancelled());
}
#[test]
fn object_budget_cancels_after_reaching_limit() {
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(2),
..Default::default()
},
);
budget.record_object_scanned();
assert!(!budget.budget_elapsed());
budget.record_object_scanned();
assert!(budget.budget_elapsed());
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
assert!(budget.token().is_cancelled());
}
#[test]
fn directory_budget_rejects_directory_after_limit() {
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_directories: Some(1),
..Default::default()
},
);
assert!(budget.try_start_directory());
assert!(!budget.budget_elapsed());
assert!(!budget.try_start_directory());
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Directories));
assert!(budget.token().is_cancelled());
}
}
crates/scanner/src/scanner_folder.rs
+143 -1
View File
@@ -21,6 +21,7 @@ use std::time::{Duration, Instant, SystemTime};
use crate::ReplTargetSizeSummary;
use crate::data_usage_define::{DataUsageCache, DataUsageEntry, DataUsageHash, DataUsageHashMap, SizeSummary, hash_path};
use crate::error::ScannerError;
use crate::scanner_budget::ScannerCycleBudget;
use crate::scanner_io::ScannerIODisk as _;
use crate::sleeper::{DynamicSleeper, scanner_yield_every_n_objects};
use metrics::{counter, describe_counter};
@@ -708,6 +709,7 @@ pub struct FolderScanner {
update_current_path: UpdateCurrentPathFn,
budget: Arc<ScannerCycleBudget>,
skip_heal: Arc<std::sync::atomic::AtomicBool>,
local_disk: Arc<Disk>,
}
@@ -878,6 +880,9 @@ impl FolderScanner {
if ctx.is_cancelled() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
}
if !self.budget.try_start_directory() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
}
let this_hash = hash_path(&folder.name);
// Store initial compaction state.
@@ -1105,9 +1110,14 @@ impl FolderScanner {
apply_scanner_size_summary(into, &sz);
into.objects += 1;
object_count += 1;
self.budget.record_object_scanned();
timer.sleep().await;
if ctx.is_cancelled() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
}
if should_yield_after_object(object_count, yield_every_objects) {
let yield_start = Instant::now();
tokio::task::yield_now().await;
@@ -1115,6 +1125,10 @@ impl FolderScanner {
}
}
if ctx.is_cancelled() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
}
if found_objects && is_erasure().await {
// If we found an object in erasure mode, we skip subdirs (only datadirs)...
info!("scan_folder: done for now found an object in erasure mode");
@@ -1194,6 +1208,9 @@ impl FolderScanner {
// Use Box::pin for recursive async call
let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), &mut dst));
if let Err(e) = fut.await {
if ctx.is_cancelled() {
return Err(e);
}
warn!("scan_folder: failed to scan child folder {}: {}", folder_item.name, e);
continue;
}
@@ -1247,6 +1264,9 @@ impl FolderScanner {
// Use Box::pin for recursive async call
let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), &mut dst));
if let Err(e) = fut.await {
if ctx.is_cancelled() {
return Err(e);
}
warn!("scan_folder: failed to scan child folder {}: {}", folder_item.name, e);
continue;
}
@@ -1483,6 +1503,9 @@ impl FolderScanner {
// Use Box::pin for recursive async call
let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), &mut dst));
if let Err(e) = fut.await {
if ctx.is_cancelled() {
return Err(e);
}
warn!("scan_folder: failed to scan child folder {}: {}", folder_item.name, e);
continue;
}
@@ -1570,6 +1593,7 @@ impl FolderScanner {
#[allow(clippy::too_many_arguments)]
pub async fn scan_data_folder(
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
disks: Vec<Arc<Disk>>,
local_disk: Arc<Disk>,
cache: DataUsageCache,
@@ -1630,6 +1654,7 @@ pub async fn scan_data_folder(
updates,
last_update: SystemTime::UNIX_EPOCH,
update_current_path,
budget,
skip_heal,
local_disk,
};
@@ -1648,7 +1673,7 @@ pub async fn scan_data_folder(
};
// Scan the folder
match scanner.scan_folder(ctx, folder, &mut root).await {
match scanner.scan_folder(ctx.clone(), folder, &mut root).await {
Ok(()) => {
// Get the new cache and finalize it
let new_cache = scanner.as_mut_new_cache();
@@ -1660,6 +1685,26 @@ pub async fn scan_data_folder(
Ok(new_cache.clone())
}
Err(e) => {
if ctx.is_cancelled() {
let root_has_progress = !root.children.is_empty()
|| root.size > 0
|| root.objects > 0
|| root.versions > 0
|| root.delete_markers > 0
|| root.failed_objects > 0
|| root.replication_stats.is_some();
let new_cache = scanner.as_mut_new_cache();
if root_has_progress {
new_cache.replace_hashed(&hash_path(&cache.info.name), &None, &root);
}
if new_cache.root().is_some() {
new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
new_cache.info.last_update = Some(SystemTime::now());
new_cache.info.next_cycle = cache.info.next_cycle;
close_disk().await;
return Err(ScannerError::PartialCache(Box::new(new_cache.clone())));
}
}
close_disk().await;
// No useful information, return original cache
Err(e)
@@ -1716,6 +1761,7 @@ mod tests {
updates: None,
last_update: SystemTime::UNIX_EPOCH,
update_current_path,
budget: ScannerCycleBudget::new(&CancellationToken::new(), Default::default()),
skip_heal: Arc::new(AtomicBool::new(false)),
local_disk: disk,
};
@@ -2138,6 +2184,102 @@ mod tests {
.expect("scan_folder should finish successfully");
}
#[tokio::test]
#[serial]
async fn test_scan_folder_directory_budget_cancels_after_limit() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
let bucket_dir = temp_dir.join("bucket");
tokio::fs::create_dir_all(bucket_dir.join("child"))
.await
.expect("failed to create child directory");
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.update_cache.info.name = "bucket".to_string();
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_directories: Some(1),
..Default::default()
},
);
let ctx = budget.token();
scanner.budget = budget.clone();
let folder = CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let mut into = DataUsageEntry::default();
let result = scanner.scan_folder(ctx, folder, &mut into).await;
assert!(result.is_err(), "directory budget cancellation should make the scan partial");
assert!(budget.budget_elapsed());
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
assert!(budget.token().is_cancelled());
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
let bucket_dir = temp_dir.join("bucket");
tokio::fs::create_dir_all(bucket_dir.join("child-a"))
.await
.expect("failed to create first child directory");
tokio::fs::create_dir_all(bucket_dir.join("child-b"))
.await
.expect("failed to create second child directory");
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_directories: Some(2),
..Default::default()
},
);
let cache = DataUsageCache {
info: crate::data_usage_define::DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
..Default::default()
},
..Default::default()
};
let result = scan_data_folder(
budget.token(),
budget.clone(),
vec![scanner.local_disk.clone()],
scanner.local_disk.clone(),
cache,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
let partial_cache = match result {
Err(ScannerError::PartialCache(partial_cache)) => partial_cache,
other => panic!("expected partial cache after directory budget cancellation, got {other:?}"),
};
assert!(partial_cache.info.last_update.is_some());
assert_eq!(partial_cache.info.next_cycle, 7);
assert!(partial_cache.root().is_some(), "partial cache should keep completed scan progress");
assert!(budget.budget_elapsed());
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
}
#[tokio::test]
#[serial]
#[cfg(unix)]
crates/scanner/src/scanner_io.rs
+233 -84
View File
@@ -12,11 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::scanner_budget::ScannerCycleBudget;
use crate::scanner_folder::{ScannerItem, scan_data_folder};
use crate::sleeper::SCANNER_SLEEPER;
use crate::{
DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageEntryInfo,
DataUsageInfo, SizeSummary, TierStats,
DataUsageInfo, ScannerError, SizeSummary, TierStats,
};
use futures::future::join_all;
use metrics::counter;
@@ -66,6 +67,41 @@ const METRIC_SCANNER_SET_SCANS_QUEUED: &str = "rustfs_scanner_set_scans_queued";
const METRIC_SCANNER_DISK_BUCKET_SCANS_ACTIVE: &str = "rustfs_scanner_disk_bucket_scans_active";
const METRIC_SCANNER_DISK_BUCKET_SCANS_QUEUED: &str = "rustfs_scanner_disk_bucket_scans_queued";
fn record_set_scan_concurrency_limit(limit: usize) {
metrics::gauge!(METRIC_SCANNER_SET_SCAN_CONCURRENCY_LIMIT).set(limit as f64);
global_metrics().record_scanner_set_scan_state(Some(limit), None, None);
}
fn record_set_scans_queued(count: usize) {
metrics::gauge!(METRIC_SCANNER_SET_SCANS_QUEUED).set(count as f64);
global_metrics().record_scanner_set_scan_state(None, Some(count), None);
}
fn record_set_scans_active(count: usize) {
metrics::gauge!(METRIC_SCANNER_SET_SCANS_ACTIVE).set(count as f64);
global_metrics().record_scanner_set_scan_state(None, None, Some(count));
}
fn record_disk_scan_concurrency_limit(pool: &str, set: &str, limit: usize) {
metrics::gauge!(
METRIC_SCANNER_DISK_SCAN_CONCURRENCY_LIMIT,
"pool" => pool.to_owned(),
"set" => set.to_owned()
)
.set(limit as f64);
global_metrics().record_scanner_disk_bucket_scan_state(pool, set, Some(limit), None, None);
}
fn record_disk_bucket_scans_active(count: usize, pool: &str, set: &str) {
metrics::gauge!(
METRIC_SCANNER_DISK_BUCKET_SCANS_ACTIVE,
"pool" => pool.to_owned(),
"set" => set.to_owned()
)
.set(count as f64);
global_metrics().record_scanner_disk_bucket_scan_state(pool, set, None, None, Some(count));
}
struct SetScanActiveGuard {
active: Arc<AtomicUsize>,
}
@@ -73,7 +109,7 @@ struct SetScanActiveGuard {
impl SetScanActiveGuard {
fn new(active: Arc<AtomicUsize>) -> Self {
let active_count = active.fetch_add(1, Ordering::Relaxed) + 1;
metrics::gauge!(METRIC_SCANNER_SET_SCANS_ACTIVE).set(active_count as f64);
record_set_scans_active(active_count);
Self { active }
}
}
@@ -81,7 +117,7 @@ impl SetScanActiveGuard {
impl Drop for SetScanActiveGuard {
fn drop(&mut self) {
let active_count = decrement_atomic_usize(&self.active);
metrics::gauge!(METRIC_SCANNER_SET_SCANS_ACTIVE).set(active_count as f64);
record_set_scans_active(active_count);
}
}
@@ -94,12 +130,7 @@ struct DiskBucketScanActiveGuard {
impl DiskBucketScanActiveGuard {
fn new(active: Arc<AtomicUsize>, pool: String, set: String) -> Self {
let active_count = active.fetch_add(1, Ordering::Relaxed) + 1;
metrics::gauge!(
METRIC_SCANNER_DISK_BUCKET_SCANS_ACTIVE,
"pool" => pool.clone(),
"set" => set.clone()
)
.set(active_count as f64);
record_disk_bucket_scans_active(active_count, &pool, &set);
Self { active, pool, set }
}
}
@@ -107,12 +138,7 @@ impl DiskBucketScanActiveGuard {
impl Drop for DiskBucketScanActiveGuard {
fn drop(&mut self) {
let active_count = decrement_atomic_usize(&self.active);
metrics::gauge!(
METRIC_SCANNER_DISK_BUCKET_SCANS_ACTIVE,
"pool" => self.pool.clone(),
"set" => self.set.clone()
)
.set(active_count as f64);
record_disk_bucket_scans_active(active_count, &self.pool, &self.set);
}
}
@@ -138,6 +164,23 @@ impl Drop for BucketDriveFailureGuard {
}
}
struct DiskBucketScanGaugeReset {
pool: String,
set: String,
}
impl DiskBucketScanGaugeReset {
fn new(pool: String, set: String) -> Self {
Self { pool, set }
}
}
impl Drop for DiskBucketScanGaugeReset {
fn drop(&mut self) {
reset_disk_bucket_scan_gauges(&self.pool, &self.set);
}
}
fn decrement_atomic_usize(counter: &AtomicUsize) -> usize {
counter
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)))
@@ -152,6 +195,7 @@ fn record_disk_bucket_scans_queued(count: usize, pool: &str, set: &str) {
"set" => set.to_owned()
)
.set(count as f64);
global_metrics().record_scanner_disk_bucket_scan_state(pool, set, None, Some(count), None);
}
fn decrement_disk_bucket_scans_queued(counter: &AtomicUsize, pool: &str, set: &str) {
@@ -160,25 +204,16 @@ fn decrement_disk_bucket_scans_queued(counter: &AtomicUsize, pool: &str, set: &s
}
fn reset_set_scan_gauges() {
metrics::gauge!(METRIC_SCANNER_SET_SCAN_CONCURRENCY_LIMIT).set(0.0);
metrics::gauge!(METRIC_SCANNER_SET_SCANS_QUEUED).set(0.0);
metrics::gauge!(METRIC_SCANNER_SET_SCANS_ACTIVE).set(0.0);
record_set_scan_concurrency_limit(0);
record_set_scans_queued(0);
record_set_scans_active(0);
global_metrics().reset_scanner_set_scan_state();
}
fn reset_disk_bucket_scan_gauges(pool: &str, set: &str) {
metrics::gauge!(
METRIC_SCANNER_DISK_SCAN_CONCURRENCY_LIMIT,
"pool" => pool.to_owned(),
"set" => set.to_owned()
)
.set(0.0);
record_disk_scan_concurrency_limit(pool, set, 0);
record_disk_bucket_scans_queued(0, pool, set);
metrics::gauge!(
METRIC_SCANNER_DISK_BUCKET_SCANS_ACTIVE,
"pool" => pool.to_owned(),
"set" => set.to_owned()
)
.set(0.0);
record_disk_bucket_scans_active(0, pool, set);
}
fn scanner_concurrency_limit(configured: usize, available: usize) -> usize {
@@ -232,6 +267,23 @@ fn is_xl_meta_path(path: &str) -> bool {
.is_some_and(|name| name == STORAGE_FORMAT_FILE)
}
fn cache_root_entry_info(cache: &DataUsageCache) -> DataUsageEntryInfo {
let entry = cache.root().map(|root| cache.flatten(&root)).unwrap_or_default();
DataUsageEntryInfo {
name: cache.info.name.clone(),
parent: DATA_USAGE_ROOT.to_string(),
entry,
}
}
async fn send_cache_root_entry_info(
bucket_result_tx: &Arc<Mutex<mpsc::Sender<DataUsageEntryInfo>>>,
cache: &DataUsageCache,
) -> std::result::Result<(), mpsc::error::SendError<DataUsageEntryInfo>> {
bucket_result_tx.lock().await.send(cache_root_entry_info(cache)).await
}
async fn persist_and_publish_cache_snapshot<S: StorageAPI>(
store: Arc<S>,
updates: &mpsc::Sender<DataUsageCache>,
@@ -257,6 +309,7 @@ pub trait ScannerIO: Send + Sync + Debug + 'static {
async fn nsscanner(
&self,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
updates: mpsc::Sender<DataUsageInfo>,
want_cycle: u64,
scan_mode: HealScanMode,
@@ -268,6 +321,7 @@ pub trait ScannerIOCache: Send + Sync + Debug + 'static {
async fn nsscanner_cache(
self: Arc<Self>,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
buckets: Vec<BucketInfo>,
updates: mpsc::Sender<DataUsageCache>,
want_cycle: u64,
@@ -280,20 +334,28 @@ pub trait ScannerIODisk: Send + Sync + Debug + 'static {
async fn nsscanner_disk(
&self,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
cache: DataUsageCache,
updates: Option<mpsc::Sender<DataUsageEntry>>,
scan_mode: HealScanMode,
) -> Result<DataUsageCache>;
) -> Result<ScannerDiskScanOutcome>;
async fn get_size(&self, item: ScannerItem) -> Result<SizeSummary>;
}
#[derive(Debug)]
pub enum ScannerDiskScanOutcome {
Complete(DataUsageCache),
Partial(DataUsageCache),
}
#[async_trait::async_trait]
impl ScannerIO for ECStore {
#[tracing::instrument(skip(self, updates))]
#[tracing::instrument(skip(self, budget, updates))]
async fn nsscanner(
&self,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
updates: mpsc::Sender<DataUsageInfo>,
want_cycle: u64,
scan_mode: HealScanMode,
@@ -321,7 +383,7 @@ impl ScannerIO for ECStore {
}
let set_scan_limit = scanner_max_concurrent_set_scans(total_results);
metrics::gauge!(METRIC_SCANNER_SET_SCAN_CONCURRENCY_LIMIT).set(set_scan_limit as f64);
record_set_scan_concurrency_limit(set_scan_limit);
debug!(
total_sets = total_results,
concurrency_limit = set_scan_limit,
@@ -330,8 +392,8 @@ impl ScannerIO for ECStore {
let set_scan_semaphore = Arc::new(Semaphore::new(set_scan_limit));
let queued_set_scans = Arc::new(AtomicUsize::new(total_results));
let active_set_scans = Arc::new(AtomicUsize::new(0));
metrics::gauge!(METRIC_SCANNER_SET_SCANS_QUEUED).set(total_results as f64);
metrics::gauge!(METRIC_SCANNER_SET_SCANS_ACTIVE).set(0.0);
record_set_scans_queued(total_results);
record_set_scans_active(0);
let results = vec![DataUsageCache::default(); total_results];
let results_mutex: Arc<Mutex<Vec<DataUsageCache>>> = Arc::new(Mutex::new(results));
@@ -350,6 +412,7 @@ impl ScannerIO for ECStore {
let set_label = set.set_index.to_string();
let child_token_clone = child_token.clone();
let budget_clone = budget.clone();
let want_cycle_clone = want_cycle;
let scan_mode_clone = scan_mode;
let results_mutex_clone = results_mutex.clone();
@@ -388,11 +451,18 @@ impl ScannerIO for ECStore {
)
.record(permit_wait_start.elapsed().as_secs_f64());
let queued_count = decrement_atomic_usize(&queued_set_scans_clone);
metrics::gauge!(METRIC_SCANNER_SET_SCANS_QUEUED).set(queued_count as f64);
record_set_scans_queued(queued_count);
let _active_guard = SetScanActiveGuard::new(active_set_scans_clone);
if let Err(e) = set_clone
.nsscanner_cache(child_token_clone.clone(), all_buckets_clone, tx, want_cycle_clone, scan_mode_clone)
.nsscanner_cache(
child_token_clone.clone(),
budget_clone,
all_buckets_clone,
tx,
want_cycle_clone,
scan_mode_clone,
)
.await
{
if child_token_clone.is_cancelled() {
@@ -488,8 +558,9 @@ impl ScannerIO for ECStore {
});
let _ = join_all(wait_futs).await;
metrics::gauge!(METRIC_SCANNER_SET_SCANS_QUEUED).set(0.0);
metrics::gauge!(METRIC_SCANNER_SET_SCANS_ACTIVE).set(0.0);
record_set_scan_concurrency_limit(0);
record_set_scans_queued(0);
record_set_scans_active(0);
let _ = update_tx.send(());
@@ -501,10 +572,11 @@ impl ScannerIO for ECStore {
#[async_trait::async_trait]
impl ScannerIOCache for SetDisks {
#[tracing::instrument(skip(self, updates))]
#[tracing::instrument(skip(self, budget, updates))]
async fn nsscanner_cache(
self: Arc<Self>,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
buckets: Vec<BucketInfo>,
updates: mpsc::Sender<DataUsageCache>,
want_cycle: u64,
@@ -525,12 +597,7 @@ impl ScannerIOCache for SetDisks {
return Ok(());
}
let disk_scan_limit = scanner_max_concurrent_disk_scans(disks.len());
metrics::gauge!(
METRIC_SCANNER_DISK_SCAN_CONCURRENCY_LIMIT,
"pool" => self.pool_index.to_string(),
"set" => self.set_index.to_string()
)
.set(disk_scan_limit as f64);
record_disk_scan_concurrency_limit(&pool_label, &set_label, disk_scan_limit);
debug!(
pool = self.pool_index,
set = self.set_index,
@@ -542,12 +609,8 @@ impl ScannerIOCache for SetDisks {
let queued_disk_bucket_scans = Arc::new(AtomicUsize::new(buckets.len()));
let active_disk_bucket_scans = Arc::new(AtomicUsize::new(0));
record_disk_bucket_scans_queued(buckets.len(), &pool_label, &set_label);
metrics::gauge!(
METRIC_SCANNER_DISK_BUCKET_SCANS_ACTIVE,
"pool" => pool_label.clone(),
"set" => set_label.clone()
)
.set(0.0);
record_disk_bucket_scans_active(0, &pool_label, &set_label);
let _reset_disk_bucket_scan_gauges = DiskBucketScanGaugeReset::new(pool_label.clone(), set_label.clone());
let mut old_cache = DataUsageCache::default();
old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await?;
@@ -597,13 +660,14 @@ impl ScannerIOCache for SetDisks {
let mut ticker = tokio::time::interval(Duration::from_secs(3 + rand::random::<u64>() % 10));
let mut last_update = None;
let mut cancelled = false;
loop {
tokio::select! {
_ = ctx_clone.cancelled() => {
break;
_ = ctx_clone.cancelled(), if !cancelled => {
cancelled = true;
}
_ = ticker.tick() => {
_ = ticker.tick(), if !cancelled => {
let cache_snapshot = {
let cache = cache_mutex_clone.lock().await;
if cache.info.last_update == last_update {
@@ -648,6 +712,7 @@ impl ScannerIOCache for SetDisks {
for disk in disks.into_iter() {
let bucket_rx_mutex_clone = bucket_rx_mutex.clone();
let ctx_clone = ctx.clone();
let budget_clone = budget.clone();
let store_clone_clone = self.clone();
let bucket_result_tx_clone_clone = bucket_result_tx_clone.clone();
let disk_clone = disk.clone();
@@ -756,11 +821,11 @@ impl ScannerIOCache for SetDisks {
let before = cache.info.last_update;
cache = match disk_clone
.nsscanner_disk(ctx_clone.clone(), cache.clone(), Some(updates_tx), scan_mode)
let scan_outcome = match disk_clone
.nsscanner_disk(ctx_clone.clone(), budget_clone.clone(), cache.clone(), Some(updates_tx), scan_mode)
.await
{
Ok(cache) => cache,
Ok(scan_outcome) => scan_outcome,
Err(e) => {
if ctx_clone.is_cancelled() {
debug!("Scanner disk scan stopped after cancellation: {}", e);
@@ -785,34 +850,39 @@ impl ScannerIOCache for SetDisks {
}
};
cache = match scan_outcome {
ScannerDiskScanOutcome::Complete(cache) => cache,
ScannerDiskScanOutcome::Partial(cache) => {
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await {
error!("Failed to save partial data usage cache: {}", e);
}
done_save();
if let Err(e) = update_fut.await {
error!("Failed to update partial data usage cache: {}", e);
}
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
error!("Failed to send partial data usage entry info: {}", e);
}
continue;
}
};
debug!("nsscanner_disk: got cache: {}", cache.info.name);
if let Err(e) = update_fut.await {
error!("nsscanner_disk: Failed to update data usage cache: {}", e);
}
let root = if let Some(r) = cache.root() {
cache.flatten(&r)
} else {
DataUsageEntry::default()
};
if ctx_clone.is_cancelled() {
break;
}
debug!("nsscanner_disk: sending data usage entry info: {}", cache.info.name);
if let Err(e) = bucket_result_tx_clone_clone
.lock()
.await
.send(DataUsageEntryInfo {
name: cache.info.name.clone(),
parent: DATA_USAGE_ROOT.to_string(),
entry: root,
})
.await
{
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
error!("nsscanner_disk: Failed to send data usage entry info: {}", e);
}
@@ -826,13 +896,9 @@ impl ScannerIOCache for SetDisks {
}
let _ = join_all(futs).await;
record_disk_scan_concurrency_limit(&pool_label, &set_label, 0);
record_disk_bucket_scans_queued(0, &pool_label, &set_label);
metrics::gauge!(
METRIC_SCANNER_DISK_BUCKET_SCANS_ACTIVE,
"pool" => pool_label.clone(),
"set" => set_label.clone()
)
.set(0.0);
record_disk_bucket_scans_active(0, &pool_label, &set_label);
drop(bucket_result_tx_clone);
@@ -931,14 +997,15 @@ impl ScannerIODisk for Disk {
Ok(size_summary)
}
#[tracing::instrument(skip(self, updates, cache))]
#[tracing::instrument(skip(self, budget, updates, cache))]
async fn nsscanner_disk(
&self,
ctx: CancellationToken,
budget: Arc<ScannerCycleBudget>,
cache: DataUsageCache,
updates: Option<mpsc::Sender<DataUsageEntry>>,
scan_mode: HealScanMode,
) -> Result<DataUsageCache> {
) -> Result<ScannerDiskScanOutcome> {
let done_drive = Metrics::time(Metric::ScanBucketDrive);
let drive_start = std::time::Instant::now();
let bucket = cache.info.name.clone();
@@ -1004,7 +1071,8 @@ impl ScannerIODisk for Disk {
let disks = disks_result.into_iter().flatten().collect::<Vec<Arc<Disk>>>();
let result = scan_data_folder(ctx.clone(), disks, local_disk, cache, updates, scan_mode, SCANNER_SLEEPER.clone()).await;
let result =
scan_data_folder(ctx.clone(), budget, disks, local_disk, cache, updates, scan_mode, SCANNER_SLEEPER.clone()).await;
match result {
Ok(mut data_usage_info) => {
@@ -1012,7 +1080,14 @@ impl ScannerIODisk for Disk {
emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed());
data_usage_info.info.last_update = Some(SystemTime::now());
failure_guard.mark_not_failed();
Ok(data_usage_info)
Ok(ScannerDiskScanOutcome::Complete(data_usage_info))
}
Err(ScannerError::PartialCache(mut partial_cache)) => {
done_drive();
emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
failure_guard.mark_not_failed();
Ok(ScannerDiskScanOutcome::Partial(*partial_cache))
}
Err(e) => {
if ctx.is_cancelled() {
@@ -1115,4 +1190,78 @@ mod tests {
fn is_xl_meta_path_accepts_forward_separator() {
assert!(is_xl_meta_path("/data/bucket/object/xl.meta"));
}
#[test]
fn cache_root_entry_info_flattens_bucket_children() {
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
cache.replace(
"bucket",
DATA_USAGE_ROOT,
DataUsageEntry {
size: 10,
objects: 1,
..Default::default()
},
);
cache.replace(
"bucket/prefix",
"bucket",
DataUsageEntry {
size: 20,
objects: 2,
..Default::default()
},
);
let info = cache_root_entry_info(&cache);
assert_eq!(info.name, "bucket");
assert_eq!(info.parent, DATA_USAGE_ROOT);
assert_eq!(info.entry.size, 30);
assert_eq!(info.entry.objects, 3);
assert!(info.entry.children.is_empty());
}
#[tokio::test]
async fn send_cache_root_entry_info_sends_after_budget_cancellation() {
let ctx = CancellationToken::new();
ctx.cancel();
assert!(ctx.is_cancelled());
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
cache.replace(
"bucket",
DATA_USAGE_ROOT,
DataUsageEntry {
size: 10,
objects: 1,
..Default::default()
},
);
let (tx, mut rx) = mpsc::channel(1);
let tx = Arc::new(Mutex::new(tx));
send_cache_root_entry_info(&tx, &cache)
.await
.expect("partial cache should be sent even after budget cancellation");
let info = rx.recv().await.expect("partial cache entry should be received");
assert_eq!(info.name, "bucket");
assert_eq!(info.parent, DATA_USAGE_ROOT);
assert_eq!(info.entry.size, 10);
assert_eq!(info.entry.objects, 1);
}
}