mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 11:56:38 +00:00
Henry Guo
105b6fbfde
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
3946 lines
145 KiB
Rust
3946 lines
145 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
use super::*;
|
|
use crate::EcstoreResult;
|
|
use crate::{
|
|
Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerGetObjectReader as GetObjectReader,
|
|
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
|
|
init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
|
|
};
|
|
use serial_test::serial;
|
|
use std::collections::HashMap;
|
|
use std::io::Cursor;
|
|
use std::task::Poll;
|
|
use temp_env::{with_var, with_var_unset};
|
|
use tokio::io::AsyncReadExt;
|
|
use tokio::sync::Mutex;
|
|
|
|
const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
|
|
|
|
async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
|
init_ecstore_config_for_scanner_tests();
|
|
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
|
|
let mut endpoints = Vec::new();
|
|
for disk_index in 0..4 {
|
|
let disk_path = temp_dir.path().join(format!("disk{disk_index}"));
|
|
tokio::fs::create_dir_all(&disk_path)
|
|
.await
|
|
.expect("scanner cycle test disk should be created");
|
|
let mut endpoint =
|
|
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
|
|
endpoint.set_pool_index(0);
|
|
endpoint.set_set_index(0);
|
|
endpoint.set_disk_index(disk_index);
|
|
endpoints.push(endpoint);
|
|
}
|
|
let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints {
|
|
legacy: false,
|
|
set_count: 1,
|
|
drives_per_set: 4,
|
|
endpoints: Endpoints::from(endpoints),
|
|
cmd_line: "scanner-cycle-metrics".to_string(),
|
|
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
|
}]);
|
|
let instance_ctx = Arc::new(InstanceContext::new());
|
|
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
|
.await
|
|
.expect("scanner cycle test disks should initialize");
|
|
let store = ECStore::new_with_instance_ctx(
|
|
"127.0.0.1:0".parse().expect("test address should parse"),
|
|
endpoint_pools,
|
|
CancellationToken::new(),
|
|
instance_ctx,
|
|
)
|
|
.await
|
|
.expect("scanner cycle test ECStore should initialize");
|
|
init_bucket_metadata_sys_for_scanner_tests(store.clone()).await;
|
|
|
|
(temp_dir, store)
|
|
}
|
|
|
|
fn assert_run_data_scanner_signature<F, Fut>(_run: F)
|
|
where
|
|
F: Fn(CancellationToken, Arc<ECStore>) -> Fut,
|
|
Fut: Future<Output = Result<(), ScannerError>>,
|
|
{
|
|
}
|
|
|
|
#[test]
|
|
fn run_data_scanner_keeps_its_two_argument_api() {
|
|
assert_run_data_scanner_signature(run_data_scanner);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn scanner_cycle_lock_fence_cancels_cycle_context() {
|
|
let cycle_ctx = CancellationToken::new();
|
|
let observed_ctx = cycle_ctx.clone();
|
|
let output = await_scanner_cycle_with_lock_fence(
|
|
&cycle_ctx,
|
|
async move {
|
|
observed_ctx.cancelled().await;
|
|
observed_ctx.is_cancelled()
|
|
},
|
|
std::future::ready(()),
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(output, Some(true));
|
|
assert!(cycle_ctx.is_cancelled());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn scanner_cycle_lock_fence_preserves_completed_cycle() {
|
|
let cycle_ctx = CancellationToken::new();
|
|
let output = await_scanner_cycle_with_lock_fence(&cycle_ctx, std::future::ready(7_u8), std::future::pending()).await;
|
|
|
|
assert_eq!(output, Some(7));
|
|
assert!(!cycle_ctx.is_cancelled());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() {
|
|
let cycle_ctx = CancellationToken::new();
|
|
let output = await_scanner_cycle_with_lock_fence(&cycle_ctx, std::future::pending::<()>(), std::future::ready(())).await;
|
|
|
|
assert_eq!(output, None);
|
|
assert!(cycle_ctx.is_cancelled());
|
|
}
|
|
|
|
struct ScannerDefaultSpeedGuard;
|
|
|
|
impl ScannerDefaultSpeedGuard {
|
|
fn set(speed: ScannerSpeed) -> Self {
|
|
set_scanner_default_speed(speed);
|
|
Self
|
|
}
|
|
}
|
|
|
|
impl Drop for ScannerDefaultSpeedGuard {
|
|
fn drop(&mut self) {
|
|
set_scanner_default_speed(ScannerSpeed::Default);
|
|
}
|
|
}
|
|
|
|
struct ScannerDefaultCycleGuard;
|
|
|
|
impl ScannerDefaultCycleGuard {
|
|
fn set(secs: u64) -> Self {
|
|
set_scanner_default_cycle_secs(Some(secs));
|
|
Self
|
|
}
|
|
}
|
|
|
|
impl Drop for ScannerDefaultCycleGuard {
|
|
fn drop(&mut self) {
|
|
set_scanner_default_cycle_secs(None);
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Default)]
|
|
struct MemoryConfigStore {
|
|
objects: Mutex<HashMap<String, Vec<u8>>>,
|
|
revisions: Mutex<HashMap<String, u64>>,
|
|
fail_put_number: Mutex<HashMap<String, usize>>,
|
|
error_after_commit_put_number: Mutex<HashMap<String, usize>>,
|
|
interleaving_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
|
|
cancel_after_interleaving_puts: Mutex<HashMap<String, CancellationToken>>,
|
|
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
|
|
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
|
|
put_counts: Mutex<HashMap<String, usize>>,
|
|
}
|
|
|
|
fn memory_config_key(bucket: &str, object: &str) -> String {
|
|
format!("{bucket}/{object}")
|
|
}
|
|
|
|
#[async_trait::async_trait]
|
|
impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
|
|
type Error = EcstoreError;
|
|
type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec;
|
|
type HeaderMap = http::HeaderMap;
|
|
type ObjectOptions = ObjectOptions;
|
|
type ObjectInfo = ObjectInfo;
|
|
type GetObjectReader = GetObjectReader;
|
|
type PutObjectReader = PutObjReader;
|
|
|
|
async fn get_object_reader(
|
|
&self,
|
|
bucket: &str,
|
|
object: &str,
|
|
_range: Option<crate::storage_api::scanner_io::HTTPRangeSpec>,
|
|
_h: http::HeaderMap,
|
|
_opts: &ObjectOptions,
|
|
) -> EcstoreResult<GetObjectReader> {
|
|
let key = memory_config_key(bucket, object);
|
|
let data = self
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.get(&key)
|
|
.cloned()
|
|
.ok_or(EcstoreError::FileNotFound)?;
|
|
let revision = *self.revisions.lock().await.entry(key).or_insert(1);
|
|
|
|
Ok(GetObjectReader {
|
|
stream: Box::new(Cursor::new(data)),
|
|
object_info: ObjectInfo {
|
|
etag: Some(format!("memory-{revision}")),
|
|
..Default::default()
|
|
},
|
|
buffered_body: None,
|
|
body_source: Default::default(),
|
|
})
|
|
}
|
|
|
|
async fn put_object(
|
|
&self,
|
|
bucket: &str,
|
|
object: &str,
|
|
data: &mut PutObjReader,
|
|
opts: &ObjectOptions,
|
|
) -> EcstoreResult<ObjectInfo> {
|
|
let mut buf = Vec::new();
|
|
data.stream.read_to_end(&mut buf).await?;
|
|
let key = memory_config_key(bucket, object);
|
|
let put_count = {
|
|
let mut put_counts = self.put_counts.lock().await;
|
|
let put_count = put_counts.entry(key.clone()).or_insert(0);
|
|
*put_count += 1;
|
|
*put_count
|
|
};
|
|
|
|
if self.fail_put_number.lock().await.get(&key) == Some(&put_count) {
|
|
return Err(EcstoreError::other("injected put failure"));
|
|
}
|
|
|
|
let interleaving_data = {
|
|
let mut interleaving_puts = self.interleaving_puts.lock().await;
|
|
if interleaving_puts
|
|
.get(&key)
|
|
.is_some_and(|(expected_put, _)| *expected_put == put_count)
|
|
{
|
|
interleaving_puts.remove(&key).map(|(_, data)| data)
|
|
} else {
|
|
None
|
|
}
|
|
};
|
|
let cancel_after_interleaving = if interleaving_data.is_some() {
|
|
self.cancel_after_interleaving_puts.lock().await.remove(&key)
|
|
} else {
|
|
None
|
|
};
|
|
let replacement = {
|
|
let mut replacements = self.replace_after_successful_puts.lock().await;
|
|
if replacements
|
|
.get(&key)
|
|
.is_some_and(|(expected_put, _)| *expected_put == put_count)
|
|
{
|
|
replacements.remove(&key).map(|(_, replacement)| replacement)
|
|
} else {
|
|
None
|
|
}
|
|
};
|
|
let mut objects = self.objects.lock().await;
|
|
let mut revisions = self.revisions.lock().await;
|
|
if let Some(interleaving_data) = interleaving_data {
|
|
let revision = revisions.get(&key).copied().unwrap_or(0) + 1;
|
|
objects.insert(key.clone(), interleaving_data);
|
|
revisions.insert(key.clone(), revision);
|
|
if let Some(cancel) = cancel_after_interleaving {
|
|
cancel.cancel();
|
|
}
|
|
}
|
|
let current_revision = objects.contains_key(&key).then(|| revisions.get(&key).copied().unwrap_or(1));
|
|
if let Some(preconditions) = &opts.http_preconditions {
|
|
if preconditions
|
|
.if_none_match
|
|
.as_deref()
|
|
.is_some_and(|condition| !condition.trim().is_empty())
|
|
&& current_revision.is_some()
|
|
{
|
|
return Err(EcstoreError::PreconditionFailed);
|
|
}
|
|
if let Some(expected) = preconditions
|
|
.if_match
|
|
.as_deref()
|
|
.map(str::trim)
|
|
.filter(|value| !value.is_empty())
|
|
{
|
|
let actual = current_revision.map(|revision| format!("memory-{revision}"));
|
|
if actual.as_deref() != Some(expected.trim_matches('"')) {
|
|
return Err(EcstoreError::PreconditionFailed);
|
|
}
|
|
}
|
|
}
|
|
|
|
let revision = current_revision.unwrap_or(0) + 1;
|
|
objects.insert(key.clone(), buf);
|
|
revisions.insert(key.clone(), revision);
|
|
if let Some(replacement) = replacement {
|
|
objects.insert(key.clone(), replacement);
|
|
revisions.insert(key.clone(), revision + 1);
|
|
}
|
|
drop(revisions);
|
|
drop(objects);
|
|
let cancel_after_success = {
|
|
let mut cancellations = self.cancel_after_successful_puts.lock().await;
|
|
if cancellations
|
|
.get(&key)
|
|
.is_some_and(|(expected_put, _)| *expected_put == put_count)
|
|
{
|
|
cancellations.remove(&key).map(|(_, cancel)| cancel)
|
|
} else {
|
|
None
|
|
}
|
|
};
|
|
if let Some(cancel) = cancel_after_success {
|
|
cancel.cancel();
|
|
}
|
|
if self.error_after_commit_put_number.lock().await.get(&key) == Some(&put_count) {
|
|
return Err(EcstoreError::other("injected post-commit put failure"));
|
|
}
|
|
Ok(ObjectInfo {
|
|
etag: Some(format!("memory-{revision}")),
|
|
..Default::default()
|
|
})
|
|
}
|
|
}
|
|
|
|
fn with_unset_scanner_timing_env(f: impl FnOnce()) {
|
|
with_var_unset(ENV_SCANNER_SPEED, || {
|
|
with_var_unset("MINIO_SCANNER_SPEED", || {
|
|
with_var_unset(ENV_SCANNER_CYCLE, || {
|
|
with_var_unset("MINIO_SCANNER_CYCLE", || {
|
|
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
|
|
with_var_unset(ENV_SCANNER_START_DELAY_SECS_DEPRECATED, f);
|
|
});
|
|
});
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn test_randomized_cycle_delay_keeps_configured_start_delay() {
|
|
// 120s with ±10% jitter should stay clearly above the historic 30s cap.
|
|
let delay = randomized_cycle_delay_for(Duration::from_secs(120));
|
|
assert!(delay > Duration::from_secs(30), "expected delay > 30s, got {delay:?}");
|
|
// Jitter window should stay within configured bounds.
|
|
assert!(delay >= Duration::from_secs(108));
|
|
assert!(delay <= Duration::from_secs(132));
|
|
}
|
|
|
|
#[test]
|
|
fn test_randomized_cycle_delay_bounds_extreme_interval() {
|
|
let delay = randomized_cycle_delay_for(Duration::MAX);
|
|
|
|
assert!(delay >= MAX_SCANNER_SCHEDULE_DELAY.mul_f64(0.9));
|
|
assert!(delay <= MAX_SCANNER_SCHEDULE_DELAY);
|
|
}
|
|
|
|
#[test]
|
|
fn test_initial_scanner_delay_uses_configured_start_delay() {
|
|
let delay = initial_scanner_delay_for(Some(120));
|
|
assert!(delay >= Duration::from_secs(108));
|
|
assert!(delay <= Duration::from_secs(132));
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_initial_scanner_delay_uses_cycle_without_explicit_start_delay() {
|
|
with_var(ENV_SCANNER_CYCLE, Some("120"), || {
|
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
|
let delay = initial_scanner_delay_for(None);
|
|
assert!(delay >= Duration::from_secs(108));
|
|
assert!(delay <= Duration::from_secs(132));
|
|
});
|
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
|
}
|
|
|
|
#[test]
|
|
fn test_initial_scanner_delay_skips_for_cold_usage_cache_with_buckets() {
|
|
let delay = initial_scanner_delay_for_startup(Some(120), true, true, false);
|
|
assert_eq!(delay, Duration::ZERO);
|
|
}
|
|
|
|
#[test]
|
|
fn test_initial_scanner_delay_keeps_configured_delay_for_warm_usage_cache_no_replication() {
|
|
let delay = initial_scanner_delay_for_startup(Some(120), false, true, false);
|
|
assert!(delay >= Duration::from_secs(108));
|
|
assert!(delay <= Duration::from_secs(132));
|
|
}
|
|
|
|
#[test]
|
|
fn test_initial_scanner_delay_skips_for_cold_usage_cache_without_buckets() {
|
|
let delay = initial_scanner_delay_for_startup(Some(120), true, false, false);
|
|
assert_eq!(delay, Duration::ZERO);
|
|
}
|
|
|
|
#[test]
|
|
fn test_initial_scanner_delay_skips_for_active_replication_warm_cache() {
|
|
// Warm cache + active replication rules → skip startup delay so that FAILED-status objects
|
|
// from a crash are healed on the first cycle, not after a 27-33 min sleep.
|
|
let delay = initial_scanner_delay_for_startup(Some(120), false, true, true);
|
|
assert_eq!(delay, Duration::ZERO);
|
|
}
|
|
|
|
#[test]
|
|
fn test_initial_scanner_delay_keeps_delay_for_replication_without_buckets() {
|
|
// Active replication but no buckets → no objects to scan, keep normal delay.
|
|
let delay = initial_scanner_delay_for_startup(Some(120), false, false, true);
|
|
assert!(delay >= Duration::from_secs(108));
|
|
assert!(delay <= Duration::from_secs(132));
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_scanner_cycle_max_duration_uses_env() {
|
|
with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("42"), || {
|
|
assert_eq!(scanner_cycle_max_duration(), Some(Duration::from_secs(42)));
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_scanner_cycle_max_duration_default_is_disabled() {
|
|
with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
|
|
assert_eq!(scanner_cycle_max_duration(), None);
|
|
});
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_scanner_cycle_budget_cancels_after_duration() {
|
|
let parent = CancellationToken::new();
|
|
let budget = ScannerCycleBudget::new(
|
|
&parent,
|
|
ScannerCycleBudgetConfig {
|
|
max_duration: Some(Duration::from_millis(1)),
|
|
..Default::default()
|
|
},
|
|
);
|
|
|
|
tokio::time::timeout(Duration::from_secs(5), budget.token().cancelled())
|
|
.await
|
|
.expect("scanner cycle budget should cancel after max duration");
|
|
|
|
assert!(budget.budget_elapsed());
|
|
assert!(budget.token().is_cancelled());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_scanner_cycle_budget_drop_cancels_child_without_elapsed() {
|
|
let parent = CancellationToken::new();
|
|
let budget = ScannerCycleBudget::new(
|
|
&parent,
|
|
ScannerCycleBudgetConfig {
|
|
max_duration: Some(Duration::from_secs(60)),
|
|
..Default::default()
|
|
},
|
|
);
|
|
let token = budget.token();
|
|
|
|
drop(budget);
|
|
|
|
assert!(token.is_cancelled());
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_scanner_cycle_budget_config_uses_work_budget_env() {
|
|
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("100"), || {
|
|
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("25"), || {
|
|
let config = scanner_cycle_budget_config();
|
|
assert_eq!(config.max_objects, Some(100));
|
|
assert_eq!(config.max_directories, Some(25));
|
|
});
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_scanner_cycle_budget_config_disables_zero_work_budgets() {
|
|
with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("0"), || {
|
|
with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("0"), || {
|
|
let config = scanner_cycle_budget_config();
|
|
assert_eq!(config.max_objects, None);
|
|
assert_eq!(config.max_directories, None);
|
|
});
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn test_scan_cycle_partial_reason_maps_budget_reason() {
|
|
assert_eq!(
|
|
scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Runtime)),
|
|
ScanCyclePartialReason::Runtime
|
|
);
|
|
assert_eq!(
|
|
scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Objects)),
|
|
ScanCyclePartialReason::Objects
|
|
);
|
|
assert_eq!(
|
|
scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Directories)),
|
|
ScanCyclePartialReason::Directories
|
|
);
|
|
assert_eq!(scan_cycle_partial_reason(None), ScanCyclePartialReason::Unknown);
|
|
}
|
|
|
|
#[test]
|
|
fn test_scan_cycle_partial_source_maps_budget_reason() {
|
|
assert_eq!(scan_cycle_partial_source(Some(ScannerCycleBudgetReason::Runtime)), None);
|
|
assert_eq!(
|
|
scan_cycle_partial_source(Some(ScannerCycleBudgetReason::Objects)),
|
|
Some(ScannerWorkSource::Usage)
|
|
);
|
|
assert_eq!(
|
|
scan_cycle_partial_source(Some(ScannerCycleBudgetReason::Directories)),
|
|
Some(ScannerWorkSource::Usage)
|
|
);
|
|
assert_eq!(scan_cycle_partial_source(None), None);
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn test_mark_scan_cycle_idle_clears_published_cycle_state() {
|
|
let mut cycle_info = CurrentCycle {
|
|
current: 12,
|
|
next: 13,
|
|
cycle_completed: vec![Utc::now()],
|
|
started: Utc::now(),
|
|
};
|
|
|
|
global_metrics().set_current_scan_mode(HealScanMode::Deep);
|
|
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
|
|
|
mark_scan_cycle_idle(&mut cycle_info, &mut cycle_metrics_guard).await;
|
|
|
|
let published = global_metrics()
|
|
.get_cycle()
|
|
.await
|
|
.expect("scanner cycle state should remain published");
|
|
|
|
assert_eq!(cycle_info.current, 0);
|
|
assert_eq!(cycle_info.next, 13);
|
|
assert_eq!(published.current, 0);
|
|
assert_eq!(published.next, 13);
|
|
assert_eq!(global_metrics().current_scan_mode(), HealScanMode::Unknown);
|
|
|
|
global_metrics().set_cycle(None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
|
|
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
|
let mut cycle_info = CurrentCycle {
|
|
current: 0,
|
|
next: 1,
|
|
started: cycle_started,
|
|
..Default::default()
|
|
};
|
|
let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
|
let setup_report = global_metrics().report().await;
|
|
assert!(setup_report.current_cycle_active);
|
|
assert_eq!(setup_report.current_cycle, 0);
|
|
assert_eq!(setup_report.current_started.as_second(), cycle_started.timestamp());
|
|
assert_eq!(
|
|
setup_report.current_started.subsec_nanosecond(),
|
|
i32::try_from(cycle_started.timestamp_subsec_nanos()).expect("chrono nanoseconds fit in i32")
|
|
);
|
|
|
|
mark_scan_cycle_idle(&mut cycle_info, &mut guard).await;
|
|
let idle_report = global_metrics().report().await;
|
|
assert!(!idle_report.current_cycle_active);
|
|
|
|
global_metrics().set_cycle(None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finalization() {
|
|
let mut cycle_info = CurrentCycle {
|
|
current: 12,
|
|
next: 13,
|
|
started: Utc::now(),
|
|
..Default::default()
|
|
};
|
|
let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
|
|
|
cycle_info.current = 0;
|
|
tokio::task::yield_now().await;
|
|
let finalizing_report = global_metrics().report().await;
|
|
assert!(finalizing_report.current_cycle_active);
|
|
assert_eq!(finalizing_report.current_cycle, 12);
|
|
|
|
guard.finish(cycle_info).await;
|
|
let idle_report = global_metrics().report().await;
|
|
assert!(!idle_report.current_cycle_active);
|
|
assert_eq!(idle_report.current_cycle, 0);
|
|
|
|
global_metrics().set_cycle(None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn scanner_cycle_metrics_guard_drop_clears_activity() {
|
|
let guard = ScannerCycleMetricsGuard::new(CurrentCycle {
|
|
current: 12,
|
|
next: 13,
|
|
started: Utc::now(),
|
|
..Default::default()
|
|
})
|
|
.await;
|
|
assert!(global_metrics().report().await.current_cycle_active);
|
|
|
|
drop(guard);
|
|
|
|
assert!(!global_metrics().report().await.current_cycle_active);
|
|
global_metrics().set_cycle(None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
|
|
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
|
let ctx = CancellationToken::new();
|
|
let mut cycle_info = CurrentCycle::default();
|
|
let mut revision = DataUsageCacheRevision::Missing;
|
|
let leader_epoch = u64::MAX - 1;
|
|
let state_persist_reached = Arc::new(Notify::new());
|
|
let _state_persist_hook = set_scanner_cycle_state_persist_test_hook(leader_epoch, state_persist_reached.clone());
|
|
let state_lock = store
|
|
.new_ns_lock(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
|
.await
|
|
.expect("scanner cycle state lock should be created");
|
|
let state_guard = state_lock
|
|
.get_write_lock(Duration::from_secs(1))
|
|
.await
|
|
.expect("scanner cycle state lock should be acquired");
|
|
let mut cycle = Box::pin(run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch));
|
|
let waker = std::task::Waker::noop();
|
|
let mut context = std::task::Context::from_waker(waker);
|
|
|
|
assert!(cycle.as_mut().poll(&mut context).is_pending());
|
|
let active = global_metrics().report().await;
|
|
assert!(active.current_cycle_active);
|
|
assert_eq!(active.current_cycle, 0);
|
|
|
|
tokio::time::timeout(Duration::from_secs(30), async {
|
|
tokio::select! {
|
|
outcome = &mut cycle => panic!("scanner cycle finished before state persistence was released: {outcome:?}"),
|
|
_ = state_persist_reached.notified() => {}
|
|
}
|
|
})
|
|
.await
|
|
.expect("scanner cycle should reach state persistence");
|
|
let finalizing = global_metrics().report().await;
|
|
assert!(finalizing.current_cycle_active);
|
|
|
|
drop(state_guard);
|
|
let outcome = tokio::time::timeout(Duration::from_secs(30), cycle)
|
|
.await
|
|
.expect("scanner cycle should finish");
|
|
assert!(matches!(
|
|
outcome,
|
|
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
|
|
));
|
|
assert!(!global_metrics().report().await.current_cycle_active);
|
|
|
|
global_metrics().set_cycle(None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let ctx = CancellationToken::new();
|
|
let mut revision = DataUsageCacheRevision::Missing;
|
|
let mut cycle_info = CurrentCycle {
|
|
current: 12,
|
|
next: 12,
|
|
cycle_completed: vec![],
|
|
started: Utc::now(),
|
|
};
|
|
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
|
|
|
assert!(finalize_partial_scan_cycle(&ctx, store.clone(), &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await);
|
|
|
|
assert_eq!(cycle_info.next, 13);
|
|
assert_eq!(cycle_info.current, 0);
|
|
assert!(cycle_info.cycle_completed.is_empty());
|
|
assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-1"));
|
|
|
|
let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
|
|
.await
|
|
.expect("cycle state should be persisted after a partial cycle");
|
|
assert_eq!(
|
|
u64::from_le_bytes(buf[0..8].try_into().expect("persisted state should start with the counter")),
|
|
13
|
|
);
|
|
let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle info should decode");
|
|
assert_eq!(decoded.next, 13);
|
|
assert_eq!(decoded.current, 0);
|
|
assert_eq!(epoch, 1);
|
|
|
|
global_metrics().set_cycle(None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn scanner_cycle_recovers_to_newer_durable_cache_floor() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let ctx = CancellationToken::new();
|
|
let mut revision = DataUsageCacheRevision::Missing;
|
|
let mut cycle_info = CurrentCycle {
|
|
current: 12,
|
|
next: 12,
|
|
cycle_completed: vec![],
|
|
started: Utc::now(),
|
|
};
|
|
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
|
|
|
assert!(
|
|
persist_required_scanner_cycle_floor(
|
|
&ctx,
|
|
store.clone(),
|
|
&mut cycle_info,
|
|
&mut revision,
|
|
7,
|
|
19,
|
|
&mut cycle_metrics_guard,
|
|
)
|
|
.await
|
|
);
|
|
assert_eq!(cycle_info.current, 0);
|
|
assert_eq!(cycle_info.next, 19);
|
|
|
|
let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
|
|
.await
|
|
.expect("recovered cycle floor should be persisted");
|
|
let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("recovered cycle state should decode");
|
|
assert_eq!(decoded.current, 0);
|
|
assert_eq!(decoded.next, 19);
|
|
assert_eq!(epoch, 7);
|
|
|
|
global_metrics().set_cycle(None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn scanner_cycle_rejects_invalid_cache_floor() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let ctx = CancellationToken::new();
|
|
let mut revision = DataUsageCacheRevision::Missing;
|
|
let mut cycle_info = CurrentCycle {
|
|
current: 12,
|
|
next: 12,
|
|
cycle_completed: vec![],
|
|
started: Utc::now(),
|
|
};
|
|
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
|
|
|
assert!(
|
|
!persist_required_scanner_cycle_floor(
|
|
&ctx,
|
|
store.clone(),
|
|
&mut cycle_info,
|
|
&mut revision,
|
|
7,
|
|
12,
|
|
&mut cycle_metrics_guard,
|
|
)
|
|
.await
|
|
);
|
|
assert_eq!(cycle_info.next, 12);
|
|
assert_eq!(revision, DataUsageCacheRevision::Missing);
|
|
let mut max_cycle_info = CurrentCycle {
|
|
current: 12,
|
|
next: 12,
|
|
..Default::default()
|
|
};
|
|
let mut max_cycle_metrics_guard = ScannerCycleMetricsGuard::new(max_cycle_info.clone()).await;
|
|
assert!(
|
|
!persist_required_scanner_cycle_floor(
|
|
&ctx,
|
|
store.clone(),
|
|
&mut max_cycle_info,
|
|
&mut revision,
|
|
7,
|
|
u64::MAX,
|
|
&mut max_cycle_metrics_guard,
|
|
)
|
|
.await
|
|
);
|
|
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
|
|
|
|
global_metrics().set_cycle(None).await;
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_cycle_state_decodes_legacy_and_fenced_formats() {
|
|
let cycle = CurrentCycle {
|
|
current: 12,
|
|
next: 13,
|
|
cycle_completed: vec![],
|
|
started: Utc::now(),
|
|
};
|
|
let mut legacy = cycle.next.to_le_bytes().to_vec();
|
|
legacy.extend(cycle.marshal().expect("legacy cycle state should encode"));
|
|
|
|
let (legacy_cycle, legacy_epoch) = decode_scanner_cycle_state(&legacy).expect("legacy cycle state should remain readable");
|
|
assert_eq!(legacy_cycle.next, 13);
|
|
assert_eq!(legacy_epoch, 0);
|
|
|
|
let fenced = encode_scanner_cycle_state(&cycle, 7).expect("fenced cycle state should encode");
|
|
let (fenced_cycle, fenced_epoch) = decode_scanner_cycle_state(&fenced).expect("fenced cycle state should decode");
|
|
assert_eq!(fenced_cycle.next, 13);
|
|
assert_eq!(fenced_epoch, 7);
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_startup_fails_closed_on_nonempty_corrupt_cycle_state() {
|
|
assert_eq!(
|
|
decode_scanner_cycle_state_for_startup(&[])
|
|
.expect("missing cycle state should use defaults")
|
|
.1,
|
|
0
|
|
);
|
|
assert!(decode_scanner_cycle_state_for_startup(&[1]).is_err());
|
|
|
|
let mut corrupt_fenced = 13_u64.to_le_bytes().to_vec();
|
|
corrupt_fenced.extend_from_slice(SCANNER_CYCLE_STATE_MAGIC);
|
|
corrupt_fenced.extend_from_slice(&7_u64.to_le_bytes());
|
|
corrupt_fenced.extend_from_slice(b"not-msgpack");
|
|
assert!(decode_scanner_cycle_state_for_startup(&corrupt_fenced).is_err());
|
|
assert!(decode_scanner_cycle_state_for_startup(&u64::MAX.to_le_bytes()).is_err());
|
|
|
|
let exhausted = CurrentCycle {
|
|
next: u64::MAX,
|
|
..Default::default()
|
|
};
|
|
assert!(encode_scanner_cycle_state(&exhausted, 7).is_err());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn scanner_startup_uses_primary_and_backup_usage_floor() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 11, 103)] {
|
|
store.objects.lock().await.insert(
|
|
memory_config_key(RUSTFS_META_BUCKET, path),
|
|
serde_json::to_vec(&DataUsageInfo {
|
|
scanner_epoch: Some(epoch),
|
|
scanner_cycle: Some(cycle),
|
|
..Default::default()
|
|
})
|
|
.expect("usage snapshot should encode"),
|
|
);
|
|
}
|
|
|
|
let floor = persisted_usage_floor(store).await.expect("usage floor should load");
|
|
assert_eq!(
|
|
floor,
|
|
PersistedUsageFloor {
|
|
next_cycle: 104,
|
|
leader_epoch: 11,
|
|
}
|
|
);
|
|
|
|
let mut cycle = CurrentCycle::default();
|
|
let mut epoch = 0;
|
|
apply_persisted_usage_floor(&mut cycle, &mut epoch, floor);
|
|
assert_eq!(cycle.next, 104);
|
|
assert_eq!(epoch, 11);
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() {
|
|
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::now()), 1);
|
|
legacy.usage_snapshot_complete = false;
|
|
|
|
assert!(data_usage_info_is_cold(&legacy));
|
|
assert!(!data_usage_info_is_cold(&complete_usage_with_bucket_count(
|
|
Some(std::time::SystemTime::now()),
|
|
1,
|
|
)));
|
|
assert!(!data_usage_info_is_cold(&DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::now()),
|
|
usage_snapshot_complete: true,
|
|
..Default::default()
|
|
}));
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_startup_prompts_only_for_a_newer_valid_observation() {
|
|
let authoritative = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
|
|
scanner_epoch: Some(4),
|
|
scanner_cycle: Some(10),
|
|
..complete_usage_with_bucket_count(None, 0)
|
|
};
|
|
let observed = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
|
|
scanner_epoch: Some(4),
|
|
scanner_cycle: Some(11),
|
|
usage_snapshot_converged: Some(false),
|
|
usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
|
|
..complete_usage_with_bucket_count(None, 0)
|
|
};
|
|
|
|
assert!(usage_cache_needs_prompt_scan(&authoritative, Some(&observed)));
|
|
assert!(!usage_cache_needs_prompt_scan(&authoritative, None));
|
|
|
|
let mut converged = observed.clone();
|
|
converged.usage_snapshot_converged = Some(true);
|
|
assert!(!usage_cache_needs_prompt_scan(&authoritative, Some(&converged)));
|
|
|
|
let mut legacy_observation = observed;
|
|
legacy_observation.usage_snapshot_converged = None;
|
|
assert!(!usage_cache_needs_prompt_scan(&authoritative, Some(&legacy_observation)));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn scanner_startup_prefers_v2_over_legacy_usage() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let legacy = DataUsageInfo {
|
|
scanner_epoch: Some(19),
|
|
scanner_cycle: Some(41),
|
|
last_update: Some(std::time::SystemTime::now()),
|
|
..Default::default()
|
|
};
|
|
let legacy_data = serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode");
|
|
store.objects.lock().await.insert(
|
|
memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
|
legacy_data.clone(),
|
|
);
|
|
|
|
assert_eq!(
|
|
read_data_usage_config_for_startup(&store)
|
|
.await
|
|
.expect("legacy startup usage should load"),
|
|
Some(legacy_data)
|
|
);
|
|
assert_eq!(
|
|
persisted_usage_floor(store.clone())
|
|
.await
|
|
.expect("legacy usage floor should seed the upgrade"),
|
|
PersistedUsageFloor {
|
|
next_cycle: 42,
|
|
leader_epoch: 19,
|
|
}
|
|
);
|
|
|
|
let authoritative = DataUsageInfo {
|
|
scanner_epoch: Some(23),
|
|
scanner_cycle: Some(51),
|
|
last_update: Some(std::time::SystemTime::now()),
|
|
usage_snapshot_complete: true,
|
|
..Default::default()
|
|
};
|
|
let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode");
|
|
store.objects.lock().await.insert(
|
|
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
|
authoritative_data.clone(),
|
|
);
|
|
|
|
assert_eq!(
|
|
read_data_usage_config_for_startup(&store)
|
|
.await
|
|
.expect("v2 startup usage should load"),
|
|
Some(authoritative_data)
|
|
);
|
|
assert_eq!(
|
|
persisted_usage_floor(store.clone())
|
|
.await
|
|
.expect("v2 usage floor should be authoritative"),
|
|
PersistedUsageFloor {
|
|
next_cycle: 52,
|
|
leader_epoch: 23,
|
|
}
|
|
);
|
|
|
|
store.objects.lock().await.insert(
|
|
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
|
b"corrupt-v2".to_vec(),
|
|
);
|
|
assert_eq!(
|
|
read_data_usage_config_for_startup(&store)
|
|
.await
|
|
.expect("startup inspection should preserve authoritative bytes"),
|
|
Some(b"corrupt-v2".to_vec())
|
|
);
|
|
assert!(
|
|
persisted_usage_floor(store).await.is_err(),
|
|
"corrupt v2 state must not fall back to a legacy writer"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
store.objects.lock().await.insert(
|
|
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
|
b"not-json".to_vec(),
|
|
);
|
|
|
|
assert!(persisted_usage_floor(store.clone()).await.is_err());
|
|
|
|
store.objects.lock().await.insert(
|
|
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
|
serde_json::to_vec(&DataUsageInfo {
|
|
scanner_cycle: Some(u64::MAX - 1),
|
|
..Default::default()
|
|
})
|
|
.expect("usage snapshot should encode"),
|
|
);
|
|
assert!(persisted_usage_floor(store).await.is_err());
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let ctx = CancellationToken::new();
|
|
|
|
for cycle in [9, 10] {
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
sender
|
|
.send(DataUsageInfo {
|
|
scanner_epoch: Some(1),
|
|
scanner_cycle: Some(cycle),
|
|
last_update: Some(std::time::SystemTime::now()),
|
|
..complete_usage_with_bucket_count(None, 0)
|
|
})
|
|
.await
|
|
.expect("usage update should queue");
|
|
drop(sender);
|
|
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome(ctx.clone(), store.clone(), receiver).await,
|
|
DataUsagePersistOutcome::Saved
|
|
);
|
|
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let backup = read_config(store.clone(), &backup_path).await;
|
|
if cycle == 9 {
|
|
assert!(matches!(backup, Err(EcstoreError::ConfigNotFound)));
|
|
} else {
|
|
let saved =
|
|
serde_json::from_slice::<DataUsageInfo>(&backup.expect("the tenth durable scanner cycle should create a backup"))
|
|
.expect("backup usage snapshot should decode");
|
|
assert_eq!(saved.scanner_cycle, Some(10));
|
|
assert_eq!(saved.scanner_epoch, Some(1));
|
|
}
|
|
}
|
|
}
|
|
|
|
#[async_trait::async_trait]
|
|
impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
|
|
async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult<ObjectInfo> {
|
|
let key = memory_config_key(bucket, object);
|
|
let mut objects = self.objects.lock().await;
|
|
if !objects.contains_key(&key) {
|
|
return Err(EcstoreError::FileNotFound);
|
|
}
|
|
let mut revisions = self.revisions.lock().await;
|
|
if let Some(expected) = opts
|
|
.http_preconditions
|
|
.as_ref()
|
|
.and_then(|preconditions| preconditions.if_match.as_deref())
|
|
{
|
|
let actual = revisions.get(&key).map(|revision| format!("memory-{revision}"));
|
|
if actual.as_deref() != Some(expected.trim_matches('"')) {
|
|
return Err(EcstoreError::PreconditionFailed);
|
|
}
|
|
}
|
|
objects.remove(&key);
|
|
revisions.remove(&key);
|
|
Ok(ObjectInfo::default())
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_cycle_advance_fails_before_reserved_exhausted_value() {
|
|
let mut cycle = CurrentCycle {
|
|
next: u64::MAX - 2,
|
|
..Default::default()
|
|
};
|
|
advance_scanner_cycle(&mut cycle).expect("last persistable scanner cycle should remain valid");
|
|
assert_eq!(cycle.next, u64::MAX - 1);
|
|
assert!(advance_scanner_cycle(&mut cycle).is_err());
|
|
assert_eq!(cycle.next, u64::MAX - 1);
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn test_finalize_partial_scan_cycle_reports_persist_failure() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let ctx = CancellationToken::new();
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
|
|
store.fail_put_number.lock().await.insert(key, 1);
|
|
let mut revision = DataUsageCacheRevision::Missing;
|
|
let mut cycle_info = CurrentCycle {
|
|
current: 12,
|
|
next: 12,
|
|
cycle_completed: vec![],
|
|
started: Utc::now(),
|
|
};
|
|
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
|
|
|
assert!(!finalize_partial_scan_cycle(&ctx, store, &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await);
|
|
assert_eq!(cycle_info.next, 13);
|
|
assert_eq!(cycle_info.current, 0);
|
|
assert_eq!(revision, DataUsageCacheRevision::Missing);
|
|
|
|
global_metrics().set_cycle(None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn test_persist_scanner_cycle_state_reconciles_newer_winner() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let ctx = CancellationToken::new();
|
|
let mut initial_revision = DataUsageCacheRevision::Missing;
|
|
let mut initial = CurrentCycle {
|
|
current: 0,
|
|
next: 12,
|
|
cycle_completed: vec![],
|
|
started: Utc::now(),
|
|
};
|
|
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut initial_revision, 1).await);
|
|
|
|
let mut current_revision = initial_revision.clone();
|
|
let mut stale_revision = initial_revision;
|
|
let mut current = CurrentCycle {
|
|
next: 14,
|
|
..initial.clone()
|
|
};
|
|
let mut stale = CurrentCycle { next: 13, ..initial };
|
|
|
|
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut current, &mut current_revision, 1).await);
|
|
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut stale, &mut stale_revision, 1).await);
|
|
|
|
let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
|
|
.await
|
|
.expect("new leader cycle state should remain persisted");
|
|
let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode");
|
|
assert_eq!(decoded.next, 14);
|
|
assert_eq!(epoch, 1);
|
|
assert_eq!(stale.next, 14);
|
|
assert!(matches!(current_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2"));
|
|
assert!(matches!(stale_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2"));
|
|
|
|
global_metrics().set_cycle(None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_persist_scanner_cycle_state_retries_after_stale_winner() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let ctx = CancellationToken::new();
|
|
let mut initial_revision = DataUsageCacheRevision::Missing;
|
|
let mut initial = CurrentCycle {
|
|
current: 0,
|
|
next: 12,
|
|
cycle_completed: vec![],
|
|
started: Utc::now(),
|
|
};
|
|
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut initial_revision, 1).await);
|
|
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
|
|
let stale = CurrentCycle {
|
|
next: 13,
|
|
..initial.clone()
|
|
};
|
|
let stale_buf = encode_scanner_cycle_state(&stale, 1).expect("stale cycle state should encode");
|
|
store.interleaving_puts.lock().await.insert(key, (2, stale_buf));
|
|
|
|
let mut current = CurrentCycle { next: 14, ..initial };
|
|
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut current, &mut initial_revision, 1).await);
|
|
|
|
let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
|
|
.await
|
|
.expect("newer cycle state should replace the stale conflict winner");
|
|
let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode");
|
|
assert_eq!(decoded.next, 14);
|
|
assert_eq!(epoch, 1);
|
|
assert_eq!(current.next, 14);
|
|
assert!(matches!(initial_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-3"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_persist_scanner_cycle_state_stops_retry_after_leader_fence() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let ctx = CancellationToken::new();
|
|
let mut revision = DataUsageCacheRevision::Missing;
|
|
let mut initial = CurrentCycle {
|
|
current: 0,
|
|
next: 12,
|
|
cycle_completed: vec![],
|
|
started: Utc::now(),
|
|
};
|
|
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut revision, 1).await);
|
|
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
|
|
let replacement = CurrentCycle {
|
|
next: 13,
|
|
..initial.clone()
|
|
};
|
|
let replacement_buf = encode_scanner_cycle_state(&replacement, 2).expect("replacement cycle state should encode");
|
|
store.interleaving_puts.lock().await.insert(key.clone(), (2, replacement_buf));
|
|
store
|
|
.cancel_after_interleaving_puts
|
|
.lock()
|
|
.await
|
|
.insert(key.clone(), ctx.clone());
|
|
|
|
let mut stale_leader = CurrentCycle { next: 14, ..initial };
|
|
assert!(!persist_scanner_cycle_state(&ctx, store.clone(), &mut stale_leader, &mut revision, 1).await);
|
|
|
|
let buf = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH)
|
|
.await
|
|
.expect("replacement leader cycle state should remain persisted");
|
|
let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode");
|
|
assert_eq!(decoded.next, 13);
|
|
assert_eq!(epoch, 2);
|
|
assert_eq!(stale_leader.next, 14);
|
|
assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2"));
|
|
assert_eq!(store.put_counts.lock().await.get(&key), Some(&2));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conflict() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let ctx = CancellationToken::new();
|
|
let mut revision = DataUsageCacheRevision::Missing;
|
|
let mut cycle = CurrentCycle {
|
|
current: 0,
|
|
next: 12,
|
|
cycle_completed: vec![],
|
|
started: Utc::now(),
|
|
};
|
|
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
|
|
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
|
|
let old_epoch_commit = CurrentCycle {
|
|
next: 14,
|
|
..cycle.clone()
|
|
};
|
|
store.interleaving_puts.lock().await.insert(
|
|
key.clone(),
|
|
(
|
|
2,
|
|
encode_scanner_cycle_state(&old_epoch_commit, 1).expect("old-epoch cycle state should encode"),
|
|
),
|
|
);
|
|
|
|
let mut persisted_epoch = 8;
|
|
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
|
|
|
|
let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH)
|
|
.await
|
|
.expect("new leadership claim should remain persisted");
|
|
let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("claimed cycle state should decode");
|
|
assert_eq!(claimed_cycle.next, 14);
|
|
assert_eq!(claimed_epoch, 9);
|
|
assert_eq!(persisted_epoch, 9);
|
|
assert_eq!(store.put_counts.lock().await.get(&key), Some(&3));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_leadership_claim_confirms_commit_after_returned_error() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let ctx = CancellationToken::new();
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
|
|
let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
store.error_after_commit_put_number.lock().await.insert(key.clone(), 1);
|
|
store.error_after_commit_put_number.lock().await.insert(usage_key.clone(), 1);
|
|
let mut revision = DataUsageCacheRevision::Missing;
|
|
let mut cycle = CurrentCycle {
|
|
current: 0,
|
|
next: 12,
|
|
cycle_completed: vec![],
|
|
started: Utc::now(),
|
|
};
|
|
let mut persisted_epoch = 0;
|
|
|
|
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await);
|
|
|
|
let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH)
|
|
.await
|
|
.expect("ambiguous leadership claim should be durable");
|
|
let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("claimed cycle state should decode");
|
|
assert_eq!(claimed_cycle.next, 12);
|
|
assert_eq!(claimed_epoch, 1);
|
|
assert_eq!(persisted_epoch, 1);
|
|
assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-1"));
|
|
assert_eq!(store.put_counts.lock().await.get(&key), Some(&1));
|
|
let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
|
.await
|
|
.expect("ambiguous usage epoch fence should be durable");
|
|
assert_eq!(
|
|
serde_json::from_slice::<DataUsageInfo>(&usage)
|
|
.expect("usage epoch fence should decode")
|
|
.scanner_epoch,
|
|
Some(1)
|
|
);
|
|
assert_eq!(store.put_counts.lock().await.get(&usage_key), Some(&1));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let mut old_usage = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
|
scanner_epoch: Some(4),
|
|
scanner_cycle: Some(11),
|
|
..Default::default()
|
|
};
|
|
old_usage.buckets_usage.insert(
|
|
"bucket-a".to_string(),
|
|
rustfs_data_usage::BucketUsageInfo {
|
|
objects_count: 2,
|
|
size: 84,
|
|
..Default::default()
|
|
},
|
|
);
|
|
old_usage.buckets_count = 1;
|
|
old_usage.calculate_totals();
|
|
let old_data = serde_json::to_vec(&old_usage).expect("old usage snapshot should encode");
|
|
store.objects.lock().await.insert(usage_key.clone(), old_data.clone());
|
|
store.revisions.lock().await.insert(usage_key, 1);
|
|
|
|
let ctx = CancellationToken::new();
|
|
let mut revision = DataUsageCacheRevision::Missing;
|
|
let mut cycle = CurrentCycle {
|
|
next: 12,
|
|
started: Utc::now(),
|
|
..Default::default()
|
|
};
|
|
let mut persisted_epoch = 4;
|
|
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await);
|
|
|
|
let (fenced_data, fenced_revision) = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
|
.await
|
|
.expect("fenced usage snapshot should load");
|
|
let fenced = serde_json::from_slice::<DataUsageInfo>(fenced_data.as_deref().expect("fenced usage snapshot should exist"))
|
|
.expect("fenced usage snapshot should decode");
|
|
assert_eq!(fenced.scanner_epoch, Some(5));
|
|
assert_eq!(fenced.objects_total_count, 2);
|
|
assert_eq!(fenced.buckets_usage.get("bucket-a").map(|usage| usage.size), Some(84));
|
|
assert!(matches!(fenced_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2"));
|
|
|
|
let stale_save = save_config_with_preconditions(
|
|
store,
|
|
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
|
old_data,
|
|
DataUsageCacheRevision::Etag("memory-1".to_string()).preconditions(),
|
|
)
|
|
.await;
|
|
assert!(matches!(stale_save, Err(EcstoreError::PreconditionFailed)));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_successful_old_epoch_commit_is_fenced_after_cancellation() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let ctx = CancellationToken::new();
|
|
let mut revision = DataUsageCacheRevision::Missing;
|
|
let mut cycle = CurrentCycle {
|
|
current: 0,
|
|
next: 12,
|
|
cycle_completed: vec![],
|
|
started: Utc::now(),
|
|
};
|
|
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
|
|
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
|
|
store
|
|
.cancel_after_successful_puts
|
|
.lock()
|
|
.await
|
|
.insert(key.clone(), (2, ctx.clone()));
|
|
cycle.next = 14;
|
|
assert!(!persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
|
|
|
|
let (persisted, persisted_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
|
.await
|
|
.expect("committed old-epoch state should load");
|
|
let mut replacement_cycle = decode_scanner_cycle_state(
|
|
persisted
|
|
.as_deref()
|
|
.expect("old-epoch state should have committed before cancellation"),
|
|
)
|
|
.expect("old-epoch state should decode")
|
|
.0;
|
|
let mut replacement_revision = persisted_revision;
|
|
let mut replacement_epoch = 1;
|
|
let replacement_ctx = CancellationToken::new();
|
|
assert!(
|
|
claim_scanner_leadership(
|
|
&replacement_ctx,
|
|
store.clone(),
|
|
&mut replacement_cycle,
|
|
&mut replacement_revision,
|
|
&mut replacement_epoch,
|
|
)
|
|
.await
|
|
);
|
|
|
|
let state = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
|
|
.await
|
|
.expect("replacement leadership claim should persist");
|
|
let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("replacement cycle state should decode");
|
|
assert_eq!(claimed_cycle.next, 14);
|
|
assert_eq!(claimed_epoch, 2);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_preserves_newer_snapshot() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let (sender, receiver) = mpsc::channel(2);
|
|
let ctx = CancellationToken::new();
|
|
|
|
let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
|
|
let older = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
|
|
|
|
sender.send(newer).await.expect("newer usage snapshot should enqueue");
|
|
sender.send(older).await.expect("older usage snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await;
|
|
|
|
let objects = store.objects.lock().await;
|
|
let saved = objects
|
|
.get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()))
|
|
.expect("data usage config should be saved");
|
|
let saved = serde_json::from_slice::<DataUsageInfo>(saved).expect("saved usage snapshot should decode");
|
|
|
|
assert_eq!(saved.buckets_count, 2);
|
|
assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)));
|
|
assert_eq!(outcome, DataUsagePersistOutcome::Current);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_fences_interleaving_newer_writer() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
let ctx = CancellationToken::new();
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
|
|
let stale = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
|
|
store
|
|
.interleaving_puts
|
|
.lock()
|
|
.await
|
|
.insert(key.clone(), (1, serde_json::to_vec(&newer).expect("newer usage snapshot should encode")));
|
|
|
|
sender.send(stale).await.expect("stale usage snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await;
|
|
|
|
let objects = store.objects.lock().await;
|
|
let saved = objects
|
|
.get(&key)
|
|
.expect("interleaving newer usage snapshot should remain saved");
|
|
let saved = serde_json::from_slice::<DataUsageInfo>(saved).expect("saved usage snapshot should decode");
|
|
assert_eq!(saved.buckets_count, 2);
|
|
assert_eq!(saved.last_update, newer.last_update);
|
|
assert_eq!(outcome, DataUsagePersistOutcome::Current);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_does_not_resurrect_deleted_bucket_after_conflict() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let mut initial = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
|
scanner_epoch: Some(8),
|
|
scanner_cycle: Some(12),
|
|
..Default::default()
|
|
};
|
|
initial.buckets_usage.insert(
|
|
"bucket-a".to_string(),
|
|
rustfs_data_usage::BucketUsageInfo {
|
|
objects_count: 2,
|
|
size: 84,
|
|
..Default::default()
|
|
},
|
|
);
|
|
initial.bucket_sizes.insert("bucket-a".to_string(), 84);
|
|
initial.buckets_count = 1;
|
|
initial.calculate_totals();
|
|
mark_usage_snapshot_complete(&mut initial);
|
|
let initial_data = serde_json::to_vec(&initial).expect("initial usage snapshot should encode");
|
|
store.objects.lock().await.insert(key.clone(), initial_data.clone());
|
|
store.revisions.lock().await.insert(key.clone(), 1);
|
|
|
|
let mut deleted = initial.clone();
|
|
deleted.buckets_usage.clear();
|
|
deleted.bucket_sizes.clear();
|
|
deleted.buckets_count = 0;
|
|
deleted.calculate_totals();
|
|
mark_usage_snapshot_complete(&mut deleted);
|
|
store
|
|
.interleaving_puts
|
|
.lock()
|
|
.await
|
|
.insert(key.clone(), (1, serde_json::to_vec(&deleted).expect("deleted snapshot should encode")));
|
|
|
|
let mut incoming = initial;
|
|
incoming.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30));
|
|
incoming.scanner_cycle = Some(13);
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
sender.send(incoming).await.expect("stale scanner snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
|
|
CancellationToken::new(),
|
|
store.clone(),
|
|
receiver,
|
|
Some(8),
|
|
Some(DataUsagePersistBaseline {
|
|
data: Some(Bytes::from(initial_data)),
|
|
revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
|
|
}),
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(outcome, DataUsagePersistOutcome::Current);
|
|
let saved = store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.get(&key)
|
|
.cloned()
|
|
.expect("deleted usage snapshot should remain");
|
|
let saved = serde_json::from_slice::<DataUsageInfo>(&saved).expect("deleted usage snapshot should decode");
|
|
assert!(!saved.buckets_usage.contains_key("bucket-a"));
|
|
assert!(!saved.bucket_sizes.contains_key("bucket-a"));
|
|
assert_eq!(store.put_counts.lock().await.get(&key), Some(&1));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_updates_backup_with_new_bucket() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path);
|
|
let deleted = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
|
scanner_epoch: Some(8),
|
|
scanner_cycle: Some(1),
|
|
..complete_usage_with_bucket_count(None, 0)
|
|
};
|
|
store.objects.lock().await.insert(
|
|
backup_key.clone(),
|
|
serde_json::to_vec(&deleted).expect("deleted backup snapshot should encode"),
|
|
);
|
|
store.revisions.lock().await.insert(backup_key.clone(), 1);
|
|
|
|
let (sender, receiver) = mpsc::channel(11);
|
|
for cycle in 2_u64..=12 {
|
|
let mut incoming = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20 + cycle)),
|
|
scanner_epoch: Some(8),
|
|
scanner_cycle: Some(cycle),
|
|
..Default::default()
|
|
};
|
|
incoming.buckets_usage.insert(
|
|
"bucket-a".to_string(),
|
|
rustfs_data_usage::BucketUsageInfo {
|
|
objects_count: 2,
|
|
size: 84,
|
|
..Default::default()
|
|
},
|
|
);
|
|
incoming.bucket_sizes.insert("bucket-a".to_string(), 84);
|
|
incoming.buckets_count = 1;
|
|
incoming.calculate_totals();
|
|
mark_usage_snapshot_complete(&mut incoming);
|
|
sender.send(incoming).await.expect("usage snapshot should enqueue");
|
|
}
|
|
drop(sender);
|
|
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await,
|
|
DataUsagePersistOutcome::Saved
|
|
);
|
|
|
|
let saved = store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.get(&backup_key)
|
|
.cloned()
|
|
.expect("deleted backup snapshot should remain");
|
|
let saved = serde_json::from_slice::<DataUsageInfo>(&saved).expect("backup snapshot should decode");
|
|
assert!(saved.buckets_usage.contains_key("bucket-a"));
|
|
assert!(saved.bucket_sizes.contains_key("bucket-a"));
|
|
assert_eq!(saved.scanner_cycle, Some(10));
|
|
assert_eq!(store.put_counts.lock().await.get(&backup_key), Some(&1));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_repairs_backup_after_primary_only_commit() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path);
|
|
let durable = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
|
|
scanner_epoch: Some(8),
|
|
scanner_cycle: Some(10),
|
|
..complete_usage_with_bucket_count(None, 0)
|
|
};
|
|
let encoded = serde_json::to_vec(&durable).expect("usage snapshot should encode");
|
|
store.objects.lock().await.insert(main_key.clone(), encoded.clone());
|
|
store.revisions.lock().await.insert(main_key.clone(), 1);
|
|
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
sender.send(durable).await.expect("usage snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await,
|
|
DataUsagePersistOutcome::AlreadyDurable
|
|
);
|
|
assert_eq!(store.objects.lock().await.get(&backup_key), Some(&encoded));
|
|
assert_eq!(store.put_counts.lock().await.get(&main_key), None);
|
|
assert_eq!(store.put_counts.lock().await.get(&backup_key), Some(&1));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_copies_concurrent_bucket_removal_to_backup() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path);
|
|
|
|
let mut incoming = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
|
|
scanner_epoch: Some(8),
|
|
scanner_cycle: Some(10),
|
|
..Default::default()
|
|
};
|
|
incoming.buckets_usage.insert(
|
|
"bucket-a".to_string(),
|
|
rustfs_data_usage::BucketUsageInfo {
|
|
objects_count: 2,
|
|
size: 84,
|
|
..Default::default()
|
|
},
|
|
);
|
|
incoming.bucket_sizes.insert("bucket-a".to_string(), 84);
|
|
incoming.buckets_count = 1;
|
|
incoming.calculate_totals();
|
|
mark_usage_snapshot_complete(&mut incoming);
|
|
|
|
let mut deleted = incoming.clone();
|
|
deleted.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(31));
|
|
deleted.buckets_usage.clear();
|
|
deleted.bucket_sizes.clear();
|
|
deleted.buckets_count = 0;
|
|
deleted.calculate_totals();
|
|
mark_usage_snapshot_complete(&mut deleted);
|
|
store.replace_after_successful_puts.lock().await.insert(
|
|
main_key.clone(),
|
|
(1, serde_json::to_vec(&deleted).expect("deleted primary snapshot should encode")),
|
|
);
|
|
store.objects.lock().await.insert(
|
|
backup_key.clone(),
|
|
serde_json::to_vec(&incoming).expect("existing backup snapshot should encode"),
|
|
);
|
|
store.revisions.lock().await.insert(backup_key.clone(), 1);
|
|
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
sender.send(incoming).await.expect("usage snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await,
|
|
DataUsagePersistOutcome::Saved
|
|
);
|
|
|
|
for key in [main_key, backup_key] {
|
|
let saved = store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.get(&key)
|
|
.cloned()
|
|
.expect("usage snapshot should remain");
|
|
let saved = serde_json::from_slice::<DataUsageInfo>(&saved).expect("usage snapshot should decode");
|
|
assert!(!saved.buckets_usage.contains_key("bucket-a"));
|
|
assert!(!saved.bucket_sizes.contains_key("bucket-a"));
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_retries_after_stale_interleaving_writer() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
let ctx = CancellationToken::new();
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let initial = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 3);
|
|
let stale_winner = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 3);
|
|
let current = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), 3);
|
|
store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.insert(key.clone(), serde_json::to_vec(&initial).expect("initial usage snapshot should encode"));
|
|
store.revisions.lock().await.insert(key.clone(), 1);
|
|
store.interleaving_puts.lock().await.insert(
|
|
key.clone(),
|
|
(1, serde_json::to_vec(&stale_winner).expect("stale usage snapshot should encode")),
|
|
);
|
|
|
|
sender
|
|
.send(current.clone())
|
|
.await
|
|
.expect("current usage snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await;
|
|
|
|
let objects = store.objects.lock().await;
|
|
let saved = objects
|
|
.get(&key)
|
|
.expect("current usage snapshot should replace the stale conflict winner");
|
|
let saved = serde_json::from_slice::<DataUsageInfo>(saved).expect("saved usage snapshot should decode");
|
|
assert_eq!(saved.buckets_count, 3);
|
|
assert_eq!(saved.last_update, current.last_update);
|
|
assert_eq!(outcome, DataUsagePersistOutcome::Saved);
|
|
drop(objects);
|
|
assert_eq!(store.put_counts.lock().await.get(&key), Some(&2));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_rejects_untimestamped_complete_snapshot() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let (sender, receiver) = mpsc::channel(2);
|
|
let ctx = CancellationToken::new();
|
|
|
|
let timestamped = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
|
|
let untimestamped = complete_usage_with_bucket_count(None, 1);
|
|
|
|
sender
|
|
.send(timestamped)
|
|
.await
|
|
.expect("timestamped usage snapshot should enqueue");
|
|
sender
|
|
.send(untimestamped)
|
|
.await
|
|
.expect("untimestamped usage snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await;
|
|
|
|
let objects = store.objects.lock().await;
|
|
let saved = objects
|
|
.get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()))
|
|
.expect("data usage config should be saved");
|
|
let saved = serde_json::from_slice::<DataUsageInfo>(saved).expect("saved usage snapshot should decode");
|
|
|
|
assert_eq!(saved.buckets_count, 2);
|
|
assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)));
|
|
assert_eq!(outcome, DataUsagePersistOutcome::Failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_recognizes_already_durable_snapshot() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
let ctx = CancellationToken::new();
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let snapshot = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
|
scanner_cycle: Some(12),
|
|
..complete_usage_with_bucket_count(None, 2)
|
|
};
|
|
store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.insert(key.clone(), serde_json::to_vec(&snapshot).expect("durable usage snapshot should encode"));
|
|
store.revisions.lock().await.insert(key.clone(), 1);
|
|
|
|
sender
|
|
.send(snapshot)
|
|
.await
|
|
.expect("ambiguous committed snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await;
|
|
|
|
assert_eq!(outcome, DataUsagePersistOutcome::AlreadyDurable);
|
|
assert_eq!(store.put_counts.lock().await.get(&key), None);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_advances_past_changed_same_epoch_cycle() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let durable = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
|
scanner_epoch: Some(8),
|
|
scanner_cycle: Some(12),
|
|
..complete_usage_with_bucket_count(None, 2)
|
|
};
|
|
store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.insert(key.clone(), serde_json::to_vec(&durable).expect("durable usage snapshot should encode"));
|
|
store.revisions.lock().await.insert(key.clone(), 1);
|
|
|
|
sender
|
|
.send(DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
|
|
scanner_epoch: Some(8),
|
|
scanner_cycle: Some(12),
|
|
..complete_usage_with_bucket_count(None, 3)
|
|
})
|
|
.await
|
|
.expect("changed retry snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
let outcome =
|
|
store_data_usage_in_backend_with_outcome_for_epoch(CancellationToken::new(), store.clone(), receiver, Some(8)).await;
|
|
|
|
assert_eq!(outcome, DataUsagePersistOutcome::PriorCycleDurable);
|
|
assert_eq!(store.put_counts.lock().await.get(&key), None);
|
|
let saved = store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.get(&key)
|
|
.cloned()
|
|
.expect("first snapshot should remain durable");
|
|
assert_eq!(
|
|
serde_json::from_slice::<DataUsageInfo>(&saved)
|
|
.expect("durable usage snapshot should decode")
|
|
.buckets_count,
|
|
2
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_orders_scanner_cycles_before_wall_clock() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let existing = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)),
|
|
scanner_cycle: Some(12),
|
|
..complete_usage_with_bucket_count(None, 2)
|
|
};
|
|
store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode"));
|
|
store.revisions.lock().await.insert(key.clone(), 1);
|
|
|
|
let (older_sender, older_receiver) = mpsc::channel(1);
|
|
let older = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
|
|
scanner_cycle: Some(11),
|
|
..complete_usage_with_bucket_count(None, 1)
|
|
};
|
|
older_sender.send(older).await.expect("older-cycle snapshot should enqueue");
|
|
drop(older_sender);
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), older_receiver).await,
|
|
DataUsagePersistOutcome::Current
|
|
);
|
|
|
|
let (newer_sender, newer_receiver) = mpsc::channel(1);
|
|
let newer = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
|
|
scanner_cycle: Some(13),
|
|
..complete_usage_with_bucket_count(None, 3)
|
|
};
|
|
newer_sender
|
|
.send(newer.clone())
|
|
.await
|
|
.expect("newer-cycle snapshot should enqueue");
|
|
drop(newer_sender);
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), newer_receiver).await,
|
|
DataUsagePersistOutcome::Saved
|
|
);
|
|
|
|
let saved = store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.get(&key)
|
|
.cloned()
|
|
.expect("newer scanner cycle should be persisted");
|
|
assert_eq!(
|
|
serde_json::from_slice::<DataUsageInfo>(&saved)
|
|
.expect("persisted usage snapshot should decode")
|
|
.scanner_cycle,
|
|
Some(13)
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_orders_leader_epochs_before_cycles() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let existing = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)),
|
|
scanner_epoch: Some(8),
|
|
scanner_cycle: Some(12),
|
|
..complete_usage_with_bucket_count(None, 2)
|
|
};
|
|
store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode"));
|
|
store.revisions.lock().await.insert(key.clone(), 1);
|
|
|
|
let (older_sender, older_receiver) = mpsc::channel(1);
|
|
older_sender
|
|
.send(DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
|
|
scanner_epoch: Some(7),
|
|
scanner_cycle: Some(99),
|
|
..complete_usage_with_bucket_count(None, 1)
|
|
})
|
|
.await
|
|
.expect("old-epoch snapshot should enqueue");
|
|
drop(older_sender);
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), older_receiver).await,
|
|
DataUsagePersistOutcome::Current
|
|
);
|
|
|
|
let (newer_sender, newer_receiver) = mpsc::channel(1);
|
|
newer_sender
|
|
.send(DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
|
|
scanner_epoch: None,
|
|
scanner_cycle: Some(1),
|
|
..complete_usage_with_bucket_count(None, 3)
|
|
})
|
|
.await
|
|
.expect("replacement-epoch snapshot should enqueue");
|
|
drop(newer_sender);
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome_for_epoch(CancellationToken::new(), store.clone(), newer_receiver, Some(9),)
|
|
.await,
|
|
DataUsagePersistOutcome::Saved
|
|
);
|
|
|
|
let saved = store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.get(&key)
|
|
.cloned()
|
|
.expect("replacement leader snapshot should persist");
|
|
let saved = serde_json::from_slice::<DataUsageInfo>(&saved).expect("persisted usage snapshot should decode");
|
|
assert_eq!(saved.scanner_epoch, Some(9));
|
|
assert_eq!(saved.scanner_cycle, Some(1));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_keeps_first_same_cycle_snapshot() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let existing = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
|
|
scanner_cycle: Some(12),
|
|
..complete_usage_with_bucket_count(None, 2)
|
|
};
|
|
store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode"));
|
|
store.revisions.lock().await.insert(key.clone(), 1);
|
|
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
sender
|
|
.send(DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
|
|
scanner_cycle: Some(12),
|
|
..complete_usage_with_bucket_count(None, 3)
|
|
})
|
|
.await
|
|
.expect("conflicting same-cycle snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await,
|
|
DataUsagePersistOutcome::Current
|
|
);
|
|
let saved = store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.get(&key)
|
|
.cloned()
|
|
.expect("first same-cycle snapshot should remain persisted");
|
|
assert_eq!(
|
|
serde_json::from_slice::<DataUsageInfo>(&saved)
|
|
.expect("persisted usage snapshot should decode")
|
|
.buckets_count,
|
|
2
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_rejects_incomplete_snapshot() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let (sender, receiver) = mpsc::channel(2);
|
|
let complete_update = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10);
|
|
|
|
sender
|
|
.send(complete_usage_with_bucket_count(Some(complete_update), 1))
|
|
.await
|
|
.expect("complete usage snapshot should enqueue");
|
|
sender
|
|
.send(DataUsageInfo {
|
|
last_update: Some(complete_update + Duration::from_secs(1)),
|
|
buckets_count: 1,
|
|
..Default::default()
|
|
})
|
|
.await
|
|
.expect("incomplete usage snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
let outcome = store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await;
|
|
|
|
let objects = store.objects.lock().await;
|
|
let saved = objects
|
|
.get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()))
|
|
.expect("complete data usage snapshot should remain saved");
|
|
let saved = serde_json::from_slice::<DataUsageInfo>(saved).expect("saved usage snapshot should decode");
|
|
assert_eq!(saved.last_update, Some(complete_update));
|
|
assert!(saved.is_complete_bucket_usage_snapshot());
|
|
assert_eq!(outcome, DataUsagePersistOutcome::Failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_preserves_superseded_status() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let authoritative_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let authoritative = DataUsageInfo {
|
|
scanner_epoch: Some(7),
|
|
scanner_cycle: Some(10),
|
|
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 1)
|
|
};
|
|
let authoritative_bytes = serde_json::to_vec(&authoritative).expect("authoritative snapshot should encode");
|
|
store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.insert(authoritative_key.clone(), authoritative_bytes.clone());
|
|
store.revisions.lock().await.insert(authoritative_key.clone(), 1);
|
|
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
sender
|
|
.send(DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
|
|
scanner_epoch: Some(7),
|
|
scanner_cycle: Some(11),
|
|
usage_snapshot_converged: Some(false),
|
|
..complete_usage_with_bucket_count(None, 1)
|
|
})
|
|
.await
|
|
.expect("superseded usage snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
let outcome = store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await;
|
|
let saved = store
|
|
.objects
|
|
.lock()
|
|
.await
|
|
.get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()))
|
|
.cloned()
|
|
.expect("superseded usage snapshot should persist");
|
|
let saved = serde_json::from_slice::<DataUsageInfo>(&saved).expect("persisted usage snapshot should decode");
|
|
|
|
assert_eq!(outcome, DataUsagePersistOutcome::Saved);
|
|
assert!(saved.is_complete_bucket_usage_snapshot());
|
|
assert_eq!(saved.usage_snapshot_converged, Some(false));
|
|
assert_eq!(saved.usage_snapshot_authoritative_baseline, Some(authoritative.snapshot_identity()));
|
|
assert_eq!(
|
|
store.objects.lock().await.get(&authoritative_key),
|
|
Some(&authoritative_bytes),
|
|
"an observation must never lower the quota-authoritative snapshot"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_removes_observed_after_authoritative_save() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let authoritative_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let authoritative = DataUsageInfo {
|
|
scanner_epoch: Some(7),
|
|
scanner_cycle: Some(10),
|
|
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 1)
|
|
};
|
|
store.objects.lock().await.insert(
|
|
authoritative_key.clone(),
|
|
serde_json::to_vec(&authoritative).expect("authoritative snapshot should encode"),
|
|
);
|
|
store.revisions.lock().await.insert(authoritative_key, 1);
|
|
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
sender
|
|
.send(DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
|
|
scanner_epoch: Some(7),
|
|
scanner_cycle: Some(11),
|
|
usage_snapshot_converged: Some(false),
|
|
..complete_usage_with_bucket_count(None, 1)
|
|
})
|
|
.await
|
|
.expect("superseded usage snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await,
|
|
DataUsagePersistOutcome::Saved
|
|
);
|
|
let observed_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str());
|
|
assert!(store.objects.lock().await.contains_key(&observed_key));
|
|
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
sender
|
|
.send(DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(2)),
|
|
scanner_epoch: Some(7),
|
|
scanner_cycle: Some(12),
|
|
usage_snapshot_converged: Some(true),
|
|
..complete_usage_with_bucket_count(None, 1)
|
|
})
|
|
.await
|
|
.expect("authoritative usage snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await,
|
|
DataUsagePersistOutcome::Saved
|
|
);
|
|
assert!(
|
|
!store.objects.lock().await.contains_key(&observed_key),
|
|
"an authoritative snapshot should retire stale observations"
|
|
);
|
|
|
|
let next_authoritative = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(3)),
|
|
scanner_epoch: Some(7),
|
|
scanner_cycle: Some(13),
|
|
usage_snapshot_converged: Some(true),
|
|
..complete_usage_with_bucket_count(None, 1)
|
|
};
|
|
let newer_observed = DataUsageInfo {
|
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(4)),
|
|
scanner_epoch: Some(7),
|
|
scanner_cycle: Some(14),
|
|
usage_snapshot_converged: Some(false),
|
|
usage_snapshot_authoritative_baseline: Some(next_authoritative.snapshot_identity()),
|
|
..complete_usage_with_bucket_count(None, 1)
|
|
};
|
|
store.objects.lock().await.insert(
|
|
observed_key.clone(),
|
|
serde_json::to_vec(&newer_observed).expect("newer observed snapshot should encode"),
|
|
);
|
|
store.revisions.lock().await.insert(observed_key.clone(), 3);
|
|
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
sender
|
|
.send(next_authoritative)
|
|
.await
|
|
.expect("next authoritative usage snapshot should enqueue");
|
|
drop(sender);
|
|
|
|
assert_eq!(
|
|
store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await,
|
|
DataUsagePersistOutcome::Saved
|
|
);
|
|
assert!(
|
|
store.objects.lock().await.contains_key(&observed_key),
|
|
"a newer observation must survive stale authoritative cleanup"
|
|
);
|
|
}
|
|
|
|
fn mark_usage_snapshot_complete(info: &mut DataUsageInfo) {
|
|
info.usage_snapshot_complete = true;
|
|
}
|
|
|
|
fn complete_usage_with_bucket_count(last_update: Option<std::time::SystemTime>, buckets_count: u64) -> DataUsageInfo {
|
|
let mut info = DataUsageInfo {
|
|
last_update,
|
|
buckets_count,
|
|
usage_snapshot_complete: true,
|
|
..Default::default()
|
|
};
|
|
for index in 0..buckets_count {
|
|
let bucket = format!("bucket-{index}");
|
|
info.buckets_usage.insert(bucket.clone(), Default::default());
|
|
info.bucket_sizes.insert(bucket, 0);
|
|
}
|
|
info
|
|
}
|
|
|
|
fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo {
|
|
complete_usage_with_bucket_count(last_update, 0)
|
|
}
|
|
|
|
#[test]
|
|
fn test_stale_data_usage_update_reason_allows_newer_incoming() {
|
|
let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
|
|
let incoming = usage_with_last_update(Some(now));
|
|
let existing = usage_with_last_update(Some(now - Duration::from_secs(60)));
|
|
assert_eq!(stale_data_usage_update_reason(&incoming, &existing, now), None);
|
|
}
|
|
|
|
#[test]
|
|
fn test_stale_data_usage_update_reason_skips_older_or_equal_incoming() {
|
|
let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
|
|
let existing = usage_with_last_update(Some(now - Duration::from_secs(60)));
|
|
|
|
let older = usage_with_last_update(Some(now - Duration::from_secs(120)));
|
|
assert_eq!(stale_data_usage_update_reason(&older, &existing, now), Some("older_or_equal_last_update"));
|
|
|
|
let equal = usage_with_last_update(existing.last_update);
|
|
assert_eq!(stale_data_usage_update_reason(&equal, &existing, now), Some("older_or_equal_last_update"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_stale_data_usage_update_reason_allows_save_when_existing_is_future_dated() {
|
|
// Existing snapshot timestamp beyond the clock tolerance is untrustworthy
|
|
// (clock step-back / slower-clock leader): the save must be allowed even
|
|
// though incoming <= existing, otherwise usage stats freeze forever.
|
|
let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
|
|
let existing =
|
|
usage_with_last_update(Some(now + rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE + Duration::from_secs(1)));
|
|
let incoming = usage_with_last_update(Some(now));
|
|
assert_eq!(stale_data_usage_update_reason(&incoming, &existing, now), None);
|
|
}
|
|
|
|
#[test]
|
|
fn test_stale_data_usage_update_reason_skips_at_exact_tolerance_boundary() {
|
|
// Exactly at now + tolerance is still within the trusted window.
|
|
let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
|
|
let existing = usage_with_last_update(Some(now + rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE));
|
|
let incoming = usage_with_last_update(Some(now));
|
|
assert_eq!(
|
|
stale_data_usage_update_reason(&incoming, &existing, now),
|
|
Some("older_or_equal_last_update")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_stale_data_usage_update_reason_preserves_none_handling() {
|
|
let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
|
|
|
|
let incoming_none = usage_with_last_update(None);
|
|
let existing_some = usage_with_last_update(Some(now - Duration::from_secs(60)));
|
|
assert_eq!(
|
|
stale_data_usage_update_reason(&incoming_none, &existing_some, now),
|
|
Some("missing_incoming_last_update")
|
|
);
|
|
|
|
let incoming_some = usage_with_last_update(Some(now));
|
|
let existing_none = usage_with_last_update(None);
|
|
assert_eq!(stale_data_usage_update_reason(&incoming_some, &existing_none, now), None);
|
|
|
|
let both_none = usage_with_last_update(None);
|
|
assert_eq!(stale_data_usage_update_reason(&both_none, &usage_with_last_update(None), now), None);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_keeps_backup_when_primary_save_fails() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let (sender, receiver) = mpsc::channel(11);
|
|
let ctx = CancellationToken::new();
|
|
|
|
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path);
|
|
let old_backup = b"old-backup".to_vec();
|
|
|
|
store.objects.lock().await.insert(backup_key.clone(), old_backup.clone());
|
|
store.fail_put_number.lock().await.insert(main_key.clone(), 11);
|
|
|
|
for idx in 1_u64..=11 {
|
|
sender
|
|
.send(complete_usage_with_bucket_count(
|
|
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(idx)),
|
|
idx,
|
|
))
|
|
.await
|
|
.expect("usage snapshot should enqueue");
|
|
}
|
|
drop(sender);
|
|
|
|
let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await;
|
|
|
|
let objects = store.objects.lock().await;
|
|
assert_eq!(
|
|
objects.get(&backup_key),
|
|
Some(&old_backup),
|
|
"primary save failure must not overwrite the previous backup"
|
|
);
|
|
let saved = objects
|
|
.get(&main_key)
|
|
.expect("last successful primary usage snapshot should remain saved");
|
|
let saved = serde_json::from_slice::<DataUsageInfo>(saved).expect("saved usage snapshot should decode");
|
|
assert_eq!(saved.buckets_count, 10);
|
|
assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)));
|
|
assert_eq!(outcome, DataUsagePersistOutcome::Failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_data_usage_in_backend_reports_missing_snapshot() {
|
|
let store = Arc::new(MemoryConfigStore::default());
|
|
let (sender, receiver) = mpsc::channel(1);
|
|
let ctx = CancellationToken::new();
|
|
drop(sender);
|
|
|
|
let outcome = store_data_usage_in_backend_with_outcome(ctx, store, receiver).await;
|
|
|
|
assert_eq!(outcome, DataUsagePersistOutcome::NoUpdate);
|
|
}
|
|
|
|
#[test]
|
|
fn test_scanner_cycle_completion_prioritizes_persist_failure() {
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(
|
|
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
|
DataUsagePersistOutcome::NoUpdate,
|
|
false,
|
|
false,
|
|
),
|
|
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(
|
|
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
|
|
DataUsagePersistOutcome::Saved,
|
|
false,
|
|
false,
|
|
),
|
|
ScannerCycleOutcome::Failed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(
|
|
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
|
|
DataUsagePersistOutcome::NoUpdate,
|
|
true,
|
|
false,
|
|
),
|
|
ScannerCycleOutcome::Failed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(
|
|
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
|
|
DataUsagePersistOutcome::Failed,
|
|
false,
|
|
false,
|
|
),
|
|
ScannerCycleOutcome::Failed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::NoUpdate, false, false),
|
|
ScannerCycleOutcome::Failed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Failed, true, true),
|
|
ScannerCycleOutcome::Failed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::NoUpdate, true, true),
|
|
ScannerCycleOutcome::Failed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Saved, true, false),
|
|
ScannerCycleOutcome::Partial
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Saved, true, true),
|
|
ScannerCycleOutcome::Failed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Saved, true, false),
|
|
ScannerCycleOutcome::Completed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::AlreadyDurable, true, false,),
|
|
ScannerCycleOutcome::Completed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::PriorCycleDurable, true, false,),
|
|
ScannerCycleOutcome::Completed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current, false, false),
|
|
ScannerCycleOutcome::Completed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current, true, false),
|
|
ScannerCycleOutcome::Failed
|
|
);
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::NoUpdate, false, false),
|
|
ScannerCycleOutcome::Failed
|
|
);
|
|
for persist_outcome in [
|
|
DataUsagePersistOutcome::NoUpdate,
|
|
DataUsagePersistOutcome::Current,
|
|
DataUsagePersistOutcome::Saved,
|
|
] {
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Superseded, persist_outcome, true, false),
|
|
ScannerCycleOutcome::Superseded
|
|
);
|
|
}
|
|
assert_eq!(
|
|
scanner_cycle_completion_outcome(ScannerCycleStatus::Superseded, DataUsagePersistOutcome::Saved, true, true),
|
|
ScannerCycleOutcome::Failed
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
|
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
|
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
|
|
|
let remote_acknowledgement = ScannerDirtyUsageAcknowledgement {
|
|
host: "node-2".to_string(),
|
|
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
|
generation: 11,
|
|
};
|
|
let unsaved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot.clone()))
|
|
.with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]);
|
|
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(unsaved, DataUsagePersistOutcome::NoUpdate);
|
|
assert_eq!(outcome, ScannerCycleOutcome::Failed);
|
|
assert!(acknowledgements.is_empty());
|
|
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
|
|
|
let saved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot))
|
|
.with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]);
|
|
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(saved, DataUsagePersistOutcome::Saved);
|
|
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
|
assert_eq!(acknowledgements, vec![remote_acknowledgement]);
|
|
assert!(!crate::scanner_io::dirty_usage_buckets_pending());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
|
|
let pending = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(true))).await;
|
|
assert_eq!(
|
|
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, pending),
|
|
ScannerCycleOutcome::CompletedWithPendingMaintenance
|
|
);
|
|
|
|
let cleared = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(false))).await;
|
|
assert_eq!(
|
|
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, cleared),
|
|
ScannerCycleOutcome::Completed
|
|
);
|
|
|
|
let failed = remote_dirty_usage_acknowledgement_pending(
|
|
7,
|
|
1,
|
|
std::future::ready(Err::<bool, _>(std::io::Error::other("injected acknowledgement failure"))),
|
|
)
|
|
.await;
|
|
assert_eq!(
|
|
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, failed),
|
|
ScannerCycleOutcome::CompletedWithPendingMaintenance
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() {
|
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
|
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
|
|
|
let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
|
|
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::AlreadyDurable);
|
|
|
|
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
|
assert!(acknowledgements.is_empty());
|
|
assert!(!crate::scanner_io::dirty_usage_buckets_pending());
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() {
|
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
|
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
|
|
|
let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
|
|
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::PriorCycleDurable);
|
|
|
|
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
|
assert!(acknowledgements.is_empty());
|
|
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() {
|
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
|
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
|
|
|
let superseded = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Superseded, Some(dirty_snapshot));
|
|
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(superseded, DataUsagePersistOutcome::Saved);
|
|
|
|
assert_eq!(outcome, ScannerCycleOutcome::Superseded);
|
|
assert!(acknowledgements.is_empty());
|
|
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
|
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn data_usage_persist_wait_covers_cache_retries_and_backup() {
|
|
with_var(rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, Some("7"), || {
|
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
|
assert_eq!(data_usage_persist_timeout(), Duration::from_millis(31_350));
|
|
});
|
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn data_usage_persist_wait_aborts_when_scanner_is_cancelled() {
|
|
let ctx = CancellationToken::new();
|
|
let mut task = AbortOnDropHandle::new(tokio::spawn(async {
|
|
std::future::pending::<()>().await;
|
|
DataUsagePersistOutcome::Saved
|
|
}));
|
|
ctx.cancel();
|
|
|
|
let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(60)).await;
|
|
|
|
assert!(matches!(result, DataUsagePersistTaskResult::Cancelled));
|
|
assert!(task.is_finished());
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn data_usage_persist_wait_aborts_after_timeout() {
|
|
let ctx = CancellationToken::new();
|
|
let mut task = AbortOnDropHandle::new(tokio::spawn(async {
|
|
std::future::pending::<()>().await;
|
|
DataUsagePersistOutcome::Saved
|
|
}));
|
|
|
|
let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(30)).await;
|
|
|
|
assert!(matches!(result, DataUsagePersistTaskResult::TimedOut));
|
|
assert!(task.is_finished());
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn maintenance_feature_inspection_preserves_base_cycle_after_timeout() {
|
|
let ctx = CancellationToken::new();
|
|
|
|
let result = wait_for_maintenance_feature_inspection(
|
|
&ctx,
|
|
std::future::pending::<ScannerMaintenanceFeatures>(),
|
|
Duration::from_secs(30),
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(result, MaintenanceInspectionAttempt::TimedOut);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
#[serial]
|
|
async fn stable_maintenance_detection_preserves_base_cycle_after_timeout() {
|
|
let ctx = CancellationToken::new();
|
|
|
|
let (features, generation) = detect_stable_scanner_maintenance_features_with(
|
|
&ctx,
|
|
std::future::pending::<ScannerMaintenanceFeatures>,
|
|
Duration::from_secs(30),
|
|
)
|
|
.await
|
|
.expect("timeout should preserve the scanner rather than stop it");
|
|
|
|
assert!(features.inspection_failed);
|
|
assert_eq!(generation, scanner_maintenance_generation());
|
|
assert!(!scanner_clean_idle_backoff_enabled(
|
|
true,
|
|
true,
|
|
features,
|
|
&ScannerRuntimeConfig::default()
|
|
));
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn failed_maintenance_inspection_uses_bounded_retry_backoff() {
|
|
let failed = ScannerMaintenanceFeatures {
|
|
inspection_failed: true,
|
|
..Default::default()
|
|
};
|
|
let mut retry = ScannerMaintenanceInspectionRetry::from_features(failed, Instant::now());
|
|
|
|
assert_eq!(retry.retry_interval(), Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL));
|
|
assert!(!retry.retry_due(failed, ScannerCycleWakeReason::Timer, Instant::now()));
|
|
tokio::time::advance(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL).await;
|
|
assert!(retry.retry_due(failed, ScannerCycleWakeReason::Timer, Instant::now()));
|
|
assert!(!retry.retry_due(failed, ScannerCycleWakeReason::DirtyUsage, Instant::now()));
|
|
|
|
retry.record_inspection(failed, Instant::now());
|
|
assert_eq!(
|
|
retry.retry_interval(),
|
|
Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL.saturating_mul(2))
|
|
);
|
|
for _ in 0..8 {
|
|
retry.record_inspection(failed, Instant::now());
|
|
}
|
|
assert_eq!(retry.retry_interval(), Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL));
|
|
|
|
retry.record_inspection(ScannerMaintenanceFeatures::default(), Instant::now());
|
|
assert_eq!(retry, ScannerMaintenanceInspectionRetry::default());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn maintenance_feature_inspection_stops_on_cancellation() {
|
|
let ctx = CancellationToken::new();
|
|
ctx.cancel();
|
|
|
|
let result = wait_for_maintenance_feature_inspection(
|
|
&ctx,
|
|
std::future::pending::<ScannerMaintenanceFeatures>(),
|
|
Duration::from_secs(30),
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(result, MaintenanceInspectionAttempt::Cancelled);
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_cycle_interval_prefers_explicit_cycle_override() {
|
|
with_var(ENV_SCANNER_SPEED, Some("slowest"), || {
|
|
with_var(ENV_SCANNER_CYCLE, Some("42"), || {
|
|
assert_eq!(cycle_interval(), Duration::from_secs(42));
|
|
});
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() {
|
|
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
|
|
|
with_var(ENV_SCANNER_CYCLE, Some("42"), || {
|
|
assert_eq!(cycle_interval(), Duration::from_secs(42));
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() {
|
|
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
|
|
|
|
with_unset_scanner_timing_env(|| {
|
|
assert_eq!(cycle_interval(), Duration::from_secs(30 * 60));
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() {
|
|
let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest);
|
|
|
|
with_var_unset(ENV_SCANNER_CYCLE, || {
|
|
with_var_unset("MINIO_SCANNER_CYCLE", || {
|
|
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
|
|
with_var_unset(ENV_SCANNER_START_DELAY_SECS_DEPRECATED, || {
|
|
with_var(ENV_SCANNER_SPEED, Some("fastest"), || {
|
|
assert_eq!(cycle_interval(), Duration::from_secs(1));
|
|
});
|
|
});
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_cycle_interval_uses_default_cycle_override_when_unconfigured() {
|
|
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
|
|
|
with_unset_scanner_timing_env(|| {
|
|
assert_eq!(cycle_interval(), Duration::from_secs(TEST_DEFAULT_SCANNER_CYCLE_SECS));
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn test_single_disk_default_speed_uses_regular_scanner_default() {
|
|
assert_eq!(single_disk_default_speed(), ScannerSpeed::Default);
|
|
}
|
|
|
|
#[test]
|
|
fn test_maintenance_feature_inspection_is_bounded_and_conservative() {
|
|
assert_eq!(maintenance_inspection_decision(1, 1, 1), MaintenanceInspectionDecision::Accept);
|
|
assert_eq!(maintenance_inspection_decision(1, 2, 1), MaintenanceInspectionDecision::Retry);
|
|
assert_eq!(
|
|
maintenance_inspection_decision(1, 2, MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS),
|
|
MaintenanceInspectionDecision::PreserveBaseCycle
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn clean_idle_backoff_grows_to_cap() {
|
|
let base_interval = Duration::from_secs(60);
|
|
let max_interval = CLEAN_IDLE_MAX_INTERVAL;
|
|
let mut backoff = ScannerCleanIdleBackoff::default();
|
|
|
|
assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(60));
|
|
for expected_secs in [
|
|
120, 240, 480, 960, 1_920, 3_840, 7_680, 15_360, 30_720, 61_440, 86_400, 86_400,
|
|
] {
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
assert_eq!(
|
|
backoff.effective_interval(base_interval, max_interval, true),
|
|
Duration::from_secs(expected_secs)
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn superseded_retry_backoff_grows_caps_and_resets_after_convergence() {
|
|
let mut backoff = ScannerRetryBackoff::default();
|
|
assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None);
|
|
|
|
for expected in [5, 10, 20, 40, 80, 160, 320] {
|
|
backoff.record_retryable_cycle(true);
|
|
assert_eq!(
|
|
backoff.retry_interval(Duration::from_secs(24 * 60 * 60)),
|
|
Some(Duration::from_secs(expected))
|
|
);
|
|
}
|
|
for _ in 0..20 {
|
|
backoff.record_retryable_cycle(true);
|
|
}
|
|
assert_eq!(
|
|
backoff.retry_interval(Duration::from_secs(24 * 60 * 60)),
|
|
Some(Duration::from_secs(24 * 60 * 60))
|
|
);
|
|
|
|
backoff.record_retryable_cycle(false);
|
|
assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None);
|
|
}
|
|
|
|
#[test]
|
|
fn superseded_retry_backoff_respects_a_faster_configured_cycle() {
|
|
let mut backoff = ScannerRetryBackoff::default();
|
|
backoff.record_retryable_cycle(true);
|
|
|
|
// A configured cycle shorter than the base still wins: retrying sooner
|
|
// than the operator's own cadence buys nothing.
|
|
assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(3)));
|
|
backoff.record_retryable_cycle(true);
|
|
assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(6)));
|
|
}
|
|
|
|
#[test]
|
|
fn superseded_retry_backoff_grows_from_the_default_cycle() {
|
|
let mut backoff = ScannerRetryBackoff::default();
|
|
// The first race after a write burst retries in seconds, not a whole
|
|
// cycle, while repeated supersedes still climb toward the cap.
|
|
for expected in [5, 10, 20, 40] {
|
|
backoff.record_retryable_cycle(true);
|
|
assert_eq!(backoff.retry_interval(Duration::from_secs(60)), Some(Duration::from_secs(expected)));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_cycle_wait_plan_drives_growth_resets_and_bitrot_cap() {
|
|
let runtime_config = ScannerRuntimeConfig {
|
|
cycle_interval: Duration::from_secs(60),
|
|
bitrot_cycle: None,
|
|
..Default::default()
|
|
};
|
|
let mut clean_idle_backoff = ScannerCleanIdleBackoff::default();
|
|
|
|
let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity);
|
|
assert_eq!(plan.delay, Duration::from_secs(60));
|
|
|
|
for expected in [120, 240] {
|
|
record_scanner_cycle_result(
|
|
&mut clean_idle_backoff,
|
|
&runtime_config,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity);
|
|
assert_eq!(plan.delay, Duration::from_secs(expected));
|
|
}
|
|
|
|
for (wake_reason, outcome, dirty_work_observed) in [
|
|
(ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Completed, true),
|
|
(ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Partial, false),
|
|
(ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Failed, false),
|
|
(ScannerCycleWakeReason::Timer, ScannerCycleOutcome::CompletedWithPendingMaintenance, false),
|
|
(ScannerCycleWakeReason::DirtyUsage, ScannerCycleOutcome::Completed, false),
|
|
] {
|
|
record_scanner_cycle_result(&mut clean_idle_backoff, &runtime_config, true, wake_reason, outcome, dirty_work_observed);
|
|
let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity);
|
|
assert_eq!(plan.effective_interval, Duration::from_secs(60));
|
|
assert_eq!(plan.delay, Duration::from_secs(60));
|
|
|
|
record_scanner_cycle_result(
|
|
&mut clean_idle_backoff,
|
|
&runtime_config,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
}
|
|
|
|
clean_idle_backoff.reset();
|
|
for _ in 0..32 {
|
|
record_scanner_cycle_result(
|
|
&mut clean_idle_backoff,
|
|
&runtime_config,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
}
|
|
let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, |interval| interval.mul_f64(1.1));
|
|
assert_eq!(plan.effective_interval, CLEAN_IDLE_MAX_INTERVAL);
|
|
assert!(plan.delay < CLEAN_IDLE_MAX_INTERVAL);
|
|
assert_eq!(
|
|
plan.delay,
|
|
CLEAN_IDLE_MAX_INTERVAL.saturating_sub(CLEAN_IDLE_MAX_INTERVAL.mul_f64(1.1) - CLEAN_IDLE_MAX_INTERVAL)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn scanner_cycle_schedule_status_reports_effective_backoff() {
|
|
record_scanner_cycle_schedule(Duration::from_millis(86_400_001), true, 2_048, true, 7);
|
|
|
|
let status = scanner_cycle_schedule_status();
|
|
|
|
assert_eq!(status.effective_interval_seconds, 86_401);
|
|
assert!(status.clean_idle_backoff_enabled);
|
|
assert_eq!(status.clean_idle_backoff_multiplier, 2_048);
|
|
assert!(status.superseded_retry_backoff_enabled);
|
|
assert_eq!(status.superseded_cycles, 7);
|
|
|
|
reset_scanner_cycle_schedule();
|
|
let status = scanner_cycle_schedule_status();
|
|
assert_eq!(status.effective_interval_seconds, 0);
|
|
assert!(!status.clean_idle_backoff_enabled);
|
|
assert_eq!(status.clean_idle_backoff_multiplier, 1);
|
|
assert!(!status.superseded_retry_backoff_enabled);
|
|
assert_eq!(status.superseded_cycles, 0);
|
|
}
|
|
|
|
#[test]
|
|
fn clean_idle_backoff_resets_for_non_idle_work() {
|
|
let base_interval = Duration::from_secs(60);
|
|
let max_interval = CLEAN_IDLE_MAX_INTERVAL;
|
|
let mut backoff = ScannerCleanIdleBackoff::default();
|
|
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(240));
|
|
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::DirtyUsage,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval);
|
|
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Partial,
|
|
false,
|
|
);
|
|
assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval);
|
|
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Failed,
|
|
false,
|
|
);
|
|
assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval);
|
|
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
true,
|
|
);
|
|
assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval);
|
|
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
|
false,
|
|
);
|
|
assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval);
|
|
}
|
|
|
|
#[test]
|
|
fn test_dirty_work_is_observed_across_cycle_waits() {
|
|
assert!(scanner_cycle_observed_dirty_work(true, 7, 7));
|
|
assert!(scanner_cycle_observed_dirty_work(false, 7, 8));
|
|
assert!(!scanner_cycle_observed_dirty_work(false, 7, 7));
|
|
}
|
|
|
|
#[test]
|
|
fn clean_idle_backoff_never_shortens_base_interval() {
|
|
let base_interval = Duration::from_secs(48 * 60 * 60);
|
|
let mut backoff = ScannerCleanIdleBackoff::default();
|
|
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
CLEAN_IDLE_MAX_INTERVAL,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
|
|
assert_eq!(backoff.effective_interval(base_interval, CLEAN_IDLE_MAX_INTERVAL, true), base_interval);
|
|
}
|
|
|
|
#[test]
|
|
fn clean_idle_backoff_resets_while_disabled() {
|
|
let base_interval = Duration::from_secs(60);
|
|
let max_interval = CLEAN_IDLE_MAX_INTERVAL;
|
|
let mut backoff = ScannerCleanIdleBackoff::default();
|
|
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(240));
|
|
|
|
backoff.record_cycle(
|
|
base_interval,
|
|
max_interval,
|
|
false,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
false,
|
|
);
|
|
|
|
assert_eq!(backoff.effective_interval(base_interval, max_interval, false), base_interval);
|
|
assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval);
|
|
}
|
|
|
|
#[test]
|
|
fn clean_idle_backoff_policy_preserves_explicit_and_maintenance_cycles() {
|
|
let no_features = ScannerMaintenanceFeatures::default();
|
|
let default_config = ScannerRuntimeConfig::default();
|
|
assert!(scanner_clean_idle_backoff_enabled(true, true, no_features, &default_config));
|
|
assert!(!scanner_clean_idle_backoff_enabled(false, true, no_features, &default_config));
|
|
assert!(!scanner_clean_idle_backoff_enabled(true, false, no_features, &default_config));
|
|
|
|
for source in [ScannerRuntimeConfigSource::Env, ScannerRuntimeConfigSource::Config] {
|
|
let mut config = default_config.clone();
|
|
config.cycle_interval_source = source;
|
|
assert!(!scanner_clean_idle_backoff_enabled(true, true, no_features, &config));
|
|
}
|
|
|
|
for source in [
|
|
ScannerRuntimeConfigSource::Env,
|
|
ScannerRuntimeConfigSource::Config,
|
|
ScannerRuntimeConfigSource::ScannerCompatConfig,
|
|
] {
|
|
let mut explicit_bitrot_config = default_config.clone();
|
|
explicit_bitrot_config.bitrot_cycle = Some(Duration::from_secs(60 * 60));
|
|
explicit_bitrot_config.bitrot_cycle_source = source;
|
|
assert!(!scanner_clean_idle_backoff_enabled(true, true, no_features, &explicit_bitrot_config));
|
|
|
|
explicit_bitrot_config.bitrot_cycle = None;
|
|
assert!(scanner_clean_idle_backoff_enabled(true, true, no_features, &explicit_bitrot_config));
|
|
}
|
|
|
|
for features in [
|
|
ScannerMaintenanceFeatures {
|
|
lifecycle: true,
|
|
..Default::default()
|
|
},
|
|
ScannerMaintenanceFeatures {
|
|
replication: true,
|
|
..Default::default()
|
|
},
|
|
ScannerMaintenanceFeatures {
|
|
inspection_failed: true,
|
|
..Default::default()
|
|
},
|
|
] {
|
|
assert!(!scanner_clean_idle_backoff_enabled(true, true, features, &default_config));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn clean_idle_backoff_requires_activity_probes() {
|
|
let default_config = ScannerRuntimeConfig::default();
|
|
let no_features = ScannerMaintenanceFeatures::default();
|
|
assert!(scanner_activity_probe_required(true, false, no_features, &default_config));
|
|
assert!(!scanner_activity_probe_required(false, false, no_features, &default_config));
|
|
assert!(!scanner_activity_probe_required(true, true, no_features, &default_config));
|
|
|
|
let mut explicit_cycle = default_config.clone();
|
|
explicit_cycle.cycle_interval_source = ScannerRuntimeConfigSource::Env;
|
|
assert!(!scanner_activity_probe_required(true, false, no_features, &explicit_cycle));
|
|
|
|
let lifecycle = ScannerMaintenanceFeatures {
|
|
lifecycle: true,
|
|
..Default::default()
|
|
};
|
|
assert!(!scanner_activity_probe_required(true, false, lifecycle, &default_config));
|
|
}
|
|
|
|
#[test]
|
|
fn dirty_usage_wakes_are_disabled_for_explicit_cycle_policy() {
|
|
let default_config = ScannerRuntimeConfig::default();
|
|
|
|
let default_observed = ScannerCycleObservedGenerations::for_wait(&default_config, None, 7, 11, 13);
|
|
assert_eq!(default_observed.dirty_usage, Some(7));
|
|
assert_eq!(default_observed.runtime_config, 11);
|
|
assert_eq!(default_observed.maintenance, 13);
|
|
assert!(!default_observed.defer_cluster_activity);
|
|
|
|
let retry_observed = ScannerCycleObservedGenerations::for_wait(&default_config, Some(Duration::from_secs(11)), 7, 11, 13);
|
|
assert_eq!(retry_observed.dirty_usage, None);
|
|
assert!(retry_observed.defer_cluster_activity);
|
|
|
|
for source in [ScannerRuntimeConfigSource::Env, ScannerRuntimeConfigSource::Config] {
|
|
let explicit_cycle = ScannerRuntimeConfig {
|
|
cycle_interval_source: source,
|
|
..default_config.clone()
|
|
};
|
|
let explicit_observed = ScannerCycleObservedGenerations::for_wait(&explicit_cycle, None, 7, 11, 13);
|
|
assert_eq!(explicit_observed.dirty_usage, None);
|
|
assert!(!explicit_observed.defer_cluster_activity);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn clean_idle_cap_preserves_default_bitrot_coverage_window() {
|
|
let config = ScannerRuntimeConfig {
|
|
bitrot_cycle: Some(Duration::from_secs(30 * 24 * 60 * 60)),
|
|
bitrot_cycle_source: ScannerRuntimeConfigSource::Default,
|
|
..Default::default()
|
|
};
|
|
|
|
with_var("RUSTFS_HEAL_OBJECT_SELECT_PROB", Some("1024"), || {
|
|
let max_interval = scanner_clean_idle_max_interval(Duration::from_secs(60), &config);
|
|
assert_eq!(max_interval, Duration::from_millis(2_531_250));
|
|
let positive_jitter = max_interval.mul_f64(1.1);
|
|
let actual_delay = cap_clean_idle_cycle_delay(positive_jitter, max_interval, true);
|
|
assert!(actual_delay < max_interval);
|
|
assert_eq!(actual_delay, max_interval.saturating_sub(positive_jitter - max_interval));
|
|
assert!(actual_delay.saturating_mul(1024) <= config.bitrot_cycle.expect("bitrot cycle should be configured"));
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn clean_idle_cap_allows_policy_max_when_bitrot_is_disabled() {
|
|
let config = ScannerRuntimeConfig {
|
|
bitrot_cycle: None,
|
|
..Default::default()
|
|
};
|
|
|
|
assert_eq!(scanner_clean_idle_max_interval(Duration::from_secs(60), &config), CLEAN_IDLE_MAX_INTERVAL);
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn clean_idle_cap_never_shortens_the_base_cycle() {
|
|
let config = ScannerRuntimeConfig {
|
|
bitrot_cycle: Some(Duration::from_secs(60)),
|
|
bitrot_cycle_source: ScannerRuntimeConfigSource::Default,
|
|
..Default::default()
|
|
};
|
|
|
|
with_var("RUSTFS_HEAL_OBJECT_SELECT_PROB", Some("1024"), || {
|
|
assert_eq!(scanner_clean_idle_max_interval(Duration::from_secs(60), &config), Duration::from_secs(60));
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() {
|
|
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
|
|
|
with_var_unset(ENV_SCANNER_CYCLE, || {
|
|
with_var_unset("MINIO_SCANNER_CYCLE", || {
|
|
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
|
|
with_var_unset(ENV_SCANNER_START_DELAY_SECS_DEPRECATED, || {
|
|
with_var(ENV_SCANNER_SPEED, Some("slowest"), || {
|
|
assert_eq!(cycle_interval(), Duration::from_secs(TEST_DEFAULT_SCANNER_CYCLE_SECS));
|
|
});
|
|
});
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() {
|
|
let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
|
|
|
|
with_var_unset(ENV_SCANNER_CYCLE, || {
|
|
with_var_unset("MINIO_SCANNER_CYCLE", || {
|
|
with_var(ENV_SCANNER_START_DELAY_SECS, Some("120"), || {
|
|
assert_eq!(cycle_interval(), Duration::from_secs(120));
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_cycle_interval_supports_minio_speed_alias() {
|
|
with_var_unset(ENV_SCANNER_SPEED, || {
|
|
with_var_unset(ENV_SCANNER_CYCLE, || {
|
|
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
|
|
with_var("MINIO_SCANNER_SPEED", Some("slowest"), || {
|
|
assert_eq!(cycle_interval(), Duration::from_secs(30 * 60));
|
|
});
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_cycle_interval_supports_minio_cycle_alias() {
|
|
with_var_unset(ENV_SCANNER_CYCLE, || {
|
|
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
|
|
with_var("MINIO_SCANNER_CYCLE", Some("90"), || {
|
|
assert_eq!(cycle_interval(), Duration::from_secs(90));
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn test_randomized_cycle_delay_handles_small_start_delay() {
|
|
// 0 is treated as minimum 1 second before jitter, with lower bound preserved.
|
|
let delay = randomized_cycle_delay_for(Duration::from_secs(0));
|
|
assert!(delay >= Duration::from_secs(1), "expected delay >= 1s");
|
|
assert!(delay < Duration::from_secs(2), "expected delay < 2s");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
|
|
let ctx = CancellationToken::new();
|
|
let dirty_generation = crate::scanner_io::dirty_usage_generation();
|
|
let mut wait = Box::pin(wait_for_next_scanner_cycle(
|
|
&ctx,
|
|
Duration::from_secs(60),
|
|
Some(dirty_generation),
|
|
crate::runtime_config::scanner_runtime_config_generation(),
|
|
crate::scanner_io::scanner_maintenance_generation(),
|
|
|| false,
|
|
));
|
|
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
|
|
|
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
|
let reason = tokio::time::timeout(Duration::from_secs(1), wait)
|
|
.await
|
|
.expect("dirty usage should wake scanner before timer");
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage);
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
let dirty_generation = crate::scanner_io::dirty_usage_generation();
|
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
|
|
|
let ctx = CancellationToken::new();
|
|
let reason = wait_for_next_scanner_cycle(
|
|
&ctx,
|
|
Duration::from_secs(60),
|
|
Some(dirty_generation),
|
|
crate::runtime_config::scanner_runtime_config_generation(),
|
|
crate::scanner_io::scanner_maintenance_generation(),
|
|
|| false,
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage);
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
#[serial]
|
|
async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
|
let dirty_generation = crate::scanner_io::dirty_usage_generation();
|
|
let ctx = CancellationToken::new();
|
|
let wait = wait_for_next_scanner_cycle(
|
|
&ctx,
|
|
Duration::from_secs(60),
|
|
Some(dirty_generation),
|
|
crate::runtime_config::scanner_runtime_config_generation(),
|
|
crate::scanner_io::scanner_maintenance_generation(),
|
|
|| false,
|
|
);
|
|
|
|
let reason = wait.await;
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::Timer);
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
#[serial]
|
|
async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
let ctx = CancellationToken::new();
|
|
let wait = wait_for_next_scanner_cycle(
|
|
&ctx,
|
|
Duration::from_secs(60),
|
|
None,
|
|
crate::runtime_config::scanner_runtime_config_generation(),
|
|
crate::scanner_io::scanner_maintenance_generation(),
|
|
|| false,
|
|
);
|
|
|
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
|
assert_eq!(wait.await, ScannerCycleWakeReason::Timer);
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
|
let dirty_generation = crate::scanner_io::dirty_usage_generation();
|
|
let ctx = CancellationToken::new();
|
|
let mut wait = Box::pin(wait_for_next_scanner_cycle(
|
|
&ctx,
|
|
Duration::from_secs(60),
|
|
Some(dirty_generation),
|
|
crate::runtime_config::scanner_runtime_config_generation(),
|
|
crate::scanner_io::scanner_maintenance_generation(),
|
|
|| false,
|
|
));
|
|
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
|
|
|
|
crate::scanner_io::record_dirty_usage_bucket("photos");
|
|
let reason = tokio::time::timeout(Duration::from_secs(1), wait)
|
|
.await
|
|
.expect("a newer mutation of an already-dirty bucket should wake scanner");
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage);
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
let observed_generation = crate::runtime_config::scanner_runtime_config_generation();
|
|
let ctx = CancellationToken::new();
|
|
let mut wait = Box::pin(wait_for_next_scanner_cycle(
|
|
&ctx,
|
|
Duration::from_secs(60),
|
|
Some(crate::scanner_io::dirty_usage_generation()),
|
|
observed_generation,
|
|
crate::scanner_io::scanner_maintenance_generation(),
|
|
|| false,
|
|
));
|
|
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
|
|
|
|
let mut config = rustfs_config::server_config::Config::new();
|
|
config.set_defaults();
|
|
crate::runtime_config::apply_scanner_runtime_config(&config).expect("default scanner config should apply");
|
|
let reason = tokio::time::timeout(Duration::from_secs(1), wait)
|
|
.await
|
|
.expect("runtime config should wake scanner before timer");
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::RuntimeConfig);
|
|
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[serial]
|
|
async fn test_wait_for_next_scanner_cycle_reschedules_for_maintenance_change() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
let observed_generation = crate::scanner_io::scanner_maintenance_generation();
|
|
let ctx = CancellationToken::new();
|
|
let mut wait = Box::pin(wait_for_next_scanner_cycle(
|
|
&ctx,
|
|
Duration::from_secs(60),
|
|
Some(crate::scanner_io::dirty_usage_generation()),
|
|
crate::runtime_config::scanner_runtime_config_generation(),
|
|
observed_generation,
|
|
|| false,
|
|
));
|
|
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
|
|
|
|
crate::scanner_io::record_scanner_maintenance_change("photos");
|
|
let reason = tokio::time::timeout(Duration::from_secs(1), wait)
|
|
.await
|
|
.expect("maintenance change should wake scanner before timer");
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::MaintenanceConfig);
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_wait_for_next_scanner_cycle_stops_after_leader_lock_loss() {
|
|
let ctx = CancellationToken::new();
|
|
let reason = wait_for_next_scanner_cycle(
|
|
&ctx,
|
|
Duration::from_secs(60),
|
|
Some(crate::scanner_io::dirty_usage_generation()),
|
|
crate::runtime_config::scanner_runtime_config_generation(),
|
|
crate::scanner_io::scanner_maintenance_generation(),
|
|
|| true,
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost);
|
|
}
|
|
|
|
fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_generation: u64) -> ScannerNodeActivity {
|
|
ScannerNodeActivity {
|
|
instance_id: epoch.to_string(),
|
|
namespace_generation,
|
|
maintenance_generation,
|
|
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
|
topology_digest: [3; 32],
|
|
data_movement_active: false,
|
|
dirty_usage_generation: 5,
|
|
dirty_usage_pending: false,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_activity_snapshot_digest_fences_storage_topology() {
|
|
let first = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
|
let mut changed = first.clone();
|
|
changed.get_mut("node-2").expect("node should exist").topology_digest = [4; 32];
|
|
|
|
assert_ne!(scanner_activity_snapshot_digest(&first), scanner_activity_snapshot_digest(&changed));
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_activity_snapshot_digest_fences_peer_protocol_upgrades() {
|
|
let legacy = BTreeMap::from([(
|
|
"node-2".to_string(),
|
|
ScannerNodeActivity {
|
|
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
|
..scanner_node_activity("epoch-a", 7, 3)
|
|
},
|
|
)]);
|
|
let previous = BTreeMap::from([(
|
|
"node-2".to_string(),
|
|
ScannerNodeActivity {
|
|
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
|
|
..scanner_node_activity("epoch-a", 7, 3)
|
|
},
|
|
)]);
|
|
let current = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
|
|
|
assert_ne!(scanner_activity_snapshot_digest(&legacy), scanner_activity_snapshot_digest(¤t));
|
|
assert_ne!(scanner_activity_snapshot_digest(&previous), scanner_activity_snapshot_digest(¤t));
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_activity_snapshot_fences_data_movement() {
|
|
let idle = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
|
let mut moving = idle.clone();
|
|
moving.get_mut("node-2").expect("node should exist").data_movement_active = true;
|
|
|
|
assert!(scanner_activity_allows_usage_publication(&idle));
|
|
assert!(!scanner_activity_allows_usage_publication(&moving));
|
|
assert_ne!(scanner_activity_snapshot_digest(&idle), scanner_activity_snapshot_digest(&moving));
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_activity_snapshot_digest_fences_dirty_usage_state() {
|
|
let clean = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
|
let pending = BTreeMap::from([(
|
|
"node-2".to_string(),
|
|
ScannerNodeActivity {
|
|
dirty_usage_generation: 6,
|
|
dirty_usage_pending: true,
|
|
..scanner_node_activity("epoch-a", 7, 3)
|
|
},
|
|
)]);
|
|
|
|
assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending));
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() {
|
|
let snapshot = BTreeMap::from([
|
|
(
|
|
LOCAL_SCANNER_ACTIVITY_NODE.to_string(),
|
|
ScannerNodeActivity {
|
|
dirty_usage_generation: 7,
|
|
dirty_usage_pending: true,
|
|
..scanner_node_activity("epoch-local", 7, 3)
|
|
},
|
|
),
|
|
("node-2".to_string(), scanner_node_activity("epoch-clean", 7, 3)),
|
|
(
|
|
"node-3".to_string(),
|
|
ScannerNodeActivity {
|
|
dirty_usage_generation: 11,
|
|
dirty_usage_pending: true,
|
|
..scanner_node_activity("epoch-dirty", 7, 3)
|
|
},
|
|
),
|
|
]);
|
|
|
|
assert_eq!(
|
|
scanner_dirty_usage_acknowledgements(&snapshot),
|
|
vec![ScannerDirtyUsageAcknowledgement {
|
|
host: "node-3".to_string(),
|
|
instance_id: "epoch-dirty".to_string(),
|
|
generation: 11,
|
|
}]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_activity_rejects_one_process_claimed_by_multiple_hosts() {
|
|
let mut instances = BTreeMap::new();
|
|
record_scanner_activity_instance(&mut instances, "node-1", "0123456789abcdef0123456789abcdef")
|
|
.expect("first host should establish the instance identity");
|
|
let err = record_scanner_activity_instance(&mut instances, "node-2", "0123456789abcdef0123456789abcdef")
|
|
.expect_err("a process identity must not represent two cluster nodes");
|
|
|
|
assert!(err.contains("node-1 and node-2"));
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_activity_observation_requires_a_complete_baseline() {
|
|
let mut seen = None;
|
|
let first = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
|
|
|
let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(first.clone()));
|
|
assert_eq!(observation, ScannerActivityObservation::Unverified);
|
|
assert!(error.is_none());
|
|
|
|
let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(first));
|
|
assert_eq!(observation, ScannerActivityObservation::Unchanged);
|
|
assert!(error.is_none());
|
|
|
|
let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]);
|
|
let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(changed));
|
|
assert_eq!(observation, ScannerActivityObservation::Changed);
|
|
assert!(error.is_none());
|
|
|
|
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-b", 8, 0))]);
|
|
let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(restarted));
|
|
assert_eq!(observation, ScannerActivityObservation::Changed);
|
|
assert!(error.is_none());
|
|
|
|
let (observation, error) =
|
|
apply_scanner_activity_probe_result(&mut seen, Err("peer does not support activity probes".to_string()));
|
|
assert_eq!(observation, ScannerActivityObservation::Unverified);
|
|
assert_eq!(error.as_deref(), Some("peer does not support activity probes"));
|
|
assert!(seen.is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn remote_maintenance_change_is_distinct_from_namespace_activity() {
|
|
let previous = BTreeMap::from([
|
|
(LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 2)),
|
|
("node-2".to_string(), scanner_node_activity("remote", 7, 3)),
|
|
]);
|
|
let remote_maintenance_changed = BTreeMap::from([
|
|
(LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 2)),
|
|
("node-2".to_string(), scanner_node_activity("remote", 7, 4)),
|
|
]);
|
|
assert_eq!(
|
|
compare_scanner_activity(&previous, &remote_maintenance_changed),
|
|
ScannerActivityObservation::MaintenanceChanged
|
|
);
|
|
|
|
let local_maintenance_changed = BTreeMap::from([
|
|
(LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 3)),
|
|
("node-2".to_string(), scanner_node_activity("remote", 7, 3)),
|
|
]);
|
|
assert_eq!(
|
|
compare_scanner_activity(&previous, &local_maintenance_changed),
|
|
ScannerActivityObservation::Changed
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn local_maintenance_wakeup_releases_a_remote_maintenance_block() {
|
|
let blocked = scanner_activity_backoff_blocked_after_wake(false, ScannerCycleWakeReason::ClusterMaintenance);
|
|
assert!(blocked);
|
|
|
|
let unblocked = scanner_activity_backoff_blocked_after_wake(blocked, ScannerCycleWakeReason::MaintenanceConfig);
|
|
assert!(!unblocked);
|
|
assert!(scanner_activity_backoff_blocked_after_wake(
|
|
blocked,
|
|
ScannerCycleWakeReason::ClusterActivity
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn scanner_activity_after_a_cycle_restores_the_base_interval() {
|
|
let runtime_config = ScannerRuntimeConfig {
|
|
cycle_interval: Duration::from_secs(60),
|
|
..Default::default()
|
|
};
|
|
let mut backoff = ScannerCleanIdleBackoff { interval_multiplier: 8 };
|
|
|
|
record_scanner_cycle_result(
|
|
&mut backoff,
|
|
&runtime_config,
|
|
true,
|
|
ScannerCycleWakeReason::Timer,
|
|
ScannerCycleOutcome::Completed,
|
|
scanner_activity_observed_work(ScannerActivityObservation::Changed),
|
|
);
|
|
|
|
let plan = scanner_cycle_wait_plan(&runtime_config, backoff, true, std::convert::identity);
|
|
assert_eq!(plan.effective_interval, Duration::from_secs(60));
|
|
assert_eq!(plan.delay, Duration::from_secs(60));
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
#[serial]
|
|
async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
let ctx = CancellationToken::new();
|
|
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]));
|
|
let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]);
|
|
|
|
let reason = wait_for_next_scanner_cycle_with_activity(
|
|
&ctx,
|
|
Duration::from_secs(120),
|
|
Some(Duration::from_secs(60)),
|
|
&mut seen,
|
|
ScannerCycleObservedGenerations {
|
|
dirty_usage: Some(crate::scanner_io::dirty_usage_generation()),
|
|
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
|
|
maintenance: crate::scanner_io::scanner_maintenance_generation(),
|
|
defer_cluster_activity: false,
|
|
},
|
|
|| false,
|
|
|| std::future::ready(Ok(changed.clone())),
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity);
|
|
assert_eq!(seen, Some(changed));
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
#[serial]
|
|
async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
let ctx = CancellationToken::new();
|
|
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]));
|
|
let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]);
|
|
|
|
let reason = wait_for_next_scanner_cycle_with_activity(
|
|
&ctx,
|
|
Duration::from_secs(120),
|
|
Some(Duration::from_secs(60)),
|
|
&mut seen,
|
|
ScannerCycleObservedGenerations {
|
|
dirty_usage: None,
|
|
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
|
|
maintenance: crate::scanner_io::scanner_maintenance_generation(),
|
|
defer_cluster_activity: true,
|
|
},
|
|
|| false,
|
|
|| std::future::ready(Ok(changed.clone())),
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::Timer);
|
|
assert_eq!(seen, Some(changed));
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
#[serial]
|
|
async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
let ctx = CancellationToken::new();
|
|
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]));
|
|
let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 4))]);
|
|
|
|
let reason = wait_for_next_scanner_cycle_with_activity(
|
|
&ctx,
|
|
Duration::from_secs(120),
|
|
Some(Duration::from_secs(60)),
|
|
&mut seen,
|
|
ScannerCycleObservedGenerations {
|
|
dirty_usage: Some(crate::scanner_io::dirty_usage_generation()),
|
|
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
|
|
maintenance: crate::scanner_io::scanner_maintenance_generation(),
|
|
defer_cluster_activity: false,
|
|
},
|
|
|| false,
|
|
|| std::future::ready(Ok(changed.clone())),
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::ClusterMaintenance);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
#[serial]
|
|
async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
let ctx = CancellationToken::new();
|
|
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]));
|
|
|
|
let reason = wait_for_next_scanner_cycle_with_activity(
|
|
&ctx,
|
|
Duration::from_secs(120),
|
|
Some(Duration::from_secs(60)),
|
|
&mut seen,
|
|
ScannerCycleObservedGenerations {
|
|
dirty_usage: Some(crate::scanner_io::dirty_usage_generation()),
|
|
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
|
|
maintenance: crate::scanner_io::scanner_maintenance_generation(),
|
|
defer_cluster_activity: false,
|
|
},
|
|
|| false,
|
|
|| std::future::ready(Err("node-2 is unreachable".to_string())),
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::ClusterActivityUnavailable);
|
|
assert!(seen.is_none());
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
#[serial]
|
|
async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_clean() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
let ctx = CancellationToken::new();
|
|
let expected = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
|
let mut seen = Some(expected.clone());
|
|
|
|
let reason = wait_for_next_scanner_cycle_with_activity(
|
|
&ctx,
|
|
Duration::from_secs(120),
|
|
Some(Duration::from_secs(60)),
|
|
&mut seen,
|
|
ScannerCycleObservedGenerations {
|
|
dirty_usage: Some(crate::scanner_io::dirty_usage_generation()),
|
|
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
|
|
maintenance: crate::scanner_io::scanner_maintenance_generation(),
|
|
defer_cluster_activity: false,
|
|
},
|
|
|| false,
|
|
|| std::future::ready(Ok(expected.clone())),
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::Timer);
|
|
assert_eq!(seen, Some(expected));
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
#[serial]
|
|
async fn scanner_activity_probe_wait_is_cancellation_aware() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
let ctx = CancellationToken::new();
|
|
let cancel = ctx.clone();
|
|
tokio::spawn(async move {
|
|
tokio::time::sleep(Duration::from_secs(61)).await;
|
|
cancel.cancel();
|
|
});
|
|
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]));
|
|
|
|
let reason = wait_for_next_scanner_cycle_with_activity(
|
|
&ctx,
|
|
Duration::from_secs(120),
|
|
Some(Duration::from_secs(60)),
|
|
&mut seen,
|
|
ScannerCycleObservedGenerations {
|
|
dirty_usage: Some(crate::scanner_io::dirty_usage_generation()),
|
|
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
|
|
maintenance: crate::scanner_io::scanner_maintenance_generation(),
|
|
defer_cluster_activity: false,
|
|
},
|
|
|| false,
|
|
std::future::pending::<Result<ScannerActivitySnapshot, String>>,
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::Cancelled);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
#[serial]
|
|
async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() {
|
|
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
|
let ctx = CancellationToken::new();
|
|
let lock_lost = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
|
let lose_lock = Arc::clone(&lock_lost);
|
|
tokio::spawn(async move {
|
|
tokio::time::sleep(Duration::from_secs(61)).await;
|
|
lose_lock.store(true, std::sync::atomic::Ordering::Release);
|
|
});
|
|
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]));
|
|
|
|
let reason = wait_for_next_scanner_cycle_with_activity(
|
|
&ctx,
|
|
Duration::from_secs(120),
|
|
Some(Duration::from_secs(60)),
|
|
&mut seen,
|
|
ScannerCycleObservedGenerations {
|
|
dirty_usage: Some(crate::scanner_io::dirty_usage_generation()),
|
|
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
|
|
maintenance: crate::scanner_io::scanner_maintenance_generation(),
|
|
defer_cluster_activity: false,
|
|
},
|
|
|| lock_lost.load(std::sync::atomic::Ordering::Acquire),
|
|
std::future::pending::<Result<ScannerActivitySnapshot, String>>,
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost);
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
|
|
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
|
let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle());
|
|
assert_eq!(mode, HealScanMode::Deep);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() {
|
|
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
|
let recent = Utc::now() - chrono::Duration::minutes(30);
|
|
let old = Utc::now() - chrono::Duration::hours(2);
|
|
|
|
assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent), bitrot_scan_cycle()), HealScanMode::Normal);
|
|
assert_eq!(get_cycle_scan_mode(2048, 0, Some(old), bitrot_scan_cycle()), HealScanMode::Deep);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
|
|
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || {
|
|
assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_background_heal_info_for_scan_start_marks_deep_active() {
|
|
let now = Utc::now();
|
|
let info =
|
|
background_heal_info_for_scan_start(BackgroundHealInfo::default(), 7, HealScanMode::Deep, now, bitrot_scan_cycle())
|
|
.expect("deep scan should update background heal info");
|
|
|
|
assert_eq!(info.current_scan_mode, HealScanMode::Deep);
|
|
assert_eq!(info.bitrot_start_cycle, 7);
|
|
assert_eq!(info.bitrot_start_time, Some(now));
|
|
}
|
|
|
|
#[test]
|
|
#[serial]
|
|
fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
|
|
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
|
|
let started_at = Utc::now();
|
|
let info = BackgroundHealInfo {
|
|
bitrot_start_time: Some(started_at),
|
|
bitrot_start_cycle: 7,
|
|
current_scan_mode: HealScanMode::Normal,
|
|
};
|
|
|
|
let info = background_heal_info_for_scan_start(info, 8, HealScanMode::Deep, Utc::now(), bitrot_scan_cycle())
|
|
.expect("deep scan should mark active status");
|
|
|
|
assert_eq!(info.current_scan_mode, HealScanMode::Deep);
|
|
assert_eq!(info.bitrot_start_cycle, 7);
|
|
assert_eq!(info.bitrot_start_time, Some(started_at));
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn test_background_heal_info_for_scan_complete_marks_deep_idle() {
|
|
let started_at = Utc::now();
|
|
let info = BackgroundHealInfo {
|
|
bitrot_start_time: Some(started_at),
|
|
bitrot_start_cycle: 7,
|
|
current_scan_mode: HealScanMode::Deep,
|
|
};
|
|
|
|
let info = background_heal_info_for_scan_complete(info, HealScanMode::Deep)
|
|
.expect("completed deep scan should update background heal info");
|
|
|
|
assert_eq!(info.current_scan_mode, HealScanMode::Normal);
|
|
assert_eq!(info.bitrot_start_cycle, 7);
|
|
assert_eq!(info.bitrot_start_time, Some(started_at));
|
|
}
|
|
|
|
#[test]
|
|
fn test_background_heal_info_for_scan_complete_leaves_normal_scan_unchanged() {
|
|
let info = BackgroundHealInfo {
|
|
bitrot_start_time: Some(Utc::now()),
|
|
bitrot_start_cycle: 7,
|
|
current_scan_mode: HealScanMode::Normal,
|
|
};
|
|
|
|
assert!(background_heal_info_for_scan_complete(info, HealScanMode::Normal).is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn test_background_heal_info_for_failed_scan_preserves_deep_mode() {
|
|
let info = BackgroundHealInfo {
|
|
bitrot_start_time: Some(Utc::now()),
|
|
bitrot_start_cycle: 7,
|
|
current_scan_mode: HealScanMode::Deep,
|
|
};
|
|
|
|
assert!(background_heal_info_for_scan_result(info, HealScanMode::Deep, false).is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn test_retain_recent_cycle_completions_keeps_last_entries() {
|
|
let base = Utc::now();
|
|
let keep = data_usage_update_dir_cycles() as usize;
|
|
let mut completed: Vec<_> = (0..keep + 2).map(|i| base + chrono::Duration::seconds(i as i64)).collect();
|
|
|
|
retain_recent_cycle_completions(&mut completed);
|
|
|
|
assert_eq!(completed.len(), keep);
|
|
assert_eq!(completed.first().copied(), Some(base + chrono::Duration::seconds(2)));
|
|
assert_eq!(completed.last().copied(), Some(base + chrono::Duration::seconds((keep + 1) as i64)));
|
|
}
|