mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 12:09:12 +00:00
houseme
71859ff83c
Run one supervised cleanup attempt for an existing operator reset, with strict phase and revision checks under the original leader lock. Keep v3 reset authorization and responses unchanged, report deferred status, and bound probe and shutdown waits without aborting in-flight reset ownership. Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: zhi22915 <qiuzgang@gmail.com>
3537 lines
144 KiB
Rust
3537 lines
144 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::future::Future;
|
|
#[cfg(test)]
|
|
use std::sync::Mutex as StdMutex;
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
use std::sync::{Arc, LazyLock, RwLock};
|
|
|
|
use self::heal_info::{BackgroundHealInfoReadStatus, read_background_heal_info_with_epoch, save_background_heal_info_for_epoch};
|
|
use crate::data_usage_define::{
|
|
BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH,
|
|
DATA_USAGE_RECOVERY_PATH, DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision,
|
|
read_config_with_revision,
|
|
};
|
|
use crate::runtime_config::{
|
|
ScannerRuntimeConfig, ScannerRuntimeConfigSource, refresh_scanner_runtime_config_from_global, scanner_bitrot_cycle,
|
|
scanner_cycle_interval, scanner_runtime_config_changed, scanner_runtime_config_generation, scanner_start_delay,
|
|
set_scanner_default_cycle_secs,
|
|
};
|
|
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason};
|
|
use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
|
|
use crate::scanner_io::{
|
|
ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, dirty_usage_bucket_notified, dirty_usage_buckets_pending,
|
|
dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation,
|
|
};
|
|
use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed};
|
|
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard};
|
|
use crate::{ScannerConfigObjectDelete, ScannerObjectIO, ScannerObjectOptions};
|
|
use bytes::Bytes;
|
|
use chrono::{DateTime, Utc};
|
|
use rustfs_config::ScannerSpeed;
|
|
#[cfg(test)]
|
|
use rustfs_config::{
|
|
ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CYCLE_MAX_DIRECTORIES, ENV_SCANNER_CYCLE_MAX_DURATION_SECS,
|
|
ENV_SCANNER_CYCLE_MAX_OBJECTS,
|
|
};
|
|
use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
|
|
use rustfs_data_usage::observed_data_usage_is_newer;
|
|
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
|
use rustfs_lock::{NamespaceLockGuard, error::LockError};
|
|
use rustfs_scanner_metrics::metrics::{
|
|
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
|
|
emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded,
|
|
global_metrics,
|
|
};
|
|
use serde::{Deserialize, Serialize};
|
|
use sha2::{Digest as _, Sha256};
|
|
use tokio::sync::{Notify, mpsc};
|
|
use tokio::time::{Duration, Instant};
|
|
use tokio_util::sync::CancellationToken;
|
|
use tokio_util::task::AbortOnDropHandle;
|
|
use tracing::{debug, error, info, instrument, warn};
|
|
use uuid::Uuid;
|
|
|
|
use crate::storage_api::ScannerStorage;
|
|
use crate::storage_api::scan::{
|
|
BucketOptions, NamespaceLocking as _, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
|
|
SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
|
};
|
|
use crate::{
|
|
ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
|
|
ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
|
|
get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
|
|
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions_and_lease_fence_and_scope,
|
|
save_config_with_preconditions, save_config_with_publication_admission_for_epoch, scanner_publication_admission_for_epoch,
|
|
scanner_publication_epoch, scanner_publication_epoch_changed,
|
|
};
|
|
|
|
const LOG_COMPONENT_SCANNER: &str = "scanner";
|
|
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
|
|
const LOG_SUBSYSTEM_BACKGROUND_HEAL: &str = "background_heal";
|
|
const EVENT_SCANNER_CYCLE_STATE: &str = "scanner_cycle_state";
|
|
const EVENT_SCANNER_LOCK_STATE: &str = "scanner_lock_state";
|
|
const EVENT_SCANNER_PERSIST_STATE: &str = "scanner_persist_state";
|
|
const EVENT_SCANNER_RUNTIME_CONFIG: &str = "scanner_runtime_config";
|
|
const EVENT_SCANNER_BACKGROUND_HEAL_STATE: &str = "scanner_background_heal_state";
|
|
const METRIC_SCANNER_LEADER_LOCK_TOTAL: &str = "rustfs_scanner_leader_lock_total";
|
|
const CLEAN_IDLE_MAX_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
|
|
const MAX_SCANNER_SCHEDULE_DELAY: Duration = Duration::from_secs(365 * 24 * 60 * 60);
|
|
|
|
#[cfg(test)]
|
|
static SCANNER_STARTUP_OBSERVED_PROBE: LazyLock<StdMutex<Option<Arc<ScannerStartupObservedProbeState>>>> =
|
|
LazyLock::new(|| StdMutex::new(None));
|
|
|
|
#[cfg(test)]
|
|
struct ScannerStartupObservedProbeState {
|
|
observed: Notify,
|
|
resume: Notify,
|
|
}
|
|
|
|
#[cfg(test)]
|
|
struct ScannerObservedProbeState {
|
|
store_key: usize,
|
|
paused: bool,
|
|
notify: Notify,
|
|
}
|
|
|
|
#[cfg(test)]
|
|
pub(super) struct ScannerStartupObservedProbe {
|
|
state: Arc<ScannerStartupObservedProbeState>,
|
|
}
|
|
|
|
#[cfg(test)]
|
|
static SCANNER_RUNTIME_OBSERVED_PROBE: LazyLock<StdMutex<Option<Arc<ScannerObservedProbeState>>>> =
|
|
LazyLock::new(|| StdMutex::new(None));
|
|
|
|
#[cfg(test)]
|
|
pub(super) struct ScannerRuntimeObservedProbe {
|
|
state: Arc<ScannerObservedProbeState>,
|
|
}
|
|
|
|
#[cfg(test)]
|
|
impl ScannerStartupObservedProbe {
|
|
pub(super) fn install() -> Self {
|
|
let state = Arc::new(ScannerStartupObservedProbeState {
|
|
observed: Notify::new(),
|
|
resume: Notify::new(),
|
|
});
|
|
let mut probe = SCANNER_STARTUP_OBSERVED_PROBE
|
|
.lock()
|
|
.expect("scanner startup observed probe should not be poisoned");
|
|
assert!(probe.is_none(), "scanner startup observed probe must be unique");
|
|
*probe = Some(state.clone());
|
|
Self { state }
|
|
}
|
|
|
|
pub(super) async fn wait(&self) {
|
|
tokio::time::timeout(Duration::from_secs(5), self.state.observed.notified())
|
|
.await
|
|
.expect("scanner should complete startup pause-backlog observation");
|
|
}
|
|
|
|
pub(super) fn resume(&self) {
|
|
self.state.resume.notify_one();
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
impl ScannerRuntimeObservedProbe {
|
|
pub(super) fn install(storeapi: &Arc<ECStore>, paused: bool) -> Self {
|
|
let state = Arc::new(ScannerObservedProbeState {
|
|
store_key: scanner_observed_probe_store_key(storeapi),
|
|
paused,
|
|
notify: Notify::new(),
|
|
});
|
|
let mut probe = SCANNER_RUNTIME_OBSERVED_PROBE
|
|
.lock()
|
|
.expect("scanner runtime observed probe should not be poisoned");
|
|
assert!(probe.is_none(), "scanner runtime observed probe must be unique");
|
|
*probe = Some(state.clone());
|
|
Self { state }
|
|
}
|
|
|
|
pub(super) async fn wait(&self) {
|
|
tokio::time::timeout(Duration::from_secs(10), self.state.notify.notified())
|
|
.await
|
|
.expect("scanner should complete runtime pause-backlog observation");
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
impl Drop for ScannerStartupObservedProbe {
|
|
fn drop(&mut self) {
|
|
let mut probe = SCANNER_STARTUP_OBSERVED_PROBE
|
|
.lock()
|
|
.expect("scanner startup observed probe should not be poisoned");
|
|
if probe.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
|
*probe = None;
|
|
}
|
|
self.state.resume.notify_one();
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
impl Drop for ScannerRuntimeObservedProbe {
|
|
fn drop(&mut self) {
|
|
let mut probe = SCANNER_RUNTIME_OBSERVED_PROBE
|
|
.lock()
|
|
.expect("scanner runtime observed probe should not be poisoned");
|
|
if probe.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
|
*probe = None;
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
async fn notify_scanner_startup_observed_for_test() {
|
|
let probe = {
|
|
SCANNER_STARTUP_OBSERVED_PROBE
|
|
.lock()
|
|
.expect("scanner startup observed probe should not be poisoned")
|
|
.clone()
|
|
};
|
|
if let Some(probe) = probe {
|
|
probe.observed.notify_one();
|
|
probe.resume.notified().await;
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn scanner_observed_probe_store_key<S>(storeapi: &Arc<S>) -> usize
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
storeapi.scanner_observed_probe_store_key()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn notify_scanner_runtime_observed_for_test<S>(storeapi: &Arc<S>, observation: ScannerPauseBacklogObservation)
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
if let Some(probe) = SCANNER_RUNTIME_OBSERVED_PROBE
|
|
.lock()
|
|
.expect("scanner runtime observed probe should not be poisoned")
|
|
.clone()
|
|
&& probe.store_key == scanner_observed_probe_store_key(storeapi)
|
|
&& probe.paused == observation.paused
|
|
{
|
|
probe.notify.notify_one();
|
|
}
|
|
}
|
|
|
|
const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
|
|
/// First-retry delay after a scanner cycle cannot publish authoritative usage.
|
|
///
|
|
/// A superseded cycle is the *expected* outcome of the dirty-usage fast path:
|
|
/// a write burst marks buckets dirty, the scanner wakes within milliseconds,
|
|
/// and the still-landing writes then supersede the snapshot it took. Charging
|
|
/// that first race a full cycle interval means a burst of writes surfaces in
|
|
/// usage/quota accounting roughly two cycles late (measured: ~120 s on an
|
|
/// otherwise idle instance whose clean-idle backoff had doubled a 60 s
|
|
/// interval), which defeats the fast path it is meant to protect.
|
|
///
|
|
/// The exponential growth in [`ScannerRetryBackoff::retry_interval`] is
|
|
/// what protects against a persistently hot bucket driving an unbroken
|
|
/// full-scan loop, so it can start small: 5 s, 10 s, 20 s … capped by
|
|
/// [`SCANNER_RETRY_MAX_INTERVAL`]. A one-off race recovers in seconds; a
|
|
/// genuinely hot bucket still reaches minute-scale backoff within a handful of
|
|
/// cycles. Preflight deferrals use the same bounded schedule so a temporarily
|
|
/// unavailable peer cannot drive a tight retry loop.
|
|
const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
|
|
const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
|
/// A transient backend outage remains self-healing after the short retry
|
|
/// budget is exhausted, but the probe is intentionally sparse until storage
|
|
/// recovers or an operator reset wakes the scanner.
|
|
const SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL: Duration = Duration::from_secs(5 * 60);
|
|
/// Permanent recovery states still get a sparse status probe so a reset that
|
|
/// races the wait registration cannot leave the scanner asleep forever.
|
|
const SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL: Duration = Duration::from_secs(5 * 60);
|
|
const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1);
|
|
#[cfg(not(test))]
|
|
const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
|
|
#[cfg(test)]
|
|
const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(50);
|
|
const MAINTENANCE_FEATURE_INSPECTION_TIMEOUT: Duration = Duration::from_secs(30);
|
|
const MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5 * 60);
|
|
const MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(60 * 60);
|
|
const MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS: usize = 2;
|
|
const SCANNER_PERSIST_CAS_RETRIES: usize = 2;
|
|
const DATA_USAGE_BACKUP_INTERVAL_CYCLES: u64 = 10;
|
|
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
|
|
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
|
|
const SCANNER_CYCLE_STATE_MAGIC: &[u8; 8] = b"RSCYC001";
|
|
const SCANNER_CYCLE_STATE_HEADER_LEN: usize = 24;
|
|
#[cfg(test)]
|
|
const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
|
|
#[cfg(test)]
|
|
type ScannerCycleStatePersistTestHook = (u64, Arc<Notify>);
|
|
#[cfg(test)]
|
|
static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCycleStatePersistTestHook>>> =
|
|
LazyLock::new(|| StdMutex::new(None));
|
|
|
|
static SCANNER_CYCLE_RECOVERY_WAKE: LazyLock<Notify> = LazyLock::new(Notify::new);
|
|
|
|
fn remote_publication_lease_fence_targets_are_required(target_count: usize, grants_present: bool, fence_present: bool) -> bool {
|
|
target_count > 0 && (!grants_present || !fence_present)
|
|
}
|
|
|
|
pub(super) fn notify_scanner_cycle_recovery_wake() {
|
|
SCANNER_CYCLE_RECOVERY_WAKE.notify_one();
|
|
}
|
|
|
|
#[cfg(test)]
|
|
struct ScannerCycleStatePersistTestHookGuard;
|
|
|
|
#[cfg(test)]
|
|
impl Drop for ScannerCycleStatePersistTestHookGuard {
|
|
fn drop(&mut self) {
|
|
*SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
|
|
.lock()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn set_scanner_cycle_state_persist_test_hook(leader_epoch: u64, reached: Arc<Notify>) -> ScannerCycleStatePersistTestHookGuard {
|
|
*SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
|
|
.lock()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((leader_epoch, reached));
|
|
ScannerCycleStatePersistTestHookGuard
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn notify_scanner_cycle_state_persist_test_hook(leader_epoch: u64) {
|
|
let reached = SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
|
|
.lock()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
|
.as_ref()
|
|
.filter(|(expected_epoch, _)| *expected_epoch == leader_epoch)
|
|
.map(|(_, reached)| reached.clone());
|
|
if let Some(reached) = reached {
|
|
reached.notify_one();
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Serialize)]
|
|
#[non_exhaustive]
|
|
pub struct ScannerCycleScheduleStatus {
|
|
execution_role: &'static str,
|
|
effective_interval_available: bool,
|
|
effective_interval_seconds: u64,
|
|
clean_idle_backoff_enabled: bool,
|
|
clean_idle_backoff_multiplier: u64,
|
|
superseded_retry_backoff_enabled: bool,
|
|
superseded_cycles: u32,
|
|
}
|
|
|
|
impl Default for ScannerCycleScheduleStatus {
|
|
fn default() -> Self {
|
|
Self {
|
|
execution_role: "unknown",
|
|
effective_interval_available: false,
|
|
effective_interval_seconds: 0,
|
|
clean_idle_backoff_enabled: false,
|
|
clean_idle_backoff_multiplier: 1,
|
|
superseded_retry_backoff_enabled: false,
|
|
superseded_cycles: 0,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl ScannerCycleScheduleStatus {
|
|
pub fn effective_interval_seconds(self) -> u64 {
|
|
self.effective_interval_seconds
|
|
}
|
|
}
|
|
|
|
static SCANNER_CYCLE_SCHEDULE: LazyLock<RwLock<ScannerCycleScheduleStatus>> =
|
|
LazyLock::new(|| RwLock::new(ScannerCycleScheduleStatus::default()));
|
|
|
|
pub fn scanner_cycle_schedule_status() -> ScannerCycleScheduleStatus {
|
|
*SCANNER_CYCLE_SCHEDULE.read().unwrap_or_else(|poisoned| poisoned.into_inner())
|
|
}
|
|
|
|
fn record_scanner_cycle_schedule(
|
|
effective_interval: Duration,
|
|
clean_idle_backoff_enabled: bool,
|
|
clean_idle_backoff_multiplier: u64,
|
|
superseded_retry_backoff_enabled: bool,
|
|
superseded_cycles: u32,
|
|
) {
|
|
let effective_interval_seconds = effective_interval
|
|
.as_secs()
|
|
.saturating_add(u64::from(effective_interval.subsec_nanos() != 0));
|
|
let mut schedule = SCANNER_CYCLE_SCHEDULE
|
|
.write()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
|
*schedule = ScannerCycleScheduleStatus {
|
|
execution_role: "leader",
|
|
effective_interval_available: true,
|
|
effective_interval_seconds,
|
|
clean_idle_backoff_enabled,
|
|
clean_idle_backoff_multiplier: clean_idle_backoff_multiplier.max(1),
|
|
superseded_retry_backoff_enabled,
|
|
superseded_cycles,
|
|
};
|
|
}
|
|
|
|
fn record_scanner_cycle_schedule_role(execution_role: &'static str) {
|
|
let mut schedule = SCANNER_CYCLE_SCHEDULE
|
|
.write()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
|
*schedule = ScannerCycleScheduleStatus {
|
|
execution_role,
|
|
..ScannerCycleScheduleStatus::default()
|
|
};
|
|
}
|
|
|
|
fn reset_scanner_cycle_schedule() {
|
|
record_scanner_cycle_schedule_role("unknown");
|
|
}
|
|
|
|
enum ScannerLeaderLockFailure<'a> {
|
|
Contended,
|
|
Failed(&'a LockError),
|
|
}
|
|
|
|
fn classify_scanner_leader_lock_failure(error: &LockError) -> ScannerLeaderLockFailure<'_> {
|
|
match error {
|
|
LockError::Timeout { .. } => ScannerLeaderLockFailure::Contended,
|
|
error => ScannerLeaderLockFailure::Failed(error),
|
|
}
|
|
}
|
|
|
|
/// Returns the base cycle interval.
|
|
/// Priority order:
|
|
/// 1. RUSTFS_SCANNER_CYCLE (if set, overrides everything)
|
|
/// 2. RUSTFS_SCANNER_START_DELAY_SECS (for backward compatibility)
|
|
/// 3. Deployment-specific default cycle override
|
|
/// 4. RUSTFS_SCANNER_SPEED preset
|
|
#[cfg(test)]
|
|
fn cycle_interval() -> Duration {
|
|
resolve_scanner_runtime_config().cycle_interval
|
|
}
|
|
|
|
fn scanner_cycle_budget_config() -> ScannerCycleBudgetConfig {
|
|
resolve_scanner_runtime_config().cycle_budget
|
|
}
|
|
|
|
fn record_scanner_leader_lock_state(state: &'static str) {
|
|
metrics::counter!(
|
|
METRIC_SCANNER_LEADER_LOCK_TOTAL,
|
|
"state" => state
|
|
)
|
|
.increment(1);
|
|
}
|
|
|
|
async fn finish_scanner_leader_iteration(lock_lost: bool, state: &'static str, error: String) {
|
|
reset_scanner_cycle_schedule();
|
|
let liveness_already_recorded = lock_lost && !global_metrics().report().await.leader_lock_held_by_this_process;
|
|
if !liveness_already_recorded {
|
|
global_metrics().record_scanner_leader_liveness(state, false, error).await;
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn scanner_cycle_max_duration() -> Option<Duration> {
|
|
resolve_scanner_runtime_config().cycle_budget.max_duration
|
|
}
|
|
|
|
fn resolve_scanner_runtime_config() -> crate::runtime_config::ScannerRuntimeConfig {
|
|
#[cfg(test)]
|
|
{
|
|
crate::runtime_config::resolve_scanner_runtime_config_from_global()
|
|
}
|
|
#[cfg(not(test))]
|
|
{
|
|
crate::runtime_config::current_scanner_runtime_config()
|
|
}
|
|
}
|
|
|
|
fn scan_cycle_partial_reason(reason: Option<ScannerCycleBudgetReason>) -> ScanCyclePartialReason {
|
|
match reason {
|
|
Some(ScannerCycleBudgetReason::Runtime) => ScanCyclePartialReason::Runtime,
|
|
Some(ScannerCycleBudgetReason::Objects) => ScanCyclePartialReason::Objects,
|
|
Some(ScannerCycleBudgetReason::Directories) => ScanCyclePartialReason::Directories,
|
|
None => ScanCyclePartialReason::Unknown,
|
|
}
|
|
}
|
|
|
|
fn scan_cycle_partial_source(reason: Option<ScannerCycleBudgetReason>) -> Option<ScannerWorkSource> {
|
|
match reason {
|
|
Some(ScannerCycleBudgetReason::Objects | ScannerCycleBudgetReason::Directories) => Some(ScannerWorkSource::Usage),
|
|
Some(ScannerCycleBudgetReason::Runtime) | None => None,
|
|
}
|
|
}
|
|
|
|
/// Compute a randomized inter-cycle sleep.
|
|
// Delay is scan interval +- 10%, with a floor of 1 second.
|
|
fn randomized_cycle_delay() -> Duration {
|
|
randomized_cycle_delay_for(scanner_cycle_interval())
|
|
}
|
|
|
|
fn randomized_cycle_delay_for(interval: Duration) -> Duration {
|
|
let interval = interval.max(Duration::from_secs(1)).min(MAX_SCANNER_SCHEDULE_DELAY);
|
|
// Uniform in [-0.1, 0.1), keeping actual delay within 10% of interval.
|
|
let jitter_factor = (rand::random::<f64>() * 0.2) - 0.1;
|
|
let delay = interval.mul_f64(1.0 + jitter_factor);
|
|
delay.max(Duration::from_secs(1)).min(MAX_SCANNER_SCHEDULE_DELAY)
|
|
}
|
|
|
|
fn cap_clean_idle_cycle_delay(delay: Duration, max_interval: Duration, enabled: bool) -> Duration {
|
|
if !enabled {
|
|
return delay;
|
|
}
|
|
|
|
let max_interval = max_interval.max(Duration::from_secs(1));
|
|
if delay <= max_interval {
|
|
return delay;
|
|
}
|
|
|
|
// Reflect positive jitter below the cap instead of collapsing every
|
|
// positive sample onto the same instant once backoff reaches its ceiling.
|
|
max_interval
|
|
.saturating_sub(delay.saturating_sub(max_interval))
|
|
.max(Duration::from_secs(1))
|
|
}
|
|
|
|
fn initial_scanner_delay_for(start_delay_secs: Option<u64>) -> Duration {
|
|
start_delay_secs
|
|
.map(|secs| randomized_cycle_delay_for(Duration::from_secs(secs)))
|
|
.unwrap_or_else(randomized_cycle_delay)
|
|
}
|
|
|
|
fn initial_scanner_delay_for_startup(
|
|
start_delay_secs: Option<u64>,
|
|
usage_cache_is_cold: bool,
|
|
has_buckets: bool,
|
|
has_active_replication: bool,
|
|
) -> Duration {
|
|
// Skip the startup delay when the cache is cold (first ever scan) OR when active replication
|
|
// rules exist. A cold usage cache also covers startup-before-bucket-creation: running the
|
|
// first cycle promptly keeps later bucket metrics bounded by the normal scanner cycle instead
|
|
// of an extra startup delay. Replication config is live-read at startup by
|
|
// configure_scanner_defaults, so this signal is always current regardless of when the persisted
|
|
// DataUsageInfo was last written.
|
|
if usage_cache_is_cold || (has_active_replication && has_buckets) {
|
|
Duration::ZERO
|
|
} else {
|
|
initial_scanner_delay_for(start_delay_secs)
|
|
}
|
|
}
|
|
|
|
fn data_usage_info_is_cold(info: &DataUsageInfo) -> bool {
|
|
!info.is_complete_bucket_usage_snapshot()
|
|
}
|
|
|
|
pub(super) fn data_usage_info_has_persisted_baseline_identity(info: &DataUsageInfo) -> bool {
|
|
if info.is_complete_bucket_usage_snapshot() {
|
|
return true;
|
|
}
|
|
|
|
// Pre-marker snapshots remain readable only when their legacy identity is
|
|
// complete: a timestamp, a scanner cycle, and an exact bucket cardinality.
|
|
// A current snapshot with only scanner_epoch/scanner_cycle (or an explicit
|
|
// incomplete marker) is not evidence of a durable usage baseline.
|
|
!info.usage_snapshot_bootstrap_pending
|
|
&& !info.usage_snapshot_complete
|
|
&& info.scanner_epoch.is_none()
|
|
&& info.usage_snapshot_converged != Some(false)
|
|
&& info.last_update.is_some()
|
|
&& info.scanner_cycle.is_some()
|
|
&& u64::try_from(info.buckets_usage.len()).ok() == Some(info.buckets_count)
|
|
}
|
|
|
|
pub(super) fn data_usage_info_is_bootstrap_pending(info: &DataUsageInfo) -> bool {
|
|
let Some(last_update) = info.last_update else {
|
|
return false;
|
|
};
|
|
|
|
info == &scanner_usage_bootstrap_marker(last_update, info.scanner_epoch)
|
|
}
|
|
|
|
pub(super) fn scanner_usage_bootstrap_marker(last_update: std::time::SystemTime, scanner_epoch: Option<u64>) -> DataUsageInfo {
|
|
DataUsageInfo {
|
|
last_update: Some(last_update),
|
|
scanner_epoch,
|
|
usage_snapshot_converged: Some(false),
|
|
usage_snapshot_bootstrap_pending: true,
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
fn usage_cache_needs_prompt_scan(authoritative: &DataUsageInfo, observed: Option<&DataUsageInfo>) -> bool {
|
|
data_usage_info_is_cold(authoritative)
|
|
|| observed.is_some_and(|observed| observed_data_usage_is_newer(observed, authoritative))
|
|
}
|
|
|
|
async fn read_data_usage_config_for_startup(storeapi: &Arc<impl ScannerObjectIO>) -> Result<Option<Vec<u8>>, EcstoreError> {
|
|
async fn read_pair(storeapi: &Arc<impl ScannerObjectIO>, primary_path: &str) -> Result<Option<Vec<u8>>, EcstoreError> {
|
|
match read_config(storeapi.clone(), primary_path).await {
|
|
Ok(data) => Ok(Some(data)),
|
|
Err(EcstoreError::ConfigNotFound) => {
|
|
let backup_path = format!("{primary_path}.bkp");
|
|
match read_config(storeapi.clone(), backup_path.as_str()).await {
|
|
Ok(data) => Ok(Some(data)),
|
|
Err(EcstoreError::ConfigNotFound) => Ok(None),
|
|
Err(err) => Err(err),
|
|
}
|
|
}
|
|
Err(err) => Err(err),
|
|
}
|
|
}
|
|
|
|
for path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] {
|
|
if let Some(data) = read_pair(storeapi, path).await? {
|
|
return Ok(Some(data));
|
|
}
|
|
}
|
|
Ok(None)
|
|
}
|
|
|
|
fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool {
|
|
data_usage_info
|
|
.scanner_cycle
|
|
.is_some_and(|cycle| cycle % DATA_USAGE_BACKUP_INTERVAL_CYCLES == 0)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[allow(dead_code)]
|
|
async fn sync_data_usage_backup_from_primary(
|
|
ctx: &CancellationToken,
|
|
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
|
) -> Result<(), EcstoreError> {
|
|
sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(ctx, storeapi, None, None, None).await
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
|
|
ctx: &CancellationToken,
|
|
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
|
expected_publication_epoch: Option<u64>,
|
|
remote_lease_deadline: Option<std::time::Instant>,
|
|
scanner_publication_lease_fence: Option<&str>,
|
|
) -> Result<(), EcstoreError> {
|
|
sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence_and_scope(
|
|
ctx,
|
|
storeapi,
|
|
expected_publication_epoch,
|
|
remote_lease_deadline,
|
|
scanner_publication_lease_fence,
|
|
Vec::new(),
|
|
Arc::new(AtomicBool::new(true)),
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence_and_scope(
|
|
ctx: &CancellationToken,
|
|
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
|
expected_publication_epoch: Option<u64>,
|
|
remote_lease_deadline: Option<std::time::Instant>,
|
|
scanner_publication_lease_fence: Option<&str>,
|
|
remote_lease_tokens: Vec<Uuid>,
|
|
lease_release_safe: Arc<AtomicBool>,
|
|
) -> Result<(), EcstoreError> {
|
|
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
|
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
|
|
if ctx.is_cancelled() {
|
|
return Ok(());
|
|
}
|
|
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
|
|
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
|
}
|
|
|
|
let read_epoch = match expected_publication_epoch {
|
|
Some(expected_epoch) => {
|
|
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
|
.await
|
|
.is_none()
|
|
{
|
|
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
|
}
|
|
expected_epoch
|
|
}
|
|
None => scanner_publication_epoch(storeapi.clone())
|
|
.await
|
|
.ok_or_else(|| EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED))?,
|
|
};
|
|
let (primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?;
|
|
let primary = primary.ok_or(EcstoreError::ConfigNotFound)?;
|
|
let primary_info = serde_json::from_slice::<DataUsageInfo>(&primary)
|
|
.map_err(|err| EcstoreError::other(format!("authoritative data usage snapshot is invalid: {err}")))?;
|
|
if !data_usage_info_has_persisted_baseline_identity(&primary_info) {
|
|
return Err(EcstoreError::other(
|
|
"authoritative data usage snapshot has no persisted baseline identity",
|
|
));
|
|
}
|
|
let primary = Bytes::from(primary);
|
|
|
|
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
|
|
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
|
}
|
|
|
|
let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?;
|
|
if backup.as_deref() == Some(primary.as_ref()) {
|
|
if scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
|
|
.await
|
|
.is_some()
|
|
{
|
|
return Ok(());
|
|
}
|
|
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
|
continue;
|
|
}
|
|
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
|
}
|
|
|
|
let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower));
|
|
let save_result = {
|
|
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
|
|
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
|
}
|
|
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
|
|
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
|
continue;
|
|
}
|
|
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
|
};
|
|
let publication_scope = match expected_publication_epoch {
|
|
Some(expected_epoch) => {
|
|
storeapi
|
|
.scanner_data_usage_publication_commit_scope_with_release_flag(
|
|
expected_epoch,
|
|
usage_store::scanner_publication_scope_deadline(data_usage_persist_timeout(), remote_lease_deadline),
|
|
remote_lease_tokens.clone(),
|
|
Arc::clone(&lease_release_safe),
|
|
)
|
|
.await
|
|
}
|
|
None => None,
|
|
};
|
|
if expected_publication_epoch.is_some() && publication_scope.is_none() {
|
|
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
|
continue;
|
|
}
|
|
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
|
}
|
|
let save_result = save_config_shared_with_preconditions_and_lease_fence_and_scope(
|
|
storeapi.clone(),
|
|
&backup_path,
|
|
primary.clone(),
|
|
sha256hex,
|
|
revision.preconditions(),
|
|
scanner_publication_lease_fence,
|
|
publication_scope.clone(),
|
|
)
|
|
.await;
|
|
if let Some(scope) = publication_scope {
|
|
match scope.wait_for_completion().await {
|
|
crate::storage_api::owner::ScannerPublicationCommitState::Committed
|
|
| crate::storage_api::owner::ScannerPublicationCommitState::AbortedBeforeCommit => save_result,
|
|
crate::storage_api::owner::ScannerPublicationCommitState::Indeterminate
|
|
| crate::storage_api::owner::ScannerPublicationCommitState::Admitted
|
|
| crate::storage_api::owner::ScannerPublicationCommitState::InFlight => Err(EcstoreError::other(
|
|
"scanner backup publication commit scope did not reach a safe terminal state",
|
|
)),
|
|
}
|
|
} else {
|
|
save_result
|
|
}
|
|
};
|
|
|
|
match save_result {
|
|
Ok(_) => {}
|
|
Err(err) => {
|
|
let (observed, _) = read_config_with_revision(storeapi.clone(), &backup_path).await?;
|
|
if observed.as_deref() == Some(primary.as_ref()) {
|
|
// The write committed even though the response was lost.
|
|
} else if err == EcstoreError::PreconditionFailed && retry < SCANNER_PERSIST_CAS_RETRIES {
|
|
continue;
|
|
} else {
|
|
return Err(err);
|
|
}
|
|
}
|
|
}
|
|
|
|
let (current_primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?;
|
|
if current_primary.as_deref() == Some(primary.as_ref())
|
|
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
|
|
.await
|
|
.is_some()
|
|
{
|
|
return Ok(());
|
|
}
|
|
if expected_publication_epoch.is_some()
|
|
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
|
|
.await
|
|
.is_none()
|
|
{
|
|
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
|
}
|
|
if retry < SCANNER_PERSIST_CAS_RETRIES {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
Err(EcstoreError::other(
|
|
"authoritative data usage snapshot changed while synchronizing its backup",
|
|
))
|
|
}
|
|
|
|
async fn persisted_usage_cache_is_cold_for_startup<S>(storeapi: &Arc<S>) -> bool
|
|
where
|
|
S: ScannerObjectIO + ScannerConfigObjectDelete,
|
|
{
|
|
let Some(data) = (match read_data_usage_config_for_startup(storeapi).await {
|
|
Ok(data) => data,
|
|
Err(err) => {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
|
state = "startup_inspect_failed",
|
|
error = %err,
|
|
"Scanner startup cache inspection failed"
|
|
);
|
|
return false;
|
|
}
|
|
}) else {
|
|
return true;
|
|
};
|
|
|
|
match serde_json::from_slice::<DataUsageInfo>(&data) {
|
|
Ok(info) => {
|
|
if data_usage_info_is_cold(&info) {
|
|
return true;
|
|
}
|
|
match read_config(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
|
|
Ok(observed) => match serde_json::from_slice::<DataUsageInfo>(&observed) {
|
|
Ok(observed) => usage_cache_needs_prompt_scan(&info, Some(&observed)),
|
|
Err(err) => {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
path = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
|
state = "startup_observed_decode_failed",
|
|
error = %err,
|
|
"Scanner startup found an invalid observational snapshot and will refresh it promptly"
|
|
);
|
|
true
|
|
}
|
|
},
|
|
Err(EcstoreError::ConfigNotFound) => false,
|
|
Err(err) => {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
path = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
|
state = "startup_observed_inspect_failed",
|
|
error = %err,
|
|
"Scanner startup could not inspect the observational snapshot"
|
|
);
|
|
false
|
|
}
|
|
}
|
|
}
|
|
Err(err) => {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
|
state = "startup_decode_failed",
|
|
error = %err,
|
|
"Scanner startup cache decode failed"
|
|
);
|
|
true
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn initial_scanner_startup_usage_state<S>(storeapi: &Arc<S>) -> (bool, bool)
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
let has_buckets = match storeapi
|
|
.list_bucket(&BucketOptions {
|
|
no_metadata: true,
|
|
..Default::default()
|
|
})
|
|
.await
|
|
{
|
|
Ok(buckets) => !buckets.is_empty(),
|
|
Err(err) => {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "startup_bucket_inspect_failed",
|
|
error = %err,
|
|
"Scanner startup bucket inspection failed"
|
|
);
|
|
false
|
|
}
|
|
};
|
|
|
|
(persisted_usage_cache_is_cold_for_startup(storeapi).await, has_buckets)
|
|
}
|
|
|
|
fn prepare_cycle_for_usage_floor_bootstrap(
|
|
cycle_info: &mut CurrentCycle,
|
|
usage_floor: PersistedUsageFloor,
|
|
startup: PersistedUsageFloorStartup,
|
|
) -> (bool, ScannerCycleResetPolicy) {
|
|
match startup {
|
|
PersistedUsageFloorStartup::Authoritative => (false, ScannerCycleResetPolicy::None),
|
|
PersistedUsageFloorStartup::Missing => {
|
|
// Cycle progress without its corresponding usage floor cannot
|
|
// prove namespace coverage. Restart from cycle zero while keeping
|
|
// the separately fenced leader epoch monotonic.
|
|
*cycle_info = CurrentCycle::default();
|
|
(true, ScannerCycleResetPolicy::ResetAll)
|
|
}
|
|
PersistedUsageFloorStartup::BootstrapPending => {
|
|
// An unfenced marker may have been written before an upgrade's old
|
|
// cycle state was replaced. A fenced marker belongs to the current
|
|
// scanner generation and may retain partial-cycle progress.
|
|
if usage_floor.leader_epoch == 0 {
|
|
*cycle_info = CurrentCycle::default();
|
|
}
|
|
(
|
|
true,
|
|
if usage_floor.leader_epoch == 0 {
|
|
ScannerCycleResetPolicy::ResetAll
|
|
} else {
|
|
ScannerCycleResetPolicy::None
|
|
},
|
|
)
|
|
}
|
|
PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence => {
|
|
// The legacy incomplete fence proves only its leader epoch, not
|
|
// namespace coverage. Clear coverage while retaining the durable
|
|
// cycle number so surviving caches cannot force a regression.
|
|
let next = cycle_info.next;
|
|
*cycle_info = CurrentCycle {
|
|
next,
|
|
..Default::default()
|
|
};
|
|
(true, ScannerCycleResetPolicy::ResetCoveragePreservingNext)
|
|
}
|
|
}
|
|
}
|
|
|
|
pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
|
init_data_scanner_with_storage(ctx, storeapi).await;
|
|
}
|
|
|
|
/// Start normal scanning when enabled, or one resume-only cleanup attempt.
|
|
/// The disabled branch returns a finite task for the startup owner to join;
|
|
/// it never enables ordinary namespace scanning or accepts a new reset intent.
|
|
pub async fn init_scanner_with_recovery(
|
|
ctx: CancellationToken,
|
|
storeapi: Arc<ECStore>,
|
|
enabled: bool,
|
|
) -> Option<tokio::task::JoinHandle<()>> {
|
|
if enabled {
|
|
init_data_scanner(ctx, storeapi).await;
|
|
return None;
|
|
}
|
|
Some(tokio::spawn(async move {
|
|
if let Err(error) = resume_scanner_cycle_cleanup(ctx, storeapi).await {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "disabled_cleanup_deferred",
|
|
error = %error,
|
|
"Disabled scanner cleanup remains pending for an operator retry"
|
|
);
|
|
}
|
|
}))
|
|
}
|
|
|
|
async fn init_data_scanner_with_storage<S>(ctx: CancellationToken, storeapi: Arc<S>)
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
let (startup_features, startup_maintenance_generation) = configure_scanner_defaults(&ctx, &storeapi).await;
|
|
// Force init global sleeper so config is read once at startup.
|
|
let _ = &*SCANNER_SLEEPER;
|
|
if let Err(err) = refresh_scanner_runtime_config_from_global() {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "startup_apply_failed",
|
|
error = %err,
|
|
"Scanner runtime config apply failed at startup"
|
|
);
|
|
}
|
|
|
|
let replication_active = startup_features.replication;
|
|
let ctx_clone = ctx;
|
|
let storeapi_clone = storeapi;
|
|
let runtime_guard = ScannerRuntimeGuard::new();
|
|
tokio::spawn(async move {
|
|
let _runtime_guard = runtime_guard;
|
|
let (usage_cache_is_cold, has_buckets) = initial_scanner_startup_usage_state(&storeapi_clone).await;
|
|
let sleep_time = initial_scanner_delay_for_startup(
|
|
scanner_start_delay().map(|duration| duration.as_secs()),
|
|
usage_cache_is_cold,
|
|
has_buckets,
|
|
replication_active,
|
|
);
|
|
if sleep_time.is_zero() {
|
|
let skip_reason = if usage_cache_is_cold {
|
|
"usage_cache_cold"
|
|
} else {
|
|
"replication_active"
|
|
};
|
|
info!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "startup_delay_skipped",
|
|
reason = skip_reason,
|
|
"Scanner startup delay skipped"
|
|
);
|
|
} else {
|
|
tokio::time::sleep(sleep_time).await;
|
|
}
|
|
|
|
let mut transient_backoff = ScannerRetryBackoff::default();
|
|
let mut recovery_retry_count = 0_u32;
|
|
loop {
|
|
if ctx_clone.is_cancelled() {
|
|
break;
|
|
}
|
|
|
|
let run_result = run_data_scanner_with_maintenance_state(
|
|
ctx_clone.clone(),
|
|
storeapi_clone.clone(),
|
|
startup_features,
|
|
startup_maintenance_generation,
|
|
)
|
|
.await;
|
|
if let Err(e) = &run_result {
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "run_failed",
|
|
error = %e,
|
|
"Scanner runtime iteration failed"
|
|
);
|
|
}
|
|
let recovery_status = scanner_cycle_recovery_status();
|
|
if recovery_status.retryable {
|
|
recovery_retry_count = recovery_retry_count.saturating_add(1);
|
|
let _ = record_scanner_cycle_recovery_retry(recovery_retry_count);
|
|
} else {
|
|
recovery_retry_count = 0;
|
|
}
|
|
|
|
let recovery_status = scanner_cycle_recovery_status();
|
|
if recovery_status.state == "paused" {
|
|
transient_backoff.record_retryable_cycle(false);
|
|
tokio::select! {
|
|
_ = ctx_clone.cancelled() => break,
|
|
_ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
|
|
_ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {},
|
|
}
|
|
recovery_retry_count = 0;
|
|
continue;
|
|
}
|
|
if !recovery_status.retryable
|
|
&& matches!(recovery_status.state.as_str(), "blocked" | "recovery-required" | "cleanup-pending")
|
|
{
|
|
transient_backoff.record_retryable_cycle(false);
|
|
tokio::select! {
|
|
_ = ctx_clone.cancelled() => break,
|
|
_ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
|
|
_ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL) => {},
|
|
}
|
|
continue;
|
|
}
|
|
|
|
let retry_delay = if recovery_status.retryable || run_result.is_err() {
|
|
transient_backoff.record_retryable_cycle(true);
|
|
transient_backoff
|
|
.retry_interval(scanner_cycle_interval())
|
|
.unwrap_or(SCANNER_RETRY_BASE_INTERVAL)
|
|
} else {
|
|
transient_backoff.record_retryable_cycle(false);
|
|
randomized_cycle_delay()
|
|
};
|
|
// Backoff before retrying after lock contention or scanner-level failures.
|
|
// Keep this cancellation-aware so shutdown is not delayed by backoff sleep.
|
|
tokio::select! {
|
|
_ = ctx_clone.cancelled() => break,
|
|
_ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
|
|
_ = tokio::time::sleep(retry_delay) => {}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
|
struct ScannerMaintenanceFeatures {
|
|
lifecycle: bool,
|
|
replication: bool,
|
|
inspection_failed: bool,
|
|
}
|
|
|
|
impl ScannerMaintenanceFeatures {
|
|
fn needs_regular_cycle(self) -> bool {
|
|
self.lifecycle || self.replication || self.inspection_failed
|
|
}
|
|
|
|
fn requires_full_scan(self, observed_generation: Option<u64>, current_generation: u64, wake: ScannerCycleWakeReason) -> bool {
|
|
self.needs_regular_cycle()
|
|
|| observed_generation != Some(current_generation)
|
|
|| !matches!(wake, ScannerCycleWakeReason::DirtyUsage | ScannerCycleWakeReason::ClusterActivity)
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
enum MaintenanceInspectionDecision {
|
|
Accept,
|
|
Retry,
|
|
PreserveBaseCycle,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
enum MaintenanceInspectionAttempt {
|
|
Completed(ScannerMaintenanceFeatures),
|
|
TimedOut,
|
|
Cancelled,
|
|
}
|
|
|
|
async fn wait_for_maintenance_feature_inspection<F>(
|
|
ctx: &CancellationToken,
|
|
inspection: F,
|
|
timeout: Duration,
|
|
) -> MaintenanceInspectionAttempt
|
|
where
|
|
F: Future<Output = ScannerMaintenanceFeatures>,
|
|
{
|
|
tokio::select! {
|
|
_ = ctx.cancelled() => MaintenanceInspectionAttempt::Cancelled,
|
|
result = tokio::time::timeout(timeout, inspection) => match result {
|
|
Ok(features) => MaintenanceInspectionAttempt::Completed(features),
|
|
Err(_) => MaintenanceInspectionAttempt::TimedOut,
|
|
},
|
|
}
|
|
}
|
|
|
|
fn maintenance_inspection_decision(generation: u64, current_generation: u64, attempts: usize) -> MaintenanceInspectionDecision {
|
|
if generation == current_generation {
|
|
MaintenanceInspectionDecision::Accept
|
|
} else if attempts < MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS {
|
|
MaintenanceInspectionDecision::Retry
|
|
} else {
|
|
MaintenanceInspectionDecision::PreserveBaseCycle
|
|
}
|
|
}
|
|
|
|
fn single_disk_default_speed() -> ScannerSpeed {
|
|
ScannerSpeed::Default
|
|
}
|
|
|
|
async fn detect_scanner_maintenance_features<S>(storeapi: &Arc<S>) -> ScannerMaintenanceFeatures
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
let mut features = ScannerMaintenanceFeatures::default();
|
|
let buckets = match storeapi
|
|
.list_bucket(&BucketOptions {
|
|
no_metadata: true,
|
|
..Default::default()
|
|
})
|
|
.await
|
|
{
|
|
Ok(buckets) => buckets,
|
|
Err(err) => {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "maintenance_feature_inspect_failed",
|
|
error = %err,
|
|
"Scanner maintenance feature inspection failed; preserving speed-based cycle"
|
|
);
|
|
features.inspection_failed = true;
|
|
return features;
|
|
}
|
|
};
|
|
|
|
for bucket in buckets {
|
|
if !features.lifecycle {
|
|
match get_lifecycle_config(&bucket.name).await {
|
|
Ok((lifecycle, _)) => {
|
|
features.lifecycle = lifecycle.has_active_rules("");
|
|
}
|
|
Err(EcstoreError::ConfigNotFound) => {}
|
|
Err(err) => {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
bucket = %bucket.name,
|
|
state = "lifecycle_inspect_failed",
|
|
error = %err,
|
|
"Scanner lifecycle inspection failed; preserving speed-based cycle"
|
|
);
|
|
features.inspection_failed = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
if !features.replication {
|
|
match get_replication_config(&bucket.name).await {
|
|
Ok((replication, _)) => {
|
|
features.replication = replication.has_active_rules("", true);
|
|
}
|
|
Err(EcstoreError::ConfigNotFound) => {}
|
|
Err(err) => {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
bucket = %bucket.name,
|
|
state = "replication_inspect_failed",
|
|
error = %err,
|
|
"Scanner replication inspection failed; preserving speed-based cycle"
|
|
);
|
|
features.inspection_failed = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
if features.needs_regular_cycle() {
|
|
break;
|
|
}
|
|
}
|
|
|
|
features
|
|
}
|
|
|
|
async fn detect_stable_scanner_maintenance_features(
|
|
ctx: &CancellationToken,
|
|
storeapi: &Arc<impl ScannerStorage>,
|
|
) -> Option<(ScannerMaintenanceFeatures, u64)> {
|
|
detect_stable_scanner_maintenance_features_with(
|
|
ctx,
|
|
|| detect_scanner_maintenance_features(storeapi),
|
|
MAINTENANCE_FEATURE_INSPECTION_TIMEOUT,
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn detect_stable_scanner_maintenance_features_with<F, Fut>(
|
|
ctx: &CancellationToken,
|
|
mut inspect: F,
|
|
timeout: Duration,
|
|
) -> Option<(ScannerMaintenanceFeatures, u64)>
|
|
where
|
|
F: FnMut() -> Fut,
|
|
Fut: Future<Output = ScannerMaintenanceFeatures>,
|
|
{
|
|
let mut attempts = 0usize;
|
|
loop {
|
|
attempts += 1;
|
|
let generation = scanner_maintenance_generation();
|
|
let mut features = match wait_for_maintenance_feature_inspection(ctx, inspect(), timeout).await {
|
|
MaintenanceInspectionAttempt::Completed(features) => features,
|
|
MaintenanceInspectionAttempt::Cancelled => return None,
|
|
MaintenanceInspectionAttempt::TimedOut => {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
timeout = ?timeout,
|
|
state = "maintenance_feature_inspection_timed_out",
|
|
"Scanner maintenance feature inspection timed out; preserving the base cycle"
|
|
);
|
|
ScannerMaintenanceFeatures {
|
|
inspection_failed: true,
|
|
..Default::default()
|
|
}
|
|
}
|
|
};
|
|
let current_generation = scanner_maintenance_generation();
|
|
match maintenance_inspection_decision(generation, current_generation, attempts) {
|
|
MaintenanceInspectionDecision::Accept => return Some((features, current_generation)),
|
|
MaintenanceInspectionDecision::Retry => {}
|
|
MaintenanceInspectionDecision::PreserveBaseCycle => {
|
|
features.inspection_failed = true;
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
attempts = MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS,
|
|
state = "maintenance_feature_inspection_unstable",
|
|
"Scanner maintenance configuration changed repeatedly during inspection; preserving the base cycle"
|
|
);
|
|
return Some((features, current_generation));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn configure_scanner_defaults(
|
|
ctx: &CancellationToken,
|
|
storeapi: &Arc<impl ScannerStorage>,
|
|
) -> (ScannerMaintenanceFeatures, Option<u64>) {
|
|
let (features, maintenance_generation) = detect_stable_scanner_maintenance_features(ctx, storeapi)
|
|
.await
|
|
.unwrap_or_else(|| {
|
|
(
|
|
ScannerMaintenanceFeatures {
|
|
inspection_failed: true,
|
|
..Default::default()
|
|
},
|
|
scanner_maintenance_generation(),
|
|
)
|
|
});
|
|
if storeapi.setup_is_erasure_sd().await {
|
|
// Single-disk keeps the speed-preset-derived default cycle (60s at the
|
|
// `default` preset) instead of a special shorter cycle: no measured
|
|
// cold-start ILM latency basis for an override, and clean-idle backoff
|
|
// already stretches idle cadence. Decision record: backlog#1878 (HS-16).
|
|
set_scanner_default_speed(single_disk_default_speed());
|
|
set_scanner_default_cycle_secs(None);
|
|
info!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
env_speed = ENV_SCANNER_SPEED,
|
|
env_cycle = ENV_SCANNER_CYCLE,
|
|
env_start_delay = ENV_SCANNER_START_DELAY_SECS,
|
|
lifecycle_active = features.lifecycle,
|
|
replication_active = features.replication,
|
|
feature_inspection_failed = features.inspection_failed,
|
|
state = "single_disk_defaults_applied",
|
|
"Scanner defaults applied"
|
|
);
|
|
(features, Some(maintenance_generation))
|
|
} else {
|
|
set_scanner_default_speed(ScannerSpeed::Default);
|
|
set_scanner_default_cycle_secs(None);
|
|
(features, Some(maintenance_generation))
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn bitrot_scan_cycle() -> Option<Duration> {
|
|
resolve_scanner_runtime_config().bitrot_cycle
|
|
}
|
|
|
|
fn get_cycle_scan_mode(
|
|
current_cycle: u64,
|
|
bitrot_start_cycle: u64,
|
|
bitrot_start_time: Option<DateTime<Utc>>,
|
|
bitrot_cycle: Option<Duration>,
|
|
) -> HealScanMode {
|
|
let Some(bitrot_cycle) = bitrot_cycle else {
|
|
return HealScanMode::Normal;
|
|
};
|
|
|
|
if bitrot_cycle.is_zero() {
|
|
return HealScanMode::Deep;
|
|
}
|
|
|
|
if current_cycle.saturating_sub(bitrot_start_cycle) < heal_object_select_prob() as u64 {
|
|
return HealScanMode::Deep;
|
|
}
|
|
|
|
let Some(bitrot_start_time) = bitrot_start_time else {
|
|
return HealScanMode::Deep;
|
|
};
|
|
|
|
let elapsed = Utc::now()
|
|
.signed_duration_since(bitrot_start_time)
|
|
.to_std()
|
|
.unwrap_or(Duration::ZERO);
|
|
if elapsed >= bitrot_cycle {
|
|
HealScanMode::Deep
|
|
} else {
|
|
HealScanMode::Normal
|
|
}
|
|
}
|
|
|
|
fn background_heal_info_for_scan_start(
|
|
mut info: BackgroundHealInfo,
|
|
current_cycle: u64,
|
|
scan_mode: HealScanMode,
|
|
now: DateTime<Utc>,
|
|
bitrot_cycle: Option<Duration>,
|
|
) -> Option<BackgroundHealInfo> {
|
|
let reset_bitrot_start =
|
|
scan_mode == HealScanMode::Deep && should_reset_bitrot_start(&info, current_cycle, now, bitrot_cycle);
|
|
if info.current_scan_mode == scan_mode && !reset_bitrot_start {
|
|
return None;
|
|
}
|
|
|
|
info.current_scan_mode = scan_mode;
|
|
if reset_bitrot_start {
|
|
info.bitrot_start_cycle = current_cycle;
|
|
info.bitrot_start_time = Some(now);
|
|
}
|
|
|
|
Some(info)
|
|
}
|
|
|
|
fn should_reset_bitrot_start(
|
|
info: &BackgroundHealInfo,
|
|
current_cycle: u64,
|
|
now: DateTime<Utc>,
|
|
bitrot_cycle: Option<Duration>,
|
|
) -> bool {
|
|
let Some(bitrot_start_time) = info.bitrot_start_time else {
|
|
return true;
|
|
};
|
|
|
|
let Some(bitrot_cycle) = bitrot_cycle else {
|
|
return false;
|
|
};
|
|
|
|
if bitrot_cycle.is_zero() {
|
|
return true;
|
|
}
|
|
|
|
if current_cycle.saturating_sub(info.bitrot_start_cycle) < heal_object_select_prob() as u64 {
|
|
return false;
|
|
}
|
|
|
|
let elapsed = now
|
|
.signed_duration_since(bitrot_start_time)
|
|
.to_std()
|
|
.unwrap_or(Duration::ZERO);
|
|
elapsed >= bitrot_cycle
|
|
}
|
|
|
|
fn background_heal_info_for_scan_complete(mut info: BackgroundHealInfo, scan_mode: HealScanMode) -> Option<BackgroundHealInfo> {
|
|
if scan_mode != HealScanMode::Deep || info.current_scan_mode != HealScanMode::Deep {
|
|
return None;
|
|
}
|
|
|
|
info.current_scan_mode = HealScanMode::Normal;
|
|
Some(info)
|
|
}
|
|
|
|
fn background_heal_info_for_scan_result(
|
|
info: BackgroundHealInfo,
|
|
scan_mode: HealScanMode,
|
|
success: bool,
|
|
) -> Option<BackgroundHealInfo> {
|
|
if !success {
|
|
return None;
|
|
}
|
|
|
|
background_heal_info_for_scan_complete(info, scan_mode)
|
|
}
|
|
|
|
fn retain_recent_cycle_completions(cycle_completed: &mut Vec<DateTime<Utc>>) {
|
|
let keep = data_usage_update_dir_cycles() as usize;
|
|
if cycle_completed.len() > keep {
|
|
let drop_count = cycle_completed.len() - keep;
|
|
cycle_completed.drain(..drop_count);
|
|
}
|
|
}
|
|
|
|
/// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds)
|
|
/// Defaults to 5 seconds if not set or invalid
|
|
/// For distributed environments with multiple nodes, a longer timeout may be needed
|
|
fn get_lock_acquire_timeout() -> Duration {
|
|
Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
|
|
}
|
|
|
|
fn data_usage_persist_timeout() -> Duration {
|
|
DataUsageCache::persistence_timeout()
|
|
}
|
|
|
|
#[cfg(not(test))]
|
|
const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_secs(30);
|
|
#[cfg(test)]
|
|
const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_millis(50);
|
|
|
|
async fn fence_scanner_epoch_after_cycle_timeout<Store, LockLost>(
|
|
ctx: &CancellationToken,
|
|
storeapi: Arc<Store>,
|
|
cycle_info: &mut CurrentCycle,
|
|
cycle_revision: &mut DataUsageCacheRevision,
|
|
leader_epoch: &mut u64,
|
|
allow_bootstrap_pending: bool,
|
|
lock_lost: LockLost,
|
|
) -> bool
|
|
where
|
|
Store: ScannerObjectIO + ScannerConfigObjectDelete,
|
|
LockLost: Future<Output = ()>,
|
|
{
|
|
let fence_ctx = ctx.child_token();
|
|
let claim = claim_scanner_leadership(
|
|
&fence_ctx,
|
|
storeapi,
|
|
cycle_info,
|
|
cycle_revision,
|
|
leader_epoch,
|
|
allow_bootstrap_pending,
|
|
ScannerCycleResetPolicy::None,
|
|
);
|
|
tokio::pin!(claim);
|
|
tokio::pin!(lock_lost);
|
|
tokio::select! {
|
|
biased;
|
|
_ = &mut lock_lost => {
|
|
fence_ctx.cancel();
|
|
false
|
|
}
|
|
result = tokio::time::timeout(SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT, &mut claim) => {
|
|
result.unwrap_or(false) && !fence_ctx.is_cancelled()
|
|
}
|
|
}
|
|
}
|
|
|
|
struct ScannerCycleDeadlineState<'a> {
|
|
cycle_info: &'a mut CurrentCycle,
|
|
cycle_revision: &'a mut DataUsageCacheRevision,
|
|
leader_epoch: &'a mut u64,
|
|
cycle_budget: &'a ScannerCycleBudget,
|
|
allow_bootstrap_pending: bool,
|
|
}
|
|
|
|
fn cycle_timeout_requires_recovery(worker_stopped: bool, cycle_state_persisted: bool, generation_fenced: bool) -> bool {
|
|
!worker_stopped || !cycle_state_persisted || !generation_fenced
|
|
}
|
|
|
|
async fn handle_scanner_cycle_deadline<Store>(
|
|
ctx: &CancellationToken,
|
|
storeapi: Arc<Store>,
|
|
state: ScannerCycleDeadlineState<'_>,
|
|
worker_stopped: bool,
|
|
guard: &mut NamespaceLockGuard,
|
|
) where
|
|
Store: ScannerObjectIO + ScannerConfigObjectDelete,
|
|
{
|
|
let fenced = fence_scanner_epoch_after_cycle_timeout(
|
|
ctx,
|
|
storeapi,
|
|
state.cycle_info,
|
|
state.cycle_revision,
|
|
state.leader_epoch,
|
|
state.allow_bootstrap_pending,
|
|
guard.lock_lost_notified(),
|
|
)
|
|
.await;
|
|
let cycle_state_persisted = state.cycle_budget.cycle_state_persisted();
|
|
let recovery_required = cycle_timeout_requires_recovery(worker_stopped, cycle_state_persisted, fenced);
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "cycle_timeout",
|
|
worker_stopped,
|
|
cycle_state_persisted,
|
|
generation_fenced = fenced,
|
|
recovery_required,
|
|
"Scanner cycle deadline expired; durable cursor/generation fencing completed when possible"
|
|
);
|
|
global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age());
|
|
// Stop renewing before releasing the lease. A new leader can then claim the
|
|
// higher persisted generation instead of inheriting the expired worker.
|
|
guard.release();
|
|
global_metrics().set_cycle(None).await;
|
|
}
|
|
|
|
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
|
|
cycle_info.current = 0;
|
|
global_metrics().clear_current_scan_mode();
|
|
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
|
}
|
|
|
|
#[cfg(test)]
|
|
async fn run_data_scanner_cycle<S>(
|
|
ctx: &CancellationToken,
|
|
storeapi: &Arc<S>,
|
|
cycle_info: &mut CurrentCycle,
|
|
cycle_revision: &mut DataUsageCacheRevision,
|
|
leader_epoch: u64,
|
|
) -> ScannerCycleOutcome
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
let cycle_budget = ScannerCycleBudget::new(ctx, scanner_cycle_budget_config());
|
|
run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget, true).await
|
|
}
|
|
|
|
#[instrument(skip_all)]
|
|
#[hotpath::measure]
|
|
async fn run_data_scanner_cycle_with_budget<S>(
|
|
ctx: &CancellationToken,
|
|
storeapi: &Arc<S>,
|
|
cycle_info: &mut CurrentCycle,
|
|
cycle_revision: &mut DataUsageCacheRevision,
|
|
leader_epoch: u64,
|
|
cycle_budget: Arc<ScannerCycleBudget>,
|
|
requires_full_scan: bool,
|
|
) -> ScannerCycleOutcome
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
let _activity_guard = ScannerActivityGuard::new();
|
|
if let Err(err) = refresh_scanner_runtime_config_from_global() {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_RUNTIME_CONFIG,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "refresh_failed",
|
|
error = %err,
|
|
"Scanner runtime config refresh failed"
|
|
);
|
|
}
|
|
let configured_cycle_interval = scanner_cycle_interval();
|
|
let configured_bitrot_cycle = scanner_bitrot_cycle();
|
|
let cycle_budget_config = ScannerCycleBudgetConfig {
|
|
max_duration: cycle_budget.max_duration(),
|
|
max_objects: cycle_budget.max_objects(),
|
|
max_directories: cycle_budget.max_directories(),
|
|
};
|
|
let usage_persist_timeout = data_usage_persist_timeout();
|
|
global_metrics().record_scanner_cycle_config(
|
|
configured_cycle_interval,
|
|
configured_bitrot_cycle,
|
|
cycle_budget_config.max_duration,
|
|
cycle_budget_config.max_objects,
|
|
cycle_budget_config.max_directories,
|
|
);
|
|
cycle_info.current = cycle_info.next;
|
|
let now = Instant::now();
|
|
cycle_info.started = Utc::now();
|
|
|
|
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
|
|
|
|
// Refresh the storage-owned movement snapshot before reading background
|
|
// heal state. A missing heal object yields an in-memory default; do not
|
|
// let that default influence a cycle while publication is blocked.
|
|
if storeapi.scanner_data_movement_pause_status().await.paused {
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
|
}
|
|
let background_heal_read = read_background_heal_info_with_epoch(storeapi.clone()).await;
|
|
match background_heal_read.status {
|
|
BackgroundHealInfoReadStatus::Blocked => {
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
|
}
|
|
BackgroundHealInfoReadStatus::Transient => {
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
|
}
|
|
BackgroundHealInfoReadStatus::Failed => {
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Failed;
|
|
}
|
|
BackgroundHealInfoReadStatus::ErasureSd
|
|
| BackgroundHealInfoReadStatus::Loaded
|
|
| BackgroundHealInfoReadStatus::Missing => {}
|
|
}
|
|
let mut background_heal_info = background_heal_read.info;
|
|
let background_heal_epoch = background_heal_read.expected_epoch;
|
|
|
|
let scan_mode = get_cycle_scan_mode(
|
|
cycle_info.current,
|
|
background_heal_info.bitrot_start_cycle,
|
|
background_heal_info.bitrot_start_time,
|
|
configured_bitrot_cycle,
|
|
);
|
|
info!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
scan_mode = ?scan_mode,
|
|
state = "started",
|
|
"Scanner cycle started"
|
|
);
|
|
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
|
|
if let Some(new_heal_info) = background_heal_info_for_scan_start(
|
|
background_heal_info.clone(),
|
|
cycle_info.current,
|
|
scan_mode,
|
|
Utc::now(),
|
|
configured_bitrot_cycle,
|
|
) {
|
|
background_heal_info = new_heal_info.clone();
|
|
save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
|
|
}
|
|
|
|
let cycle_start = std::time::Instant::now();
|
|
// Baseline reads are part of the same publication proof as the eventual
|
|
// scanner aggregate. Hold only the short storage-owned admission guard
|
|
// across this metadata read; the full bucket scan runs after it is
|
|
// released and carries the captured epoch forward.
|
|
let Some(baseline_publication_guard) = storeapi.scanner_data_usage_publication_admission().await else {
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
|
};
|
|
let baseline_publication_epoch = baseline_publication_guard.epoch();
|
|
let usage_persist_baseline_result = read_data_usage_persist_baseline(storeapi.clone()).await;
|
|
drop(baseline_publication_guard);
|
|
let usage_persist_baseline = match usage_persist_baseline_result {
|
|
Ok(baseline) => baseline,
|
|
Err(err) => {
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
|
state = "usage_baseline_load_failed",
|
|
error = %err,
|
|
"Scanner cycle could not capture the data usage persistence baseline"
|
|
);
|
|
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Failed;
|
|
}
|
|
};
|
|
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
|
|
|
|
let done_cycle = Metrics::time(Metric::ScanCycle);
|
|
let scan_result = crate::scanner_io::nsscanner_with_storage_status_scoped(
|
|
storeapi.as_ref(),
|
|
crate::scanner_io::ScannerCycleRequest {
|
|
ctx: cycle_budget.token(),
|
|
budget: cycle_budget.clone(),
|
|
updates: sender,
|
|
want_cycle: cycle_info.current,
|
|
leader_epoch,
|
|
scan_mode,
|
|
scan_scope: crate::scanner_io::ScannerBucketScanScope::default(),
|
|
persisted_usage_baseline: usage_persist_baseline.data.clone(),
|
|
requires_full_scan,
|
|
#[cfg(test)]
|
|
resolved_scope_observer: None,
|
|
},
|
|
)
|
|
.await;
|
|
let publication_defer_reason = match &scan_result {
|
|
Ok(result)
|
|
if result
|
|
.publication_epoch()
|
|
.is_some_and(|publication_epoch| publication_epoch != baseline_publication_epoch) =>
|
|
{
|
|
Some(ScannerCycleDeferReason::DataMovement)
|
|
}
|
|
// A complete walk can still be retained as an observational snapshot
|
|
// when only the final activity proof was unavailable. It must not
|
|
// block the observation receiver: the authoritative publication
|
|
// fence remains enforced by the usage store and the cycle is advanced
|
|
// as partial without acknowledging dirty usage.
|
|
Ok(result)
|
|
if result.has_observational_snapshot()
|
|
&& matches!(
|
|
result.status,
|
|
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
|
) =>
|
|
{
|
|
None
|
|
}
|
|
Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await,
|
|
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
|
};
|
|
let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch);
|
|
let remote_publication_lease_targets = if publication_defer_reason.is_none() {
|
|
scan_result
|
|
.as_ref()
|
|
.ok()
|
|
.map(|result| result.remote_publication_lease_targets().to_vec())
|
|
.unwrap_or_default()
|
|
} else {
|
|
Vec::new()
|
|
};
|
|
let mut remote_publication_leases = None;
|
|
let remote_lease_defer_reason = if remote_publication_lease_targets.is_empty() {
|
|
None
|
|
} else if let Some(notification_system) = storeapi.scanner_notification_system() {
|
|
let publication_proof_ctx = cycle_budget.token();
|
|
let lease_result = await_scanner_publication_proof(
|
|
&publication_proof_ctx,
|
|
cycle_info.current,
|
|
"lease_acquire",
|
|
|| notification_system.acquire_scanner_publication_leases(remote_publication_lease_targets.clone()),
|
|
|err| scanner_publication_lease_error_is_retryable(&err.to_string()),
|
|
)
|
|
.await;
|
|
match lease_result {
|
|
ScannerPublicationProofWait::Ready(grants) => {
|
|
remote_publication_leases = Some((notification_system, grants));
|
|
None
|
|
}
|
|
ScannerPublicationProofWait::Rejected(_) | ScannerPublicationProofWait::Cancelled => {
|
|
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
|
}
|
|
}
|
|
} else {
|
|
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
|
};
|
|
let remote_lease_deadline = remote_publication_leases
|
|
.as_ref()
|
|
.and_then(|(_, grants)| grants.iter().map(|grant| grant.lease.expires_at).min());
|
|
// The transient fence is carried only to the SetDisks rename boundary;
|
|
// it is never inserted into FileInfo metadata. Keep the representation
|
|
// bounded and require one authenticated token per remote target so a
|
|
// partial grant can never silently fall back to an unfenced rename.
|
|
let remote_lease_fence = remote_publication_leases.as_ref().and_then(|(_, grants)| {
|
|
if grants.len() != remote_publication_lease_targets.len() || grants.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES {
|
|
return None;
|
|
}
|
|
let mut fence = BTreeMap::new();
|
|
for grant in grants {
|
|
if grant.host.is_empty() || grant.host.len() > 1024 {
|
|
return None;
|
|
}
|
|
if fence.insert(grant.host.clone(), grant.lease.token.to_string()).is_some() {
|
|
return None;
|
|
}
|
|
}
|
|
if remote_publication_lease_targets
|
|
.iter()
|
|
.any(|(host, _, _)| !fence.contains_key(host))
|
|
{
|
|
return None;
|
|
}
|
|
serde_json::to_string(&fence)
|
|
.ok()
|
|
.filter(|encoded| encoded.len() <= SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES)
|
|
});
|
|
let remote_lease_fence_defer_reason = (remote_publication_lease_fence_targets_are_required(
|
|
remote_publication_lease_targets.len(),
|
|
remote_publication_leases.is_some(),
|
|
remote_lease_fence.is_some(),
|
|
))
|
|
.then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
|
let publication_defer_reason = publication_defer_reason
|
|
.or(remote_lease_defer_reason)
|
|
.or(remote_lease_fence_defer_reason);
|
|
// A PUT tail can finish between the walk and lease acquisition without
|
|
// changing the movement epoch accepted by those leases. Re-prove the
|
|
// namespace baseline only after every peer has granted publication.
|
|
let post_lease_activity_defer_reason = if publication_defer_reason.is_none()
|
|
&& remote_publication_leases.is_some()
|
|
&& let Ok(result) = &scan_result
|
|
&& result.status == ScannerCycleStatus::Complete
|
|
{
|
|
scanner_post_lease_activity_defer_reason(result.activity_digest(), probe_scanner_activity(storeapi.as_ref(), true).await)
|
|
} else {
|
|
None
|
|
};
|
|
let publication_defer_reason = publication_defer_reason.or(post_lease_activity_defer_reason);
|
|
// Include reasons discovered while acquiring or validating remote leases.
|
|
let publication_deferred = publication_defer_reason.is_some();
|
|
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
|
|
let remote_lease_probe = remote_publication_leases
|
|
.as_ref()
|
|
.map(|(notification_system, grants)| (Arc::clone(notification_system), grants.clone()));
|
|
let remote_lease_release_safe = Arc::new(AtomicBool::new(true));
|
|
let mut usage_persist_outcome = match publication_defer_reason {
|
|
Some(reason) => {
|
|
drop(receiver);
|
|
DataUsagePersistOutcome::Deferred(reason)
|
|
}
|
|
None => {
|
|
// ScannerIO emits its complete or observational update only after
|
|
// all set workers finish. Persist after the final activity fence;
|
|
// this also avoids blocking the scanner on a denied publication.
|
|
let storeapi_clone = storeapi.clone();
|
|
let ctx_clone = ctx.clone();
|
|
let route_probe_store = storeapi.clone();
|
|
let remote_lease_fence = remote_lease_fence.clone();
|
|
let remote_lease_release_safe_for_task = Arc::clone(&remote_lease_release_safe);
|
|
let remote_lease_tokens = remote_publication_leases
|
|
.as_ref()
|
|
.map(|(_, grants)| grants.iter().map(|grant| grant.lease.token).collect())
|
|
.unwrap_or_default();
|
|
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
|
|
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
|
|
ctx_clone,
|
|
storeapi_clone,
|
|
receiver,
|
|
Some(leader_epoch),
|
|
Some(usage_persist_baseline),
|
|
ScannerPublicationFence::new(
|
|
publication_epoch,
|
|
remote_lease_deadline,
|
|
remote_lease_fence,
|
|
)
|
|
.with_remote_lease_tokens(remote_lease_tokens)
|
|
.with_lease_release_flag(remote_lease_release_safe_for_task),
|
|
move || {
|
|
let storeapi = route_probe_store.clone();
|
|
let remote_lease_probe = remote_lease_probe.clone();
|
|
async move {
|
|
if let Some((notification_system, grants)) = remote_lease_probe.as_ref()
|
|
&& notification_system.validate_scanner_publication_leases(grants).await.is_err()
|
|
{
|
|
// A remote restart or movement flip invalidates
|
|
// the token proof; usage_store interprets this
|
|
// as a publication barrier and performs no PUT.
|
|
return Some(ScannerCycleDeferReason::DataMovement);
|
|
}
|
|
scanner_local_publication_defer_reason(storeapi.as_ref()).await
|
|
}
|
|
},
|
|
)
|
|
.await
|
|
}));
|
|
match wait_for_data_usage_persist_task(ctx, &mut usage_persist_task, usage_persist_timeout).await {
|
|
DataUsagePersistTaskResult::Completed(outcome) => outcome,
|
|
DataUsagePersistTaskResult::JoinFailed(err) => {
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
state = "usage_persist_task_failed",
|
|
error = %err,
|
|
"Scanner data usage persistence task failed"
|
|
);
|
|
DataUsagePersistOutcome::Failed
|
|
}
|
|
DataUsagePersistTaskResult::Cancelled => {
|
|
debug!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
state = "usage_persist_task_cancelled",
|
|
"Scanner data usage persistence task cancelled"
|
|
);
|
|
DataUsagePersistOutcome::Failed
|
|
}
|
|
DataUsagePersistTaskResult::TimedOut => {
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
timeout = ?usage_persist_timeout,
|
|
state = "usage_persist_task_timed_out",
|
|
"Scanner data usage persistence task timed out"
|
|
);
|
|
DataUsagePersistOutcome::Failed
|
|
}
|
|
}
|
|
}
|
|
};
|
|
let lease_expired = remote_publication_leases
|
|
.as_ref()
|
|
.is_some_and(|(_, grants)| grants.iter().any(|grant| !grant.lease.is_valid()));
|
|
if !remote_lease_release_safe.load(Ordering::Acquire) {
|
|
// A cancelled or detached storage mutation did not report a safe
|
|
// terminal state. Keep remote grants until their own expiry rather
|
|
// than releasing movement admission while a commit may be unknown.
|
|
usage_persist_outcome = if usage_persist_outcome == DataUsagePersistOutcome::Failed {
|
|
DataUsagePersistOutcome::Failed
|
|
} else {
|
|
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded)
|
|
};
|
|
} else if let Some((notification_system, grants)) = remote_publication_leases.take() {
|
|
let release_result = notification_system.release_scanner_publication_leases(grants).await;
|
|
let lease_release_failed = release_result.is_err();
|
|
if lease_expired || lease_release_failed {
|
|
// A lease that expired or could not be released is never treated
|
|
// as a successful authoritative publication. The peer may have
|
|
// admitted movement immediately after the lease ended.
|
|
usage_persist_outcome = if usage_persist_outcome == DataUsagePersistOutcome::Failed {
|
|
DataUsagePersistOutcome::Failed
|
|
} else if lease_release_failed {
|
|
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseReleaseFailed)
|
|
} else {
|
|
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded)
|
|
};
|
|
}
|
|
}
|
|
let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
|
|
|
|
let scan_cycle_result = match scan_result {
|
|
Ok(result) => result,
|
|
Err(e) => {
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
scan_mode = ?scan_mode,
|
|
state = "failed",
|
|
duration = ?now.elapsed(),
|
|
error = %e,
|
|
"Scanner cycle failed"
|
|
);
|
|
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
|
if !ctx.is_cancelled()
|
|
&& let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, false)
|
|
{
|
|
save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
|
|
}
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Failed;
|
|
}
|
|
};
|
|
if ctx.is_cancelled() {
|
|
debug!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
state = "cancelled_before_commit",
|
|
"Scanner cycle stopped before committing cycle state"
|
|
);
|
|
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Failed;
|
|
}
|
|
let required_cycle_floor = scan_cycle_result.required_cycle_floor();
|
|
let pre_commit_outcome = scanner_cycle_pre_commit_outcome(required_cycle_floor, &usage_persist_outcome);
|
|
update_scanner_cache_cycle_recovery_status(
|
|
cycle_info.current,
|
|
leader_epoch,
|
|
required_cycle_floor,
|
|
pre_commit_outcome,
|
|
scan_cycle_result.status == ScannerCycleStatus::Complete,
|
|
);
|
|
match pre_commit_outcome {
|
|
Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(required_cycle)) => {
|
|
record_scanner_cache_cycle_recovery_attempt();
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
required_cycle,
|
|
state = "cache_cycle_ahead",
|
|
"Scanner cycle is recovering to a newer durable cache generation"
|
|
);
|
|
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
|
let persisted = persist_required_scanner_cycle_floor_for_epoch(
|
|
ctx,
|
|
storeapi.clone(),
|
|
cycle_info,
|
|
cycle_revision,
|
|
leader_epoch,
|
|
&mut cycle_metrics_guard,
|
|
ScannerCycleFloorOptions {
|
|
required_cycle,
|
|
expected_publication_epoch: publication_epoch,
|
|
},
|
|
)
|
|
.await;
|
|
return if persisted {
|
|
cycle_budget.mark_cycle_state_persisted();
|
|
ScannerCycleOutcome::Partial
|
|
} else if let Some(expected_epoch) = publication_epoch
|
|
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
|
.await
|
|
.is_none()
|
|
{
|
|
emit_scan_cycle_deferred(cycle_start.elapsed());
|
|
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
|
|
} else {
|
|
ScannerCycleOutcome::Failed
|
|
};
|
|
}
|
|
Some(ScannerCyclePreCommitOutcome::Deferred(reason)) => {
|
|
info!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
reason = reason.as_str(),
|
|
state = "deferred",
|
|
"Scanner cycle deferred before data usage publication"
|
|
);
|
|
if publication_deferred {
|
|
global_metrics().record_scanner_usage_deferred(reason.as_str());
|
|
}
|
|
emit_scan_cycle_deferred(cycle_start.elapsed());
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Deferred(reason);
|
|
}
|
|
None => {}
|
|
}
|
|
if usage_persist_outcome == DataUsagePersistOutcome::Failed {
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
state = "usage_not_durable",
|
|
"Scanner cycle completed without a durable data usage snapshot"
|
|
);
|
|
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Failed;
|
|
}
|
|
if budget_elapsed {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
duration = ?now.elapsed(),
|
|
reason = ?cycle_budget.reason(),
|
|
max_duration = ?cycle_budget.max_duration(),
|
|
max_objects = ?cycle_budget.max_objects(),
|
|
max_directories = ?cycle_budget.max_directories(),
|
|
state = "budget_reached",
|
|
"Scanner cycle budget reached"
|
|
);
|
|
let budget_reason = cycle_budget.reason();
|
|
emit_scan_cycle_partial_with_source(
|
|
cycle_start.elapsed(),
|
|
scan_cycle_partial_reason(budget_reason),
|
|
scan_cycle_partial_source(budget_reason),
|
|
);
|
|
let persisted = finalize_partial_scan_cycle_for_epoch(
|
|
ctx,
|
|
storeapi.clone(),
|
|
cycle_info,
|
|
cycle_revision,
|
|
leader_epoch,
|
|
&mut cycle_metrics_guard,
|
|
publication_epoch,
|
|
)
|
|
.await;
|
|
return if persisted {
|
|
cycle_budget.mark_cycle_state_persisted();
|
|
ScannerCycleOutcome::Partial
|
|
} else if let Some(expected_epoch) = publication_epoch
|
|
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
|
.await
|
|
.is_none()
|
|
{
|
|
emit_scan_cycle_deferred(cycle_start.elapsed());
|
|
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
|
|
} else {
|
|
ScannerCycleOutcome::Failed
|
|
};
|
|
}
|
|
|
|
let (completion_outcome, scanner_pending_maintenance_work, remote_dirty_usage_acknowledgements) =
|
|
finalize_scanner_cycle_result(scan_cycle_result, usage_persist_outcome);
|
|
let remote_dirty_usage_pending = if remote_dirty_usage_acknowledgements.is_empty() {
|
|
false
|
|
} else if let Some(notification_system) = storeapi.scanner_notification_system() {
|
|
let acknowledgement_count = remote_dirty_usage_acknowledgements.len();
|
|
let acknowledgements = remote_dirty_usage_acknowledgements
|
|
.into_iter()
|
|
.map(|acknowledgement| (acknowledgement.host, acknowledgement.instance_id, acknowledgement.generation))
|
|
.collect();
|
|
remote_dirty_usage_acknowledgement_pending(
|
|
cycle_info.current,
|
|
acknowledgement_count,
|
|
notification_system.acknowledge_scanner_dirty_usage(acknowledgements),
|
|
)
|
|
.await
|
|
} else {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
state = "remote_dirty_usage_acknowledgement_unavailable",
|
|
"Scanner cycle cannot acknowledge remote dirty usage without a notification system"
|
|
);
|
|
true
|
|
};
|
|
let pending_maintenance_work = scanner_pending_maintenance_work || unresolved_heal_work || remote_dirty_usage_pending;
|
|
match completion_outcome {
|
|
ScannerCycleOutcome::Failed => {
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
outcome = ?usage_persist_outcome,
|
|
state = "usage_not_durable",
|
|
"Scanner cycle completed without a durable data usage snapshot"
|
|
);
|
|
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Failed;
|
|
}
|
|
ScannerCycleOutcome::Partial => {
|
|
if ctx.is_cancelled() {
|
|
debug!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
state = "incomplete_cancelled",
|
|
"Scanner cycle stopped before a complete usage snapshot was produced"
|
|
);
|
|
} else {
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
state = "incomplete",
|
|
"Scanner cycle ended without a complete usage snapshot"
|
|
);
|
|
}
|
|
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
|
let persisted = finalize_partial_scan_cycle_for_epoch(
|
|
ctx,
|
|
storeapi.clone(),
|
|
cycle_info,
|
|
cycle_revision,
|
|
leader_epoch,
|
|
&mut cycle_metrics_guard,
|
|
publication_epoch,
|
|
)
|
|
.await;
|
|
return if persisted {
|
|
cycle_budget.mark_cycle_state_persisted();
|
|
ScannerCycleOutcome::Partial
|
|
} else if let Some(expected_epoch) = publication_epoch
|
|
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
|
.await
|
|
.is_none()
|
|
{
|
|
emit_scan_cycle_deferred(cycle_start.elapsed());
|
|
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
|
|
} else {
|
|
ScannerCycleOutcome::Failed
|
|
};
|
|
}
|
|
ScannerCycleOutcome::Deferred(reason) => {
|
|
info!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
reason = reason.as_str(),
|
|
state = "deferred",
|
|
"Scanner cycle deferred before usage scanning began"
|
|
);
|
|
global_metrics().record_scanner_usage_deferred(reason.as_str());
|
|
emit_scan_cycle_deferred(cycle_start.elapsed());
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Deferred(reason);
|
|
}
|
|
ScannerCycleOutcome::Superseded => {
|
|
info!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
state = "superseded",
|
|
"Scanner cycle usage snapshot was superseded by concurrent namespace activity"
|
|
);
|
|
if finalize_partial_scan_cycle_for_epoch(
|
|
ctx,
|
|
storeapi.clone(),
|
|
cycle_info,
|
|
cycle_revision,
|
|
leader_epoch,
|
|
&mut cycle_metrics_guard,
|
|
publication_epoch,
|
|
)
|
|
.await
|
|
{
|
|
cycle_budget.mark_cycle_state_persisted();
|
|
emit_scan_cycle_superseded(cycle_start.elapsed());
|
|
return ScannerCycleOutcome::Superseded;
|
|
}
|
|
if let Some(expected_epoch) = publication_epoch
|
|
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
|
.await
|
|
.is_none()
|
|
{
|
|
emit_scan_cycle_deferred(cycle_start.elapsed());
|
|
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
|
}
|
|
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
|
return ScannerCycleOutcome::Failed;
|
|
}
|
|
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance => {}
|
|
}
|
|
if let Some(expected_epoch) = publication_epoch
|
|
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
|
.await
|
|
.is_none()
|
|
{
|
|
emit_scan_cycle_deferred(cycle_start.elapsed());
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
|
}
|
|
let previous_cycle_info = cycle_info.clone();
|
|
if let Err(err) = advance_scanner_cycle(cycle_info) {
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "cycle_counter_exhausted",
|
|
error = %err,
|
|
"Scanner completed cycle could not advance"
|
|
);
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
|
return ScannerCycleOutcome::Failed;
|
|
}
|
|
cycle_info.current = 0;
|
|
cycle_info.cycle_completed.push(Utc::now());
|
|
global_metrics().clear_current_scan_mode();
|
|
|
|
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
|
|
if !persist_scanner_cycle_state_for_epoch(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, publication_epoch)
|
|
.await
|
|
{
|
|
if let Some(expected_epoch) = publication_epoch
|
|
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
|
|
.await
|
|
.is_none()
|
|
{
|
|
*cycle_info = previous_cycle_info;
|
|
emit_scan_cycle_deferred(cycle_start.elapsed());
|
|
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
|
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
|
}
|
|
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
|
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
|
return ScannerCycleOutcome::Failed;
|
|
}
|
|
cycle_budget.mark_cycle_state_persisted();
|
|
|
|
done_cycle();
|
|
emit_scan_cycle_complete(true, cycle_start.elapsed());
|
|
if let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, true) {
|
|
save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
|
|
}
|
|
|
|
info!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
cycle = cycle_info.current,
|
|
scan_mode = ?scan_mode,
|
|
state = "completed",
|
|
duration = ?now.elapsed(),
|
|
cycles_total = cycle_info.cycle_completed.len(),
|
|
"Scanner cycle completed"
|
|
);
|
|
|
|
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
|
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, pending_maintenance_work)
|
|
}
|
|
|
|
struct ScannerCycleMetricsGuard {
|
|
start: Option<ScanCycleWorkSnapshot>,
|
|
}
|
|
|
|
impl ScannerCycleMetricsGuard {
|
|
async fn new(cycle: CurrentCycle) -> Self {
|
|
Self {
|
|
start: Some(global_metrics().start_scan_cycle_work_with_cycle(cycle).await),
|
|
}
|
|
}
|
|
|
|
async fn finish(&mut self, cycle: CurrentCycle) {
|
|
if let Some(start) = self.start {
|
|
global_metrics().finish_scan_cycle_work_with_cycle(start, cycle).await;
|
|
self.start = None;
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Drop for ScannerCycleMetricsGuard {
|
|
fn drop(&mut self) {
|
|
if let Some(start) = self.start.take() {
|
|
global_metrics().finish_scan_cycle_work(start);
|
|
}
|
|
}
|
|
}
|
|
|
|
pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) -> Result<(), ScannerError> {
|
|
run_data_scanner_with_storage(ctx, storeapi).await
|
|
}
|
|
|
|
async fn run_data_scanner_with_storage<S>(ctx: CancellationToken, storeapi: Arc<S>) -> Result<(), ScannerError>
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
let (maintenance_features, maintenance_generation) = configure_scanner_defaults(&ctx, &storeapi).await;
|
|
run_data_scanner_with_maintenance_state(ctx, storeapi, maintenance_features, maintenance_generation).await
|
|
}
|
|
|
|
async fn current_scanner_pause_backlog_observation<S>(storeapi: &Arc<S>) -> ScannerPauseBacklogObservation
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
let now_unix_secs = scanner_pause_backlog_now();
|
|
let pause = storeapi.scanner_data_movement_pause_status().await;
|
|
let metrics = global_metrics().report().await;
|
|
ScannerPauseBacklogObservation {
|
|
now_unix_secs,
|
|
paused: pause.paused,
|
|
movement_generation: pause.movement_generation,
|
|
movement_work_items: pause.movement_backlog_work_items,
|
|
pause_started_at_unix_secs: pause.started_at_unix_secs,
|
|
dirty_usage_buckets: metrics.usage_freshness.dirty_pending_buckets,
|
|
discovered_expiry_items: metrics
|
|
.lifecycle_expiry
|
|
.current_queued
|
|
.saturating_add(metrics.lifecycle_expiry.current_active),
|
|
discovered_transition_items: metrics
|
|
.lifecycle_transition
|
|
.current_queued
|
|
.saturating_add(metrics.lifecycle_transition.current_active)
|
|
.saturating_add(metrics.lifecycle_transition.compensation_pending)
|
|
.saturating_add(metrics.lifecycle_transition.compensation_running),
|
|
}
|
|
}
|
|
|
|
async fn wait_for_scanner_data_movement_resume<S>(
|
|
ctx: &CancellationToken,
|
|
storeapi: &Arc<S>,
|
|
guard: &NamespaceLockGuard,
|
|
pause_backlog: &mut ScannerPauseBacklogController<S>,
|
|
) -> bool
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
loop {
|
|
let observation = current_scanner_pause_backlog_observation(storeapi).await;
|
|
pause_backlog.observe(observation).await;
|
|
#[cfg(test)]
|
|
notify_scanner_runtime_observed_for_test(storeapi, observation);
|
|
if !observation.paused {
|
|
return !ctx.is_cancelled() && !guard.is_lock_lost();
|
|
}
|
|
|
|
let movement_changed = storeapi.scanner_data_movement_changed();
|
|
if storeapi.scanner_data_movement_generation() != observation.movement_generation {
|
|
continue;
|
|
}
|
|
tokio::select! {
|
|
_ = ctx.cancelled() => return false,
|
|
_ = guard.lock_lost_notified() => return false,
|
|
_ = movement_changed.notified() => {},
|
|
_ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {},
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn finish_scanner_pause_backlog_cycle<S>(
|
|
pause_backlog: &mut ScannerPauseBacklogController<S>,
|
|
storeapi: &Arc<S>,
|
|
attempt: ScannerPauseBacklogAttemptDecision,
|
|
outcome: ScannerCycleOutcome,
|
|
) where
|
|
S: ScannerStorage,
|
|
{
|
|
let observation = current_scanner_pause_backlog_observation(storeapi).await;
|
|
if let ScannerPauseBacklogAttemptDecision::Tracked(serial) = attempt {
|
|
pause_backlog.finish_attempt(serial, outcome, observation).await;
|
|
} else {
|
|
pause_backlog.observe_cycle_outcome(outcome, observation).await;
|
|
}
|
|
#[cfg(test)]
|
|
notify_scanner_runtime_observed_for_test(storeapi, observation);
|
|
}
|
|
|
|
async fn run_data_scanner_with_maintenance_state<S>(
|
|
ctx: CancellationToken,
|
|
storeapi: Arc<S>,
|
|
mut maintenance_features: ScannerMaintenanceFeatures,
|
|
mut maintenance_generation_seen: Option<u64>,
|
|
) -> Result<(), ScannerError>
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
reset_scanner_cycle_schedule();
|
|
// Acquire leader lock (write lock) to ensure only one scanner runs
|
|
let mut guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
|
|
Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
|
|
Ok(guard) => {
|
|
record_scanner_cycle_schedule_role("leader");
|
|
record_scanner_leader_lock_state("acquired");
|
|
global_metrics().record_scanner_leader_liveness("acquired", true, "").await;
|
|
debug!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_LOCK_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
lock_name = "leader.lock",
|
|
state = "acquired",
|
|
"Scanner leader lock acquired"
|
|
);
|
|
guard
|
|
}
|
|
Err(e) => {
|
|
match classify_scanner_leader_lock_failure(&e) {
|
|
ScannerLeaderLockFailure::Contended => {
|
|
record_scanner_cycle_schedule_role("follower");
|
|
record_scanner_leader_lock_state("contended");
|
|
global_metrics().record_scanner_leader_liveness("contended", false, "").await;
|
|
debug!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_LOCK_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
lock_name = "leader.lock",
|
|
state = "contended",
|
|
"Scanner leader lock contended"
|
|
);
|
|
}
|
|
ScannerLeaderLockFailure::Failed(error) => {
|
|
record_scanner_leader_lock_state("acquire_failed");
|
|
global_metrics()
|
|
.record_scanner_leader_liveness("acquire_failed", false, error.to_string())
|
|
.await;
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_LOCK_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
lock_name = "leader.lock",
|
|
state = "acquire_failed",
|
|
error = %error,
|
|
"Scanner leader lock acquisition failed"
|
|
);
|
|
}
|
|
}
|
|
return Ok(());
|
|
}
|
|
},
|
|
Err(e) => {
|
|
record_scanner_leader_lock_state("create_failed");
|
|
global_metrics()
|
|
.record_scanner_leader_liveness("create_failed", false, e.to_string())
|
|
.await;
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_LOCK_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
lock_name = "leader.lock",
|
|
state = "create_failed",
|
|
error = %e,
|
|
"Scanner leader lock creation failed"
|
|
);
|
|
return Ok(());
|
|
}
|
|
};
|
|
let pause_backlog_now = scanner_pause_backlog_now();
|
|
let mut pause_backlog = match ScannerPauseBacklogController::claim(storeapi.clone(), pause_backlog_now).await {
|
|
Ok(controller) => controller,
|
|
Err(err) => {
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "pause_backlog_claim_failed",
|
|
error = %err,
|
|
"Scanner pause backlog persistence is unavailable"
|
|
);
|
|
ScannerPauseBacklogController::unavailable(storeapi.clone(), err, pause_backlog_now)
|
|
}
|
|
};
|
|
if !wait_for_scanner_data_movement_resume(&ctx, &storeapi, &guard, &mut pause_backlog).await {
|
|
global_metrics().set_cycle(None).await;
|
|
return Ok(());
|
|
}
|
|
#[cfg(test)]
|
|
notify_scanner_startup_observed_for_test().await;
|
|
let single_disk = storeapi.setup_is_erasure_sd().await;
|
|
let erasure = storeapi.setup_is_erasure().await;
|
|
let distributed = storeapi.setup_is_dist_erasure().await;
|
|
let clean_idle_topology_supported = single_disk || erasure;
|
|
let mut dirty_usage_generation_seen = dirty_usage_generation();
|
|
let mut runtime_config_generation_seen = scanner_runtime_config_generation();
|
|
let mut clean_idle_backoff = ScannerCleanIdleBackoff::default();
|
|
let mut superseded_backoff = ScannerRetryBackoff::default();
|
|
let mut deferred_backoff = ScannerRetryBackoff::default();
|
|
let initial_runtime_config = resolve_scanner_runtime_config();
|
|
if clean_idle_topology_supported && maintenance_generation_seen.is_none() {
|
|
let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else {
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(false, "stopped", String::new()).await;
|
|
return Ok(());
|
|
};
|
|
maintenance_features = features;
|
|
maintenance_generation_seen = Some(generation);
|
|
}
|
|
let mut maintenance_inspection_retry = ScannerMaintenanceInspectionRetry::from_features(maintenance_features, Instant::now());
|
|
let mut scanner_activity_seen = None;
|
|
let mut scanner_activity_backoff_blocked = false;
|
|
if scanner_activity_probe_required(
|
|
clean_idle_topology_supported,
|
|
scanner_activity_backoff_blocked,
|
|
maintenance_features,
|
|
&initial_runtime_config,
|
|
) {
|
|
observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await;
|
|
}
|
|
|
|
let (mut cycle_info, mut leader_epoch, mut cycle_revision) =
|
|
match load_scanner_cycle_state_for_startup(storeapi.clone()).await {
|
|
ScannerCycleStateStartup::Ready {
|
|
cycle,
|
|
leader_epoch,
|
|
revision,
|
|
} => (cycle, leader_epoch, revision),
|
|
ScannerCycleStateStartup::Blocked => {
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(false, "stopped", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
ScannerCycleStateStartup::Transient(err) => {
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(false, "stopped", String::new()).await;
|
|
return Err(err);
|
|
}
|
|
};
|
|
let (usage_floor, usage_floor_startup) = match persisted_usage_floor_for_startup(storeapi.clone(), true).await {
|
|
Ok(result) => result,
|
|
Err(err) => {
|
|
let error = err.to_string();
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
|
state = "usage_floor_load_failed",
|
|
error = %err,
|
|
"Scanner stopped because the persisted usage floor could not be loaded"
|
|
);
|
|
record_scanner_usage_floor_failure(error.clone());
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(false, "usage_floor_load_failed", error).await;
|
|
return Ok(());
|
|
}
|
|
};
|
|
let (allow_usage_floor_bootstrap_pending, usage_floor_cycle_reset_policy) =
|
|
prepare_cycle_for_usage_floor_bootstrap(&mut cycle_info, usage_floor, usage_floor_startup);
|
|
apply_persisted_usage_floor(&mut cycle_info, &mut leader_epoch, usage_floor);
|
|
match usage_floor_startup {
|
|
PersistedUsageFloorStartup::Authoritative
|
|
| PersistedUsageFloorStartup::BootstrapPending
|
|
| PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence => {}
|
|
PersistedUsageFloorStartup::Missing => {
|
|
if ctx.is_cancelled() || guard.is_lock_lost() {
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
|
|
let bootstrap_ctx = ctx.child_token();
|
|
match await_scanner_cycle_with_lock_fence(
|
|
&bootstrap_ctx,
|
|
initialize_usage_baseline_bootstrap(storeapi.clone()),
|
|
guard.lock_lost_notified(),
|
|
)
|
|
.await
|
|
{
|
|
Some(Ok(())) => {}
|
|
Some(Err(err)) => {
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
|
state = "usage_floor_bootstrap_failed",
|
|
error = %err,
|
|
"Scanner stopped because the usage baseline bootstrap could not be initialized"
|
|
);
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(false, "usage_floor_bootstrap_failed", err.to_string()).await;
|
|
return Ok(());
|
|
}
|
|
None => {
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if ctx.is_cancelled() || guard.is_lock_lost() {
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
let claim_ctx = ctx.child_token();
|
|
let leadership_claimed = await_scanner_cycle_with_lock_fence(
|
|
&claim_ctx,
|
|
claim_scanner_leadership(
|
|
&claim_ctx,
|
|
storeapi.clone(),
|
|
&mut cycle_info,
|
|
&mut cycle_revision,
|
|
&mut leader_epoch,
|
|
allow_usage_floor_bootstrap_pending,
|
|
usage_floor_cycle_reset_policy,
|
|
),
|
|
guard.lock_lost_notified(),
|
|
)
|
|
.await
|
|
.unwrap_or(false);
|
|
if guard.is_lock_lost() {
|
|
record_scanner_leader_lock_lost("Scanner leader lock lost while claiming the leadership epoch").await;
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(true, "lost", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
if !leadership_claimed {
|
|
let observation = current_scanner_pause_backlog_observation(&storeapi).await;
|
|
pause_backlog.observe(observation).await;
|
|
#[cfg(test)]
|
|
notify_scanner_runtime_observed_for_test(&storeapi, observation);
|
|
if observation.paused {
|
|
if wait_for_scanner_data_movement_resume(&ctx, &storeapi, &guard, &mut pause_backlog).await {
|
|
return Err(ScannerError::Other(
|
|
"scanner startup was fenced by data movement; retrying from durable state".to_string(),
|
|
));
|
|
}
|
|
global_metrics().set_cycle(None).await;
|
|
return Ok(());
|
|
}
|
|
error!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_LOCK_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
lock_name = "leader.lock",
|
|
state = "epoch_claim_failed",
|
|
"Scanner stopped because the leadership epoch could not be claimed"
|
|
);
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(false, "epoch_claim_failed", "leadership epoch claim failed".to_string()).await;
|
|
return Ok(());
|
|
}
|
|
if usage_floor_startup == PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence
|
|
&& let Err(err) = complete_legacy_incomplete_usage_floor_recovery(storeapi.clone(), leader_epoch).await
|
|
{
|
|
let error = err.to_string();
|
|
warn!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_PERSIST_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "usage_floor_recovery_cleanup_deferred",
|
|
path = %DATA_USAGE_RECOVERY_PATH.as_str(),
|
|
error = %err,
|
|
"Scanner usage floor recovery marker cleanup was deferred"
|
|
);
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(false, "usage_floor_recovery_pending", error).await;
|
|
return Ok(());
|
|
}
|
|
|
|
let initial_pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await;
|
|
if !ctx.is_cancelled()
|
|
&& matches!(
|
|
initial_pause_backlog_attempt,
|
|
ScannerPauseBacklogAttemptDecision::Untracked | ScannerPauseBacklogAttemptDecision::Tracked(_)
|
|
)
|
|
{
|
|
// Preserve previous behavior: run one cycle immediately after lock acquisition.
|
|
let dirty_generation_before_cycle = dirty_usage_generation();
|
|
let dirty_usage_pending_before_cycle = dirty_usage_buckets_pending();
|
|
let maintenance_generation_before_cycle = scanner_maintenance_generation();
|
|
if guard.is_lock_lost() {
|
|
record_scanner_leader_lock_lost("Scanner leader lock lost before the initial cycle").await;
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(true, "lost", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
let cycle_ctx = ctx.child_token();
|
|
let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
|
|
let initial_outcome = match await_scanner_cycle_with_budget_fence(
|
|
&cycle_ctx,
|
|
&cycle_budget,
|
|
run_data_scanner_cycle_with_budget(
|
|
&cycle_ctx,
|
|
&storeapi,
|
|
&mut cycle_info,
|
|
&mut cycle_revision,
|
|
leader_epoch,
|
|
cycle_budget.clone(),
|
|
true,
|
|
),
|
|
guard.lock_lost_notified(),
|
|
)
|
|
.await
|
|
{
|
|
ScannerCycleWaitOutcome::Completed(outcome) => outcome,
|
|
ScannerCycleWaitOutcome::LockLost => {
|
|
record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await;
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(true, "lost", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
ScannerCycleWaitOutcome::Cancelled => {
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
|
handle_scanner_cycle_deadline(
|
|
&ctx,
|
|
storeapi.clone(),
|
|
ScannerCycleDeadlineState {
|
|
cycle_info: &mut cycle_info,
|
|
cycle_revision: &mut cycle_revision,
|
|
leader_epoch: &mut leader_epoch,
|
|
cycle_budget: &cycle_budget,
|
|
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
|
|
},
|
|
worker_stopped,
|
|
&mut guard,
|
|
)
|
|
.await;
|
|
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
};
|
|
finish_scanner_pause_backlog_cycle(&mut pause_backlog, &storeapi, initial_pause_backlog_attempt, initial_outcome).await;
|
|
superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded);
|
|
deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_)));
|
|
dirty_usage_generation_seen = dirty_generation_before_cycle;
|
|
if guard.is_lock_lost() {
|
|
record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await;
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(true, "lost", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
let runtime_config = resolve_scanner_runtime_config();
|
|
let scanner_activity_observation = if scanner_activity_probe_required(
|
|
clean_idle_topology_supported,
|
|
scanner_activity_backoff_blocked,
|
|
maintenance_features,
|
|
&runtime_config,
|
|
) {
|
|
observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await
|
|
} else {
|
|
scanner_activity_seen = None;
|
|
ScannerActivityObservation::NotRequired
|
|
};
|
|
if scanner_activity_observation == ScannerActivityObservation::MaintenanceChanged {
|
|
scanner_activity_backoff_blocked = true;
|
|
}
|
|
let scanner_activity_ready = !scanner_activity_backoff_blocked && scanner_activity_seen.is_some();
|
|
let backoff_enabled = scanner_clean_idle_backoff_enabled(
|
|
clean_idle_topology_supported,
|
|
scanner_activity_ready,
|
|
maintenance_features,
|
|
&runtime_config,
|
|
);
|
|
record_scanner_cycle_result(
|
|
&mut clean_idle_backoff,
|
|
&runtime_config,
|
|
backoff_enabled,
|
|
ScannerCycleWakeReason::Timer,
|
|
initial_outcome,
|
|
scanner_cycle_observed_dirty_work(
|
|
dirty_usage_pending_before_cycle,
|
|
dirty_generation_before_cycle,
|
|
dirty_usage_generation(),
|
|
) || maintenance_generation_before_cycle != scanner_maintenance_generation()
|
|
|| scanner_activity_observed_work(scanner_activity_observation),
|
|
);
|
|
runtime_config_generation_seen = scanner_runtime_config_generation();
|
|
}
|
|
|
|
loop {
|
|
if ctx.is_cancelled() {
|
|
break;
|
|
}
|
|
|
|
let pause_backlog_observation = current_scanner_pause_backlog_observation(&storeapi).await;
|
|
pause_backlog.observe(pause_backlog_observation).await;
|
|
#[cfg(test)]
|
|
notify_scanner_runtime_observed_for_test(&storeapi, pause_backlog_observation);
|
|
let runtime_config = resolve_scanner_runtime_config();
|
|
if clean_idle_topology_supported {
|
|
let current_generation = scanner_maintenance_generation();
|
|
if maintenance_generation_seen != Some(current_generation) {
|
|
scanner_activity_seen = None;
|
|
scanner_activity_backoff_blocked = scanner_activity_backoff_blocked_after_wake(
|
|
scanner_activity_backoff_blocked,
|
|
ScannerCycleWakeReason::MaintenanceConfig,
|
|
);
|
|
let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else {
|
|
break;
|
|
};
|
|
maintenance_features = features;
|
|
maintenance_generation_seen = Some(generation);
|
|
maintenance_inspection_retry.record_inspection(features, Instant::now());
|
|
}
|
|
}
|
|
if !scanner_activity_probe_required(
|
|
clean_idle_topology_supported,
|
|
scanner_activity_backoff_blocked,
|
|
maintenance_features,
|
|
&runtime_config,
|
|
) {
|
|
scanner_activity_seen = None;
|
|
}
|
|
let scanner_activity_ready = !scanner_activity_backoff_blocked && scanner_activity_seen.is_some();
|
|
let backoff_enabled = scanner_clean_idle_backoff_enabled(
|
|
clean_idle_topology_supported,
|
|
scanner_activity_ready,
|
|
maintenance_features,
|
|
&runtime_config,
|
|
);
|
|
let mut wait_plan =
|
|
scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for);
|
|
let superseded_retry_interval = scanner_superseded_retry_interval(superseded_backoff, &runtime_config);
|
|
let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval);
|
|
let mut convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval);
|
|
if let Some(retry_interval) = convergence_retry_interval {
|
|
wait_plan.effective_interval = retry_interval;
|
|
wait_plan.delay = randomized_cycle_delay_for(retry_interval).min(retry_interval);
|
|
}
|
|
if let Some(pause_backlog_delay) = pause_backlog.scheduling_delay(scanner_pause_backlog_now()) {
|
|
wait_plan.effective_interval = pause_backlog_delay.max(Duration::from_secs(1));
|
|
wait_plan.delay = pause_backlog_delay;
|
|
convergence_retry_interval = Some(pause_backlog_delay.max(Duration::from_secs(1)));
|
|
}
|
|
let dirty_generation_before_wait = dirty_usage_generation();
|
|
let dirty_usage_pending_before_wait = dirty_usage_buckets_pending();
|
|
let maintenance_generation_before_wait = scanner_maintenance_generation();
|
|
record_scanner_cycle_schedule(
|
|
wait_plan.effective_interval,
|
|
backoff_enabled,
|
|
u64::from(clean_idle_backoff.interval_multiplier),
|
|
superseded_retry_interval.is_some(),
|
|
superseded_backoff.consecutive_cycles,
|
|
);
|
|
debug!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
configured_interval = ?runtime_config.cycle_interval,
|
|
effective_interval = ?wait_plan.effective_interval,
|
|
clean_idle_max_interval = ?wait_plan.clean_idle_max_interval,
|
|
scheduled_delay = ?wait_plan.delay,
|
|
interval_multiplier = clean_idle_backoff.interval_multiplier,
|
|
clean_idle_backoff_enabled = backoff_enabled,
|
|
superseded_retry_backoff_enabled = superseded_retry_interval.is_some(),
|
|
superseded_cycles = superseded_backoff.consecutive_cycles,
|
|
deferred_retry_backoff_enabled = deferred_retry_interval.is_some(),
|
|
deferred_cycles = deferred_backoff.consecutive_cycles,
|
|
lifecycle_active = maintenance_features.lifecycle,
|
|
replication_active = maintenance_features.replication,
|
|
feature_inspection_failed = maintenance_features.inspection_failed,
|
|
state = "wait_scheduled",
|
|
"Scanner cycle wait scheduled"
|
|
);
|
|
|
|
let activity_poll_interval = backoff_enabled.then_some(runtime_config.cycle_interval.max(Duration::from_secs(1)));
|
|
let movement_generation_before_wait = storeapi.scanner_data_movement_generation();
|
|
let movement_changed = storeapi.scanner_data_movement_changed();
|
|
let movement_store = storeapi.clone();
|
|
let movement = ScannerMovementWaitContext {
|
|
movement_generation_seen: Some(movement_generation_before_wait),
|
|
movement_changed,
|
|
current_movement_generation: move || movement_store.scanner_data_movement_generation(),
|
|
is_lock_lost: || guard.is_lock_lost(),
|
|
};
|
|
let wake_reason = wait_for_next_scanner_cycle_with_activity_and_movement(
|
|
&ctx,
|
|
wait_plan.delay,
|
|
activity_poll_interval,
|
|
&mut scanner_activity_seen,
|
|
ScannerCycleObservedGenerations::for_wait(
|
|
&runtime_config,
|
|
convergence_retry_interval,
|
|
dirty_usage_generation_seen,
|
|
runtime_config_generation_seen,
|
|
maintenance_generation_before_wait,
|
|
),
|
|
movement,
|
|
|| probe_scanner_activity(storeapi.as_ref(), distributed),
|
|
)
|
|
.await;
|
|
scanner_activity_backoff_blocked =
|
|
scanner_activity_backoff_blocked_after_wake(scanner_activity_backoff_blocked, wake_reason);
|
|
match wake_reason {
|
|
ScannerCycleWakeReason::Cancelled => break,
|
|
ScannerCycleWakeReason::LeaderLockLost => {
|
|
record_scanner_leader_lock_lost("Scanner leader lock lost while waiting for the next cycle").await;
|
|
break;
|
|
}
|
|
ScannerCycleWakeReason::RuntimeConfig => {
|
|
runtime_config_generation_seen = scanner_runtime_config_generation();
|
|
maintenance_generation_seen = None;
|
|
scanner_activity_seen = None;
|
|
clean_idle_backoff.reset();
|
|
continue;
|
|
}
|
|
ScannerCycleWakeReason::MaintenanceConfig => {
|
|
maintenance_generation_seen = None;
|
|
scanner_activity_seen = None;
|
|
clean_idle_backoff.reset();
|
|
continue;
|
|
}
|
|
ScannerCycleWakeReason::ClusterMaintenance => {
|
|
clean_idle_backoff.reset();
|
|
}
|
|
ScannerCycleWakeReason::MovementGeneration => {
|
|
scanner_activity_seen = None;
|
|
clean_idle_backoff.reset();
|
|
}
|
|
ScannerCycleWakeReason::Timer
|
|
| ScannerCycleWakeReason::DirtyUsage
|
|
| ScannerCycleWakeReason::ClusterActivity
|
|
| ScannerCycleWakeReason::ClusterActivityUnavailable => {}
|
|
}
|
|
|
|
if wake_reason == ScannerCycleWakeReason::DirtyUsage {
|
|
debug!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "dirty_usage_wakeup",
|
|
"Scanner cycle woke for dirty usage work"
|
|
);
|
|
}
|
|
if matches!(
|
|
wake_reason,
|
|
ScannerCycleWakeReason::ClusterActivity
|
|
| ScannerCycleWakeReason::ClusterMaintenance
|
|
| ScannerCycleWakeReason::ClusterActivityUnavailable
|
|
) {
|
|
let cluster_activity_verified = wake_reason == ScannerCycleWakeReason::ClusterActivity;
|
|
debug!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "cluster_activity_wakeup",
|
|
cluster_activity_verified,
|
|
"Scanner cycle woke for cluster activity"
|
|
);
|
|
}
|
|
|
|
if guard.is_lock_lost() {
|
|
record_scanner_leader_lock_lost("Scanner leader lock lost before starting the next cycle").await;
|
|
break;
|
|
}
|
|
let pause_backlog_observation = current_scanner_pause_backlog_observation(&storeapi).await;
|
|
pause_backlog.observe(pause_backlog_observation).await;
|
|
#[cfg(test)]
|
|
notify_scanner_runtime_observed_for_test(&storeapi, pause_backlog_observation);
|
|
if pause_backlog_observation.paused {
|
|
continue;
|
|
}
|
|
let pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await;
|
|
if matches!(
|
|
pause_backlog_attempt,
|
|
ScannerPauseBacklogAttemptDecision::RateLimited | ScannerPauseBacklogAttemptDecision::PersistenceUnavailable
|
|
) {
|
|
continue;
|
|
}
|
|
let dirty_generation_before_cycle = dirty_usage_generation();
|
|
let cycle_ctx = ctx.child_token();
|
|
let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
|
|
let outcome = match await_scanner_cycle_with_budget_fence(
|
|
&cycle_ctx,
|
|
&cycle_budget,
|
|
run_data_scanner_cycle_with_budget(
|
|
&cycle_ctx,
|
|
&storeapi,
|
|
&mut cycle_info,
|
|
&mut cycle_revision,
|
|
leader_epoch,
|
|
cycle_budget.clone(),
|
|
maintenance_features.requires_full_scan(
|
|
maintenance_generation_seen,
|
|
scanner_maintenance_generation(),
|
|
wake_reason,
|
|
),
|
|
),
|
|
guard.lock_lost_notified(),
|
|
)
|
|
.await
|
|
{
|
|
ScannerCycleWaitOutcome::Completed(outcome) => outcome,
|
|
ScannerCycleWaitOutcome::LockLost => {
|
|
record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await;
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(true, "lost", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
ScannerCycleWaitOutcome::Cancelled => {
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
|
handle_scanner_cycle_deadline(
|
|
&ctx,
|
|
storeapi.clone(),
|
|
ScannerCycleDeadlineState {
|
|
cycle_info: &mut cycle_info,
|
|
cycle_revision: &mut cycle_revision,
|
|
leader_epoch: &mut leader_epoch,
|
|
cycle_budget: &cycle_budget,
|
|
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
|
|
},
|
|
worker_stopped,
|
|
&mut guard,
|
|
)
|
|
.await;
|
|
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
|
|
return Ok(());
|
|
}
|
|
};
|
|
finish_scanner_pause_backlog_cycle(&mut pause_backlog, &storeapi, pause_backlog_attempt, outcome).await;
|
|
superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded);
|
|
deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_)));
|
|
dirty_usage_generation_seen = dirty_generation_before_cycle;
|
|
if guard.is_lock_lost() {
|
|
record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await;
|
|
break;
|
|
}
|
|
let current_runtime_generation = scanner_runtime_config_generation();
|
|
let runtime_config_changed = current_runtime_generation != runtime_config_generation_seen;
|
|
runtime_config_generation_seen = current_runtime_generation;
|
|
if runtime_config_changed {
|
|
maintenance_generation_seen = None;
|
|
clean_idle_backoff.reset();
|
|
}
|
|
|
|
let runtime_config = resolve_scanner_runtime_config();
|
|
let current_maintenance_generation = scanner_maintenance_generation();
|
|
let maintenance_config_changed =
|
|
maintenance_generation_seen.is_some_and(|generation| generation != current_maintenance_generation);
|
|
let retry_failed_inspection = maintenance_inspection_retry.retry_due(maintenance_features, wake_reason, Instant::now());
|
|
if clean_idle_topology_supported && (maintenance_config_changed || retry_failed_inspection) {
|
|
let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else {
|
|
break;
|
|
};
|
|
maintenance_features = features;
|
|
maintenance_generation_seen = Some(generation);
|
|
maintenance_inspection_retry.record_inspection(features, Instant::now());
|
|
}
|
|
|
|
if runtime_config_changed {
|
|
clean_idle_backoff.reset();
|
|
continue;
|
|
}
|
|
if maintenance_config_changed {
|
|
scanner_activity_seen = None;
|
|
scanner_activity_backoff_blocked = scanner_activity_backoff_blocked_after_wake(
|
|
scanner_activity_backoff_blocked,
|
|
ScannerCycleWakeReason::MaintenanceConfig,
|
|
);
|
|
clean_idle_backoff.reset();
|
|
continue;
|
|
}
|
|
|
|
let scanner_activity_observation = if scanner_activity_probe_required(
|
|
clean_idle_topology_supported,
|
|
scanner_activity_backoff_blocked,
|
|
maintenance_features,
|
|
&runtime_config,
|
|
) {
|
|
observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await
|
|
} else {
|
|
scanner_activity_seen = None;
|
|
ScannerActivityObservation::NotRequired
|
|
};
|
|
if scanner_activity_observation == ScannerActivityObservation::MaintenanceChanged {
|
|
scanner_activity_backoff_blocked = true;
|
|
}
|
|
let scanner_activity_ready = !scanner_activity_backoff_blocked && scanner_activity_seen.is_some();
|
|
let backoff_enabled = scanner_clean_idle_backoff_enabled(
|
|
clean_idle_topology_supported,
|
|
scanner_activity_ready,
|
|
maintenance_features,
|
|
&runtime_config,
|
|
);
|
|
record_scanner_cycle_result(
|
|
&mut clean_idle_backoff,
|
|
&runtime_config,
|
|
backoff_enabled,
|
|
wake_reason,
|
|
outcome,
|
|
scanner_cycle_observed_dirty_work(
|
|
dirty_usage_pending_before_wait,
|
|
dirty_generation_before_wait,
|
|
dirty_usage_generation(),
|
|
) || scanner_activity_observed_work(scanner_activity_observation),
|
|
);
|
|
}
|
|
|
|
global_metrics().set_cycle(None).await;
|
|
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
|
|
|
|
debug!(
|
|
target: "rustfs::scanner",
|
|
event = EVENT_SCANNER_CYCLE_STATE,
|
|
component = LOG_COMPONENT_SCANNER,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "stopped",
|
|
"Scanner runtime stopped"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
struct ScannerScanModeGuard;
|
|
|
|
impl ScannerScanModeGuard {
|
|
fn new(scan_mode: HealScanMode) -> Self {
|
|
global_metrics().set_current_scan_mode(scan_mode);
|
|
Self
|
|
}
|
|
}
|
|
|
|
impl Drop for ScannerScanModeGuard {
|
|
fn drop(&mut self) {
|
|
global_metrics().clear_current_scan_mode();
|
|
}
|
|
}
|
|
|
|
async fn final_data_usage_publication_defer_reason<S>(storeapi: &S, status: ScannerCycleStatus) -> Option<ScannerCycleDeferReason>
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
match status {
|
|
ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded => {
|
|
if let Some(reason) = scanner_local_publication_defer_reason(storeapi).await {
|
|
return Some(reason);
|
|
}
|
|
if status == ScannerCycleStatus::Complete {
|
|
let distributed = storeapi.setup_is_dist_erasure().await;
|
|
match probe_scanner_activity(storeapi, distributed).await {
|
|
Ok(snapshot) if scanner_activity_allows_usage_publication(&snapshot) => None,
|
|
Ok(_) => Some(ScannerCycleDeferReason::DataMovement),
|
|
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
|
}
|
|
} else {
|
|
// A superseded cycle is explicitly observational and cannot
|
|
// replace the authoritative snapshot. It may still be
|
|
// persisted as a convergence baseline for the next cycle.
|
|
None
|
|
}
|
|
}
|
|
ScannerCycleStatus::Deferred(reason) => Some(reason),
|
|
// Incomplete cycles may publish a non-authoritative observational
|
|
// snapshot when at least one set has a usable current/LKG view.
|
|
ScannerCycleStatus::Incomplete => None,
|
|
}
|
|
}
|
|
|
|
async fn scanner_local_publication_defer_reason<S>(storeapi: &S) -> Option<ScannerCycleDeferReason>
|
|
where
|
|
S: ScannerStorage,
|
|
{
|
|
if !storeapi.scanner_data_usage_publication_blocked().await {
|
|
return None;
|
|
}
|
|
// Pending namespace commits invalidate this publication attempt, but only
|
|
// storage movement creates durable, rate-limited catch-up debt.
|
|
if storeapi.scanner_data_movement_pause_status().await.paused {
|
|
Some(ScannerCycleDeferReason::DataMovement)
|
|
} else {
|
|
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
|
}
|
|
}
|
|
|
|
fn scanner_post_lease_activity_defer_reason(
|
|
expected_digest: Option<[u8; 32]>,
|
|
activity: Result<ScannerActivitySnapshot, String>,
|
|
) -> Option<ScannerCycleDeferReason> {
|
|
match activity {
|
|
Ok(snapshot)
|
|
if scanner_activity_allows_usage_publication(&snapshot)
|
|
&& expected_digest == Some(scanner_activity_snapshot_digest(&snapshot)) =>
|
|
{
|
|
None
|
|
}
|
|
Ok(_) | Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
enum ScannerCyclePreCommitOutcome {
|
|
RecoverCacheCycle(u64),
|
|
Deferred(ScannerCycleDeferReason),
|
|
}
|
|
|
|
fn scanner_cycle_pre_commit_outcome(
|
|
required_cycle_floor: Option<u64>,
|
|
usage_persist_outcome: &DataUsagePersistOutcome,
|
|
) -> Option<ScannerCyclePreCommitOutcome> {
|
|
// Keep the publication barrier fail-closed: `.bloomcycle.bin` uses the
|
|
// same routed writer and its floor must remain pending while data movement
|
|
// hides the source pool.
|
|
match usage_persist_outcome {
|
|
DataUsagePersistOutcome::Deferred(reason) => Some(ScannerCyclePreCommitOutcome::Deferred(*reason)),
|
|
_ => required_cycle_floor.map(ScannerCyclePreCommitOutcome::RecoverCacheCycle),
|
|
}
|
|
}
|
|
|
|
fn update_scanner_cache_cycle_recovery_status(
|
|
requested_cycle: u64,
|
|
leader_epoch: u64,
|
|
required_cycle_floor: Option<u64>,
|
|
pre_commit_outcome: Option<ScannerCyclePreCommitOutcome>,
|
|
cache_scope_complete: bool,
|
|
) {
|
|
match (required_cycle_floor, pre_commit_outcome) {
|
|
(Some(required_cycle), _) => {
|
|
record_scanner_cache_cycle_ahead(requested_cycle, required_cycle, leader_epoch);
|
|
}
|
|
(None, Some(ScannerCyclePreCommitOutcome::Deferred(_))) => {
|
|
// A deferred scan may not have covered the cache that established
|
|
// the existing floor, so it cannot prove recovery is complete.
|
|
}
|
|
(None, _) if cache_scope_complete => clear_scanner_cache_cycle_ahead(),
|
|
(None, _) => {}
|
|
}
|
|
}
|
|
|
|
fn scanner_cycle_completion_outcome(
|
|
scan_status: ScannerCycleStatus,
|
|
usage_persist_outcome: DataUsagePersistOutcome,
|
|
has_dirty_usage: bool,
|
|
has_failed_dirty_usage: bool,
|
|
) -> ScannerCycleOutcome {
|
|
match (scan_status, usage_persist_outcome) {
|
|
(_, DataUsagePersistOutcome::Deferred(reason)) => ScannerCycleOutcome::Deferred(reason),
|
|
(_, DataUsagePersistOutcome::Failed) => ScannerCycleOutcome::Failed,
|
|
(ScannerCycleStatus::Deferred(reason), DataUsagePersistOutcome::NoUpdate)
|
|
if !has_dirty_usage && !has_failed_dirty_usage =>
|
|
{
|
|
ScannerCycleOutcome::Deferred(reason)
|
|
}
|
|
(ScannerCycleStatus::Deferred(_), _) => ScannerCycleOutcome::Failed,
|
|
(ScannerCycleStatus::Superseded, _) if !has_failed_dirty_usage => ScannerCycleOutcome::Superseded,
|
|
(ScannerCycleStatus::Superseded, _) => ScannerCycleOutcome::Failed,
|
|
(
|
|
ScannerCycleStatus::Incomplete,
|
|
DataUsagePersistOutcome::Saved | DataUsagePersistOutcome::AlreadyDurable | DataUsagePersistOutcome::PriorCycleDurable,
|
|
) if !has_failed_dirty_usage => ScannerCycleOutcome::Partial,
|
|
(ScannerCycleStatus::Incomplete, _) => ScannerCycleOutcome::Failed,
|
|
(
|
|
ScannerCycleStatus::Complete,
|
|
DataUsagePersistOutcome::Saved | DataUsagePersistOutcome::AlreadyDurable | DataUsagePersistOutcome::PriorCycleDurable,
|
|
) => ScannerCycleOutcome::Completed,
|
|
(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current) if !has_dirty_usage => ScannerCycleOutcome::Completed,
|
|
(ScannerCycleStatus::Complete, _) => ScannerCycleOutcome::Failed,
|
|
}
|
|
}
|
|
|
|
fn finalize_scanner_cycle_result(
|
|
scan_cycle_result: crate::scanner_io::ScannerCycleResult,
|
|
usage_persist_outcome: DataUsagePersistOutcome,
|
|
) -> (ScannerCycleOutcome, bool, Vec<ScannerDirtyUsageAcknowledgement>) {
|
|
let completion_outcome = scanner_cycle_completion_outcome_for_result(&scan_cycle_result, usage_persist_outcome);
|
|
let pending_maintenance_work = scan_cycle_result.has_pending_maintenance_work();
|
|
let durable_complete_snapshot = scan_cycle_result.status == ScannerCycleStatus::Complete
|
|
&& matches!(
|
|
usage_persist_outcome,
|
|
DataUsagePersistOutcome::Saved | DataUsagePersistOutcome::AlreadyDurable
|
|
);
|
|
let remote_dirty_usage_acknowledgements = if durable_complete_snapshot {
|
|
scan_cycle_result.acknowledge_durable_usage()
|
|
} else {
|
|
Vec::new()
|
|
};
|
|
(completion_outcome, pending_maintenance_work, remote_dirty_usage_acknowledgements)
|
|
}
|
|
|
|
fn scanner_cycle_completion_outcome_for_result(
|
|
scan_cycle_result: &crate::scanner_io::ScannerCycleResult,
|
|
usage_persist_outcome: DataUsagePersistOutcome,
|
|
) -> ScannerCycleOutcome {
|
|
let has_dirty_usage = scan_cycle_result.has_dirty_usage_to_acknowledge();
|
|
let has_failed_dirty_usage = scan_cycle_result.has_failed_dirty_usage();
|
|
if scan_cycle_result.has_observational_snapshot()
|
|
&& matches!(
|
|
scan_cycle_result.status,
|
|
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
|
)
|
|
{
|
|
return match usage_persist_outcome {
|
|
DataUsagePersistOutcome::Saved
|
|
| DataUsagePersistOutcome::AlreadyDurable
|
|
| DataUsagePersistOutcome::PriorCycleDurable
|
|
| DataUsagePersistOutcome::Current
|
|
if !has_failed_dirty_usage =>
|
|
{
|
|
ScannerCycleOutcome::Partial
|
|
}
|
|
DataUsagePersistOutcome::Deferred(reason) => ScannerCycleOutcome::Deferred(reason),
|
|
_ => ScannerCycleOutcome::Failed,
|
|
};
|
|
}
|
|
scanner_cycle_completion_outcome(scan_cycle_result.status, usage_persist_outcome, has_dirty_usage, has_failed_dirty_usage)
|
|
}
|
|
|
|
/// Decide whether an incoming usage snapshot must be skipped as stale, given the local
|
|
/// wall clock `now`. Mirrors `stale_data_usage_persist_reason` in
|
|
/// `crates/ecstore/src/data_usage/mod.rs` — keep the two consistent.
|
|
///
|
|
/// If the persisted `existing.last_update` is future-dated beyond
|
|
/// [`rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE`] (clock step-back or a
|
|
/// slower-clock scanner leader), it is untrustworthy: the save is allowed so usage
|
|
/// stats cannot freeze forever.
|
|
fn stale_data_usage_update_reason(
|
|
incoming: &DataUsageInfo,
|
|
existing: &DataUsageInfo,
|
|
now: std::time::SystemTime,
|
|
) -> Option<&'static str> {
|
|
match (incoming.scanner_epoch, existing.scanner_epoch) {
|
|
(Some(incoming_epoch), Some(existing_epoch)) if incoming_epoch < existing_epoch => {
|
|
return Some("older_scanner_epoch");
|
|
}
|
|
(Some(incoming_epoch), Some(existing_epoch)) if incoming_epoch > existing_epoch => return None,
|
|
(Some(_), None) => return None,
|
|
(None, Some(_)) => return Some("missing_incoming_scanner_epoch"),
|
|
(Some(_), Some(_)) | (None, None) => {}
|
|
}
|
|
|
|
match (incoming.scanner_cycle, existing.scanner_cycle) {
|
|
(Some(incoming_cycle), Some(existing_cycle)) if incoming_cycle < existing_cycle => {
|
|
return Some("older_scanner_cycle");
|
|
}
|
|
(Some(incoming_cycle), Some(existing_cycle)) if incoming_cycle == existing_cycle => {
|
|
return Some("conflicting_same_scanner_cycle");
|
|
}
|
|
(Some(_), Some(_)) | (Some(_), None) => return None,
|
|
(None, Some(_)) => return Some("missing_incoming_scanner_cycle"),
|
|
(None, None) => {}
|
|
}
|
|
|
|
match (incoming.last_update, existing.last_update) {
|
|
(Some(new_ts), Some(existing_ts))
|
|
if new_ts <= existing_ts && !rustfs_data_usage::usage_last_update_is_untrusted_future(existing_ts, now) =>
|
|
{
|
|
Some("older_or_equal_last_update")
|
|
}
|
|
(None, Some(_)) => Some("missing_incoming_last_update"),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
fn data_usage_reintroduces_missing_bucket(incoming: &DataUsageInfo, existing: Option<&DataUsageInfo>) -> bool {
|
|
let Some(existing) = existing else {
|
|
return !incoming.buckets_usage.is_empty() || !incoming.bucket_sizes.is_empty();
|
|
};
|
|
incoming
|
|
.buckets_usage
|
|
.keys()
|
|
.chain(incoming.bucket_sizes.keys())
|
|
.any(|bucket| !existing.buckets_usage.contains_key(bucket) && !existing.bucket_sizes.contains_key(bucket))
|
|
}
|
|
|
|
/// Store data usage info in backend. Will store all objects sent on the receiver until closed.
|
|
mod activity;
|
|
mod backlog;
|
|
mod cycle_state;
|
|
mod heal_info;
|
|
mod leadership;
|
|
mod usage_store;
|
|
|
|
use activity::*;
|
|
use backlog::*;
|
|
use cycle_state::*;
|
|
use leadership::*;
|
|
use usage_store::*;
|
|
|
|
pub use activity::scanner_topology_digest;
|
|
pub(crate) use activity::{
|
|
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
|
|
scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest,
|
|
scanner_activity_structural_digest, scanner_dirty_usage_acknowledgements,
|
|
};
|
|
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
|
|
pub use backlog::{
|
|
ScannerPauseBacklogAlertReason, ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds,
|
|
scanner_pause_backlog_status,
|
|
};
|
|
#[cfg(test)]
|
|
pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
|
|
pub use cycle_state::{
|
|
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerUsageStateResetResult, reset_scanner_cycle_recovery,
|
|
reset_scanner_usage_state_for_full_rebuild, scanner_cycle_recovery_status,
|
|
};
|
|
pub(crate) use cycle_state::{
|
|
current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence, load_scanner_cycle_state_for_startup,
|
|
};
|
|
pub use heal_info::{BackgroundHealInfo, read_background_heal_info, save_background_heal_info};
|
|
pub use usage_store::store_data_usage_in_backend;
|
|
|
|
#[cfg(test)]
|
|
mod tests;
|