feat(scanner): coordinate usage and workload boundaries (#7093)

* test(scanner): wire usage and heal rebuild gates

* docs(scanner): define usage authority protocol

* docs(heal): clarify scanner and ecstore boundaries

* refactor(scanner): split metrics from contracts

* feat(scanner): use shared workload snapshots

* fix(ecstore): recheck capacity before decommission drain
This commit is contained in:
houseme
2026-09-03 17:02:43 +08:00
committed by GitHub
parent 3ab7a1921f
commit 0e6ee3bf62
57 changed files with 632 additions and 134 deletions
+2 -1
View File
@@ -73,8 +73,9 @@ hotpath-cpu = [
hotpath.workspace = true
rustfs-config = { workspace = true, features = ["server-config-model"] }
rustfs-common = { workspace = true }
rustfs-concurrency = { workspace = true }
rustfs-heal-contracts = { workspace = true }
rustfs-scanner-contracts = { workspace = true }
rustfs-scanner-metrics = { workspace = true }
rustfs-credentials = { workspace = true }
rustfs-utils = { workspace = true }
tokio = { workspace = true, features = ["fs", "sync", "time", "macros", "rt-multi-thread"] }
+3 -1
View File
@@ -70,6 +70,7 @@ mod scanner_heal_admission_baseline;
pub mod scanner_io;
pub mod sleeper;
pub(crate) mod storage_api;
mod workload_admission;
pub use data_usage_define::*;
pub use error::ScannerError;
@@ -80,7 +81,7 @@ pub use remote_scanner::{
remote_scanner_request_matches_envelope, serve_remote_scanner_request, validate_remote_scanner_request_fence,
};
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
pub use rustfs_scanner_contracts::last_minute;
pub use rustfs_scanner_metrics::last_minute;
pub use scanner::{
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason,
ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, ScannerUsageStateResetResult,
@@ -96,6 +97,7 @@ pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
use std::sync::atomic::{AtomicU64, Ordering};
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
+1 -1
View File
@@ -28,7 +28,7 @@ use crate::{
use hmac::{Hmac, KeyInit, Mac};
use rustfs_credentials::try_get_rpc_token;
use rustfs_heal_contracts::heal_channel::HealScanMode;
use rustfs_scanner_contracts::metrics::{Metric, Metrics};
use rustfs_scanner_metrics::metrics::{Metric, Metrics};
use rustfs_utils::path::path_join_buf;
use serde::{Deserialize, Serialize};
use sha2::Sha256;
+1 -1
View File
@@ -52,7 +52,7 @@ use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELA
use rustfs_data_usage::observed_data_usage_is_newer;
use rustfs_heal_contracts::heal_channel::HealScanMode;
use rustfs_lock::{NamespaceLockGuard, error::LockError};
use rustfs_scanner_contracts::metrics::{
use rustfs_scanner_metrics::metrics::{
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded,
global_metrics,
+1 -1
View File
@@ -50,7 +50,7 @@ use rustfs_heal_contracts::heal_channel::{
HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest,
HealRequestSource, HealScanMode, send_heal_request_with_admission,
};
use rustfs_scanner_contracts::metrics::{
use rustfs_scanner_metrics::metrics::{
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
UpdateCurrentPathFn, current_path_updater, global_metrics,
};
+1 -1
View File
@@ -30,7 +30,7 @@ use rustfs_data_usage::{BucketTargetUsageInfo, BucketUsageInfo};
use rustfs_filemeta::FileMeta;
use rustfs_heal_contracts::heal_channel::HealScanMode;
use rustfs_lock::{LockError, NamespaceLockGuard};
use rustfs_scanner_contracts::metrics::{
use rustfs_scanner_metrics::metrics::{
Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics,
};
use rustfs_utils::path::path_join_buf;
+4 -4
View File
@@ -147,13 +147,13 @@ impl Drop for DiskBucketScanActiveGuard {
pub(super) struct BucketDriveFailureGuard {
failed: bool,
source: rustfs_scanner_contracts::metrics::ScannerWorkSource,
source: rustfs_scanner_metrics::metrics::ScannerWorkSource,
bucket: String,
drive: String,
}
impl BucketDriveFailureGuard {
pub(super) fn new(source: rustfs_scanner_contracts::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
pub(super) fn new(source: rustfs_scanner_metrics::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
Self {
failed: true,
source,
@@ -245,7 +245,7 @@ pub(super) fn scanner_concurrency_limit(configured: usize, available: usize) ->
return 0;
}
if crate::current_foreground_read_activity() > 0 {
if crate::workload_admission::foreground_workload_activity() > 0 {
return 1;
}
@@ -285,7 +285,7 @@ pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError)
#[cfg(test)]
mod tests {
use super::*;
use rustfs_scanner_contracts::metrics::{ScannerWorkSource, global_metrics};
use rustfs_scanner_metrics::metrics::{ScannerWorkSource, global_metrics};
use tokio::sync::oneshot;
fn active_bucket_drive_count(source: ScannerWorkSource, bucket: &str, drive: &str) -> u64 {
+2 -2
View File
@@ -161,8 +161,8 @@ impl ScannerIODisk for Disk {
let bucket = cache.info.name.clone();
let disk_path = self.path().to_string_lossy().to_string();
let source = match scan_mode {
HealScanMode::Deep => rustfs_scanner_contracts::metrics::ScannerWorkSource::Bitrot,
HealScanMode::Normal | HealScanMode::Unknown => rustfs_scanner_contracts::metrics::ScannerWorkSource::Usage,
HealScanMode::Deep => rustfs_scanner_metrics::metrics::ScannerWorkSource::Bitrot,
HealScanMode::Normal | HealScanMode::Unknown => rustfs_scanner_metrics::metrics::ScannerWorkSource::Usage,
};
global_metrics().record_scan_bucket_drive_start(source, &bucket, &disk_path);
let mut failure_guard = BucketDriveFailureGuard::new(source, &bucket, &disk_path);
+41
View File
@@ -26,12 +26,32 @@ use crate::{
ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests,
init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
};
use rustfs_concurrency::{
AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider,
WorkloadClass,
};
use rustfs_filemeta::FileInfo;
use serial_test::serial;
use std::sync::Arc;
use temp_env::with_var;
use time::OffsetDateTime;
use uuid::Uuid;
#[derive(Clone)]
struct FixedWorkloadProvider {
snapshot: WorkloadAdmissionRegistrySnapshot,
}
impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider {
fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
self.snapshot.clone()
}
}
fn install_scanner_workload_provider(snapshot: WorkloadAdmissionRegistrySnapshot) {
crate::set_scanner_workload_admission_snapshot_provider(Arc::new(FixedWorkloadProvider { snapshot }));
}
fn bucket_info(name: &str) -> BucketInfo {
BucketInfo {
name: name.to_string(),
@@ -1079,6 +1099,7 @@ async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() {
#[serial]
fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
assert_eq!(scanner_concurrency_limit(0, 4), 4);
}
@@ -1086,6 +1107,7 @@ fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
#[serial]
fn scanner_concurrency_limit_caps_to_configured_value() {
crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
assert_eq!(scanner_concurrency_limit(2, 4), 2);
}
@@ -1093,6 +1115,7 @@ fn scanner_concurrency_limit_caps_to_configured_value() {
#[serial]
fn scanner_concurrency_limit_never_exceeds_available_work() {
crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
assert_eq!(scanner_concurrency_limit(8, 4), 4);
}
@@ -1100,6 +1123,7 @@ fn scanner_concurrency_limit_never_exceeds_available_work() {
#[serial]
fn scanner_concurrency_limit_handles_no_available_work() {
crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
assert_eq!(scanner_concurrency_limit(2, 0), 0);
}
@@ -1107,16 +1131,33 @@ fn scanner_concurrency_limit_handles_no_available_work() {
#[serial]
fn scanner_concurrency_limit_yields_to_foreground_reads() {
crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
crate::set_foreground_read_activity(8);
assert_eq!(scanner_concurrency_limit(0, 4), 1);
assert_eq!(scanner_concurrency_limit(3, 4), 1);
crate::reset_foreground_read_activity_for_test();
}
#[test]
#[serial]
fn scanner_concurrency_limit_yields_to_shared_foreground_pressure() {
crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
install_scanner_workload_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Open).with_counts(Some(2), None, Some(16)),
]));
assert_eq!(scanner_concurrency_limit(0, 4), 1);
assert_eq!(scanner_concurrency_limit(3, 4), 1);
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
}
#[test]
#[serial]
fn scanner_concurrency_limit_yields_to_streaming_reads() {
crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
let _guard = crate::ForegroundReadGuard::new();
assert_eq!(scanner_concurrency_limit(0, 4), 1);
+18 -16
View File
@@ -20,7 +20,7 @@ use rustfs_config::{
DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED,
ENV_SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
};
use rustfs_scanner_contracts::metrics::global_metrics;
use rustfs_scanner_metrics::metrics::global_metrics;
use tokio::time::Duration;
const MIN_SLEEP: Duration = Duration::from_millis(1);
@@ -29,8 +29,8 @@ const SCANNER_SPEED_FAST: u8 = 1;
const SCANNER_SPEED_DEFAULT: u8 = 2;
const SCANNER_SPEED_SLOW: u8 = 3;
const SCANNER_SPEED_SLOWEST: u8 = 4;
const FOREGROUND_READ_BACKOFF_PER_REQUEST_MS: u64 = 10;
const FOREGROUND_READ_BACKOFF_MAX_MS: u64 = 250;
const FOREGROUND_WORKLOAD_BACKOFF_PER_REQUEST_MS: u64 = 10;
const FOREGROUND_WORKLOAD_BACKOFF_MAX_MS: u64 = 250;
static SCANNER_DEFAULT_SPEED_PRESET: AtomicU8 = AtomicU8::new(SCANNER_SPEED_DEFAULT);
@@ -78,15 +78,15 @@ pub(crate) fn scanner_yield_every_n_objects() -> u64 {
rustfs_utils::get_env_u64(ENV_SCANNER_YIELD_EVERY_N_OBJECTS, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS)
}
fn foreground_read_backoff_duration(active_reads: u64) -> Duration {
if active_reads == 0 {
fn foreground_workload_backoff_duration(active_foreground_workloads: u64) -> Duration {
if active_foreground_workloads == 0 {
return Duration::ZERO;
}
Duration::from_millis(
active_reads
.saturating_mul(FOREGROUND_READ_BACKOFF_PER_REQUEST_MS)
.min(FOREGROUND_READ_BACKOFF_MAX_MS),
active_foreground_workloads
.saturating_mul(FOREGROUND_WORKLOAD_BACKOFF_PER_REQUEST_MS)
.min(FOREGROUND_WORKLOAD_BACKOFF_MAX_MS),
)
}
@@ -147,14 +147,15 @@ impl DynamicSleeper {
}
let (factor, max_sleep) = self.read_params();
if factor == 0.0 || max_sleep.is_zero() {
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
let foreground_sleep =
foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
if !foreground_sleep.is_zero() {
tokio::time::sleep(foreground_sleep).await;
global_metrics().record_scanner_throttle_sleep(foreground_sleep);
}
return;
}
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
let foreground_sleep = foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor)
.min(max_sleep)
.max(foreground_sleep);
@@ -235,7 +236,8 @@ impl SleepTimer {
}
let (factor, max_sleep) = self.sleeper.read_params();
if factor == 0.0 || max_sleep.is_zero() {
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
let foreground_sleep =
foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
if !foreground_sleep.is_zero() {
tokio::time::sleep(foreground_sleep).await;
global_metrics().record_scanner_throttle_sleep(foreground_sleep);
@@ -243,7 +245,7 @@ impl SleepTimer {
return;
}
let elapsed = self.start.elapsed();
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
let foreground_sleep = foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
let sleep_dur = Duration::from_secs_f64(elapsed.as_secs_f64() * factor)
.max(MIN_SLEEP)
.min(max_sleep)
@@ -304,10 +306,10 @@ mod tests {
}
#[test]
fn foreground_read_backoff_is_capped() {
assert_eq!(foreground_read_backoff_duration(0), Duration::ZERO);
assert_eq!(foreground_read_backoff_duration(1), Duration::from_millis(10));
assert_eq!(foreground_read_backoff_duration(80), Duration::from_millis(250));
fn foreground_workload_backoff_is_capped() {
assert_eq!(foreground_workload_backoff_duration(0), Duration::ZERO);
assert_eq!(foreground_workload_backoff_duration(1), Duration::from_millis(10));
assert_eq!(foreground_workload_backoff_duration(80), Duration::from_millis(250));
}
#[test]
+165
View File
@@ -0,0 +1,165 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::{Arc, LazyLock, RwLock};
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshotProvider, WorkloadClass};
type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
static SCANNER_WORKLOAD_ADMISSION_PROVIDER: LazyLock<RwLock<Option<WorkloadSnapshotProviderRef>>> =
LazyLock::new(|| RwLock::new(None));
pub fn set_scanner_workload_admission_snapshot_provider(provider: WorkloadSnapshotProviderRef) {
*SCANNER_WORKLOAD_ADMISSION_PROVIDER
.write()
.unwrap_or_else(|err| err.into_inner()) = Some(provider);
}
fn scanner_workload_admission_snapshot_provider() -> Option<WorkloadSnapshotProviderRef> {
SCANNER_WORKLOAD_ADMISSION_PROVIDER
.read()
.unwrap_or_else(|err| err.into_inner())
.clone()
}
#[cfg(test)]
pub(crate) fn clear_scanner_workload_admission_snapshot_provider_for_test() {
*SCANNER_WORKLOAD_ADMISSION_PROVIDER
.write()
.unwrap_or_else(|err| err.into_inner()) = None;
}
pub(crate) fn foreground_workload_activity() -> u64 {
let local_activity = crate::current_foreground_read_activity();
let Some(provider) = scanner_workload_admission_snapshot_provider() else {
return local_activity;
};
local_activity.max(foreground_activity_from_snapshot(&provider.workload_admission_snapshot()))
}
fn foreground_activity_from_snapshot(snapshot: &WorkloadAdmissionRegistrySnapshot) -> u64 {
[WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite]
.into_iter()
.filter_map(|class| snapshot.get(class))
.map(|entry| {
entry
.active
.or_else(|| {
matches!(entry.state, AdmissionState::Saturated).then(|| entry.limit.filter(|limit| *limit > 0).unwrap_or(1))
})
.map(usize_to_u64_saturated)
.unwrap_or(0)
})
.max()
.unwrap_or(0)
}
fn usize_to_u64_saturated(value: usize) -> u64 {
u64::try_from(value).unwrap_or(u64::MAX)
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshot};
use serial_test::serial;
#[derive(Clone)]
struct FixedWorkloadProvider {
snapshot: WorkloadAdmissionRegistrySnapshot,
}
impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider {
fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
self.snapshot.clone()
}
}
fn install_provider(snapshot: WorkloadAdmissionRegistrySnapshot) {
set_scanner_workload_admission_snapshot_provider(Arc::new(FixedWorkloadProvider { snapshot }));
}
#[test]
#[serial]
fn foreground_workload_activity_falls_back_to_local_read_activity() {
clear_scanner_workload_admission_snapshot_provider_for_test();
crate::reset_foreground_read_activity_for_test();
crate::set_foreground_read_activity(3);
assert_eq!(foreground_workload_activity(), 3);
crate::reset_foreground_read_activity_for_test();
}
#[test]
#[serial]
fn foreground_workload_activity_uses_shared_provider_counts() {
clear_scanner_workload_admission_snapshot_provider_for_test();
crate::reset_foreground_read_activity_for_test();
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Open).with_counts(
Some(5),
None,
Some(8),
),
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Open).with_counts(
Some(2),
None,
Some(4),
),
]));
assert_eq!(foreground_workload_activity(), 5);
clear_scanner_workload_admission_snapshot_provider_for_test();
}
#[test]
#[serial]
fn foreground_workload_activity_treats_saturation_without_counts_as_pressure() {
clear_scanner_workload_admission_snapshot_provider_for_test();
crate::reset_foreground_read_activity_for_test();
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Saturated).with_counts(
None,
None,
Some(7),
),
]));
assert_eq!(foreground_workload_activity(), 7);
clear_scanner_workload_admission_snapshot_provider_for_test();
}
#[test]
#[serial]
fn foreground_workload_activity_treats_zero_limit_saturation_as_pressure() {
clear_scanner_workload_admission_snapshot_provider_for_test();
crate::reset_foreground_read_activity_for_test();
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Saturated).with_counts(
None,
None,
Some(0),
),
]));
assert_eq!(foreground_workload_activity(), 1);
clear_scanner_workload_admission_snapshot_provider_for_test();
}
}