feat(scanner): coordinate usage and workload boundaries (#7093)

* test(scanner): wire usage and heal rebuild gates

* docs(scanner): define usage authority protocol

* docs(heal): clarify scanner and ecstore boundaries

* refactor(scanner): split metrics from contracts

* feat(scanner): use shared workload snapshots

* fix(ecstore): recheck capacity before decommission drain
This commit is contained in:
houseme
2026-09-03 17:02:43 +08:00
committed by GitHub
parent 3ab7a1921f
commit 0e6ee3bf62
57 changed files with 632 additions and 134 deletions
+1 -1
View File
@@ -130,7 +130,7 @@ rustfs-concurrency.workspace = true
rustfs-credentials = { workspace = true }
rustfs-common.workspace = true
rustfs-heal-contracts.workspace = true
rustfs-scanner-contracts.workspace = true
rustfs-scanner-metrics.workspace = true
rustfs-policy.workspace = true
rustfs-protos.workspace = true
rustfs-replication.workspace = true
@@ -17,7 +17,7 @@ use crate::bucket::lifecycle::lifecycle;
use crate::object_api::ObjectInfo;
use crate::services::event_notification::{EventArgs, send_event};
use rustfs_s3_types::EventName;
use rustfs_scanner_contracts::metrics::IlmAction;
use rustfs_scanner_metrics::metrics::IlmAction;
const LIFECYCLE_EXPIRY_USER_AGENT: &str = "Internal: [ILM-Expiry]";
const LIFECYCLE_TRANSITION_USER_AGENT: &str = "Internal: [ILM-Transition]";
@@ -84,7 +84,7 @@ use rustfs_data_usage::TierStats;
use rustfs_filemeta::{
FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info, is_restored_object_on_disk,
};
use rustfs_scanner_contracts::metrics::{
use rustfs_scanner_metrics::metrics::{
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
};
use rustfs_utils::{
@@ -5596,7 +5596,7 @@ mod tests {
use rustfs_filemeta::{FileInfo, FileMeta};
#[cfg(feature = "test-util")]
use rustfs_s3_client::transition_api::ReaderImpl;
use rustfs_scanner_contracts::metrics::{IlmAction, global_metrics};
use rustfs_scanner_metrics::metrics::{IlmAction, global_metrics};
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry,
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, OutputLocation, RestoreRequest,
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_scanner_contracts::metrics::IlmAction;
use rustfs_scanner_metrics::metrics::IlmAction;
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
use crate::bucket::replication::ReplicationLifecycleBridge;
@@ -77,7 +77,7 @@ mod tests {
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::storage_api_contracts::object::ObjectToDelete;
use rustfs_scanner_contracts::metrics::IlmAction;
use rustfs_scanner_metrics::metrics::IlmAction;
use s3s::dto::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
+5 -5
View File
@@ -16,7 +16,7 @@ use super::{BucketQuota, QuotaCheckResult, QuotaError, QuotaOperation};
use crate::bucket::metadata_sys::{BucketMetadataSys, update, update_if_incarnation};
use crate::data_usage::get_bucket_usage_memory;
use rustfs_config::QUOTA_CONFIG_FILE;
use rustfs_scanner_contracts::metrics::Metric;
use rustfs_scanner_metrics::metrics::Metric;
use std::sync::Arc;
use std::time::Instant;
use time::OffsetDateTime;
@@ -120,9 +120,9 @@ impl QuotaChecker {
let duration = start_time.elapsed();
// inc_time is now a plain fn (not async) — no .await needed.
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaCheck, duration);
rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaCheck, duration);
if !allowed {
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaViolation, duration);
rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaViolation, duration);
}
Ok(result)
@@ -185,7 +185,7 @@ impl QuotaChecker {
.await
.map_err(QuotaError::StorageError)?;
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
Ok(updated_at)
}
@@ -206,7 +206,7 @@ impl QuotaChecker {
}
.map_err(QuotaError::StorageError)?;
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
Ok(updated_at)
}
+43 -1
View File
@@ -13504,7 +13504,9 @@ impl ECStore {
}
return Ok(());
}
let result = self.decommission_in_background(rx.clone(), idx, entry_budget).await;
let result = self
.decommission_in_background(rx.clone(), idx, generation, entry_budget)
.await;
if let Err(err) = &result
&& (is_decommission_capacity_blocked_error(err) || is_decommission_target_capacity_error(err))
@@ -13989,8 +13991,10 @@ impl ECStore {
self: &Arc<Self>,
rx: CancellationToken,
idx: usize,
generation: OffsetDateTime,
entry_budget: Arc<Semaphore>,
) -> Result<()> {
self.ensure_decommission_runtime_capacity_available(idx, generation).await?;
let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone();
let pending = {
@@ -17253,6 +17257,44 @@ mod tests {
);
}
#[tokio::test]
#[serial_test::serial]
async fn decommission_worker_rechecks_runtime_capacity_before_empty_background_completion() {
let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let enough = vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
];
set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("the initial reservation should be activated");
let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)];
set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]);
let canceler = DecommissionCanceler::new(CancellationToken::new());
store.decommission_cancelers.write().await[0] = Some(canceler.clone());
store
.do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
.await
.expect("runtime capacity shortage should pause the worker before background completion");
let local = store.pool_meta.read().await;
let info = local.pools[0]
.decommission
.as_ref()
.expect("the blocked decommission state should remain present");
assert!(!info.complete && !info.failed && !info.canceled);
assert!(info.capacity_blocked_reason.is_some());
assert!(
info.capacity_reservation
.as_ref()
.is_some_and(DecommissionCapacityReservation::active)
);
}
fn pool_meta_replica_test_meta(cmd_line: &str) -> PoolMeta {
PoolMeta {
version: POOL_META_VERSION,
+1 -1
View File
@@ -286,7 +286,7 @@ pub struct QuotaAdmission {
pub struct LifecycleDeleteAllRequest {
pub(crate) version_id: Option<Uuid>,
pub(crate) delete_marker: bool,
pub(crate) action: rustfs_scanner_contracts::metrics::IlmAction,
pub(crate) action: rustfs_scanner_metrics::metrics::IlmAction,
pub(crate) rule_id: String,
pub(crate) phase: LifecycleDeleteAllPhase,
}
+26 -26
View File
@@ -32,7 +32,7 @@ use rustfs_madmin::metrics::{
ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot,
ScannerUsageFreshnessSnapshot as MadminScannerUsageFreshnessSnapshot, TimedAction as MadminTimedAction,
};
use rustfs_scanner_contracts::metrics::global_metrics;
use rustfs_scanner_metrics::metrics::global_metrics;
use rustfs_utils::os::get_drive_stats;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
@@ -82,7 +82,7 @@ fn unix_millis_to_jiff_timestamp(millis: u64, fallback: Timestamp) -> Timestamp
}
}
fn to_madmin_scanner_metrics(metrics: rustfs_scanner_contracts::metrics::ScannerMetricsReport) -> MadminScannerMetrics {
fn to_madmin_scanner_metrics(metrics: rustfs_scanner_metrics::metrics::ScannerMetricsReport) -> MadminScannerMetrics {
MadminScannerMetrics {
collected_at: metrics.collected_at,
current_cycle: metrics.current_cycle,
@@ -565,7 +565,7 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
mod test {
use super::*;
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_scanner_contracts::metrics::CurrentCycle;
use rustfs_scanner_metrics::metrics::CurrentCycle;
use serial_test::serial;
use std::time::Duration;
@@ -619,7 +619,7 @@ mod test {
#[test]
fn scanner_metrics_mapping_preserves_partial_source_status() {
let current_started = Utc::now() - chrono::Duration::seconds(5);
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
current_cycle_active: true,
current_started: chrono_to_jiff_timestamp(current_started),
last_cycle_partial_source: "usage".to_string(),
@@ -628,7 +628,7 @@ mod test {
cycle_recovery_required_total: 2,
cycle_last_progress_age: 17,
leader_lease_without_progress: true,
partial_cycles_by_source: vec![rustfs_scanner_contracts::metrics::ScannerSourceCycleSnapshot {
partial_cycles_by_source: vec![rustfs_scanner_metrics::metrics::ScannerSourceCycleSnapshot {
source: "usage".to_string(),
cycles: 2,
}],
@@ -654,11 +654,11 @@ mod test {
#[tokio::test]
#[serial]
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
let previous_init_time = *rustfs_scanner_contracts::GLOBAL_INIT_TIME.read().await;
let previous_init_time = *rustfs_scanner_metrics::GLOBAL_INIT_TIME.read().await;
let previous_cycle = global_metrics().get_cycle().await;
let init_time = Utc::now() - chrono::Duration::hours(1);
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
*rustfs_scanner_contracts::GLOBAL_INIT_TIME.write().await = Some(init_time);
*rustfs_scanner_metrics::GLOBAL_INIT_TIME.write().await = Some(init_time);
let cycle = CurrentCycle {
current: 0,
next: 1,
@@ -673,7 +673,7 @@ mod test {
.finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default())
.await;
global_metrics().set_cycle(previous_cycle).await;
*rustfs_scanner_contracts::GLOBAL_INIT_TIME.write().await = previous_init_time;
*rustfs_scanner_metrics::GLOBAL_INIT_TIME.write().await = previous_init_time;
let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode");
let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode");
@@ -686,8 +686,8 @@ mod test {
#[test]
fn scanner_metrics_mapping_preserves_pacing_pressure() {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
pacing_pressure: rustfs_scanner_contracts::metrics::ScannerPacingPressureSnapshot {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
pacing_pressure: rustfs_scanner_metrics::metrics::ScannerPacingPressureSnapshot {
primary_pressure: "cycle_budget".to_string(),
current_queued_scans: 4,
current_active_scans: 2,
@@ -712,12 +712,12 @@ mod test {
#[test]
fn scanner_metrics_mapping_preserves_lifecycle_transition_status() {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
current_cycle_lifecycle_expiry_actions: 2,
current_cycle_lifecycle_transition_actions: 3,
last_cycle_lifecycle_expiry_actions: 5,
last_cycle_lifecycle_transition_actions: 7,
lifecycle_expiry: rustfs_scanner_contracts::metrics::ScannerLifecycleExpirySnapshot {
lifecycle_expiry: rustfs_scanner_metrics::metrics::ScannerLifecycleExpirySnapshot {
current_queue_capacity: 16,
current_queued: 5,
current_active: 2,
@@ -729,7 +729,7 @@ mod test {
scanner_not_enqueued: 2,
delete_failed: 1,
},
lifecycle_transition: rustfs_scanner_contracts::metrics::ScannerLifecycleTransitionSnapshot {
lifecycle_transition: rustfs_scanner_metrics::metrics::ScannerLifecycleTransitionSnapshot {
current_queue_capacity: 16,
current_queued: 5,
current_active: 2,
@@ -778,10 +778,10 @@ mod test {
#[test]
fn scanner_metrics_mapping_preserves_maintenance_control_status() {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
maintenance_control: rustfs_scanner_contracts::metrics::ScannerMaintenanceControlSnapshot {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
maintenance_control: rustfs_scanner_metrics::metrics::ScannerMaintenanceControlSnapshot {
primary_control: "blocked_source".to_string(),
sources: vec![rustfs_scanner_contracts::metrics::ScannerMaintenanceSourceSnapshot {
sources: vec![rustfs_scanner_metrics::metrics::ScannerMaintenanceSourceSnapshot {
source: "lifecycle".to_string(),
state: "blocked".to_string(),
reason: "missed_work".to_string(),
@@ -815,8 +815,8 @@ mod test {
#[test]
fn scanner_metrics_mapping_preserves_usage_freshness_status() {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
usage_freshness: rustfs_scanner_contracts::metrics::ScannerUsageFreshnessSnapshot {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
usage_freshness: rustfs_scanner_metrics::metrics::ScannerUsageFreshnessSnapshot {
dirty_pending_buckets: 3,
last_dirty_mark_unix_secs: 10,
last_dirty_clear_unix_secs: 11,
@@ -857,7 +857,7 @@ mod test {
#[test]
fn scanner_metrics_mapping_preserves_distributed_status_fields() {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
active_scan_paths: 2,
oldest_active_path_age_seconds: 45,
active_paths: vec!["disk-a/bucket-a".to_string(), "disk-b/bucket-b".to_string()],
@@ -913,7 +913,7 @@ mod test {
cycle_max_directories: 38,
bitrot_cycle_enabled: true,
bitrot_cycle_seconds: 39.0,
scan_checkpoint: Some(rustfs_scanner_contracts::metrics::ScannerCheckpointReport {
scan_checkpoint: Some(rustfs_scanner_metrics::metrics::ScannerCheckpointReport {
version: 1,
resume_after: "bucket-a/prefix-a".to_string(),
reason: "directories".to_string(),
@@ -923,7 +923,7 @@ mod test {
scan_checkpoint_cleared: 41,
scan_checkpoint_ignored: 42,
scan_checkpoint_stale: 43,
source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot {
source_work: vec![rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot {
source: "usage".to_string(),
checked: 44,
queued: 45,
@@ -932,7 +932,7 @@ mod test {
skipped: 48,
missed: 49,
}],
current_cycle_source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot {
current_cycle_source_work: vec![rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot {
source: "lifecycle".to_string(),
checked: 50,
queued: 51,
@@ -941,7 +941,7 @@ mod test {
skipped: 54,
missed: 55,
}],
last_cycle_source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot {
last_cycle_source_work: vec![rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot {
source: "heal".to_string(),
checked: 56,
queued: 57,
@@ -950,7 +950,7 @@ mod test {
skipped: 60,
missed: 61,
}],
replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot {
replication_repair: vec![rustfs_scanner_metrics::metrics::ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "object".to_string(),
scanner_role: "repair_admission".to_string(),
@@ -962,7 +962,7 @@ mod test {
skipped: 66,
missed: 67,
}],
current_cycle_replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot {
current_cycle_replication_repair: vec![rustfs_scanner_metrics::metrics::ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(),
kind: "delete_marker".to_string(),
scanner_role: "repair_admission".to_string(),
@@ -974,7 +974,7 @@ mod test {
skipped: 72,
missed: 73,
}],
last_cycle_replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot {
last_cycle_replication_repair: vec![rustfs_scanner_metrics::metrics::ScannerReplicationRepairSnapshot {
source: "site_replication".to_string(),
kind: "active_resync".to_string(),
scanner_role: "boundary_signal".to_string(),
+3 -3
View File
@@ -1168,7 +1168,7 @@ mod lifecycle_delete_all_plan_tests {
crate::object_api::LifecycleDeleteAllRequest {
version_id: Some(version_id),
delete_marker: true,
action: rustfs_scanner_contracts::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
action: rustfs_scanner_metrics::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
}
@@ -1361,7 +1361,7 @@ mod lifecycle_delete_all_plan_tests {
let request = crate::object_api::LifecycleDeleteAllRequest {
version_id: None,
delete_marker: false,
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
};
@@ -19363,7 +19363,7 @@ mod delete_objects_lock_gating_tests {
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
version_id: Some(trigger_version_id),
delete_marker: false,
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::History,
}),
+2 -2
View File
@@ -15934,7 +15934,7 @@ mod tests {
.expect("unknown transition metadata should be written");
}
let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "delete-all-versions".to_string(),
..Default::default()
};
@@ -16186,7 +16186,7 @@ mod tests {
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
version_id: original.version_id,
delete_marker: false,
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
}),
+1 -1
View File
@@ -6448,7 +6448,7 @@ mod tests {
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
version_id: Some(version_id),
delete_marker: false,
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "delete-all".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
}),
+2 -2
View File
@@ -1322,7 +1322,7 @@ mod tests {
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
version_id: Some(trigger_id),
delete_marker: false,
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
}),
@@ -1492,7 +1492,7 @@ mod tests {
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
version_id: Some(marker_id),
delete_marker: true,
action: rustfs_scanner_contracts::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
action: rustfs_scanner_metrics::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
}),