gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-11 07:36:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(scanner): expose lifecycle transition status (#3326)

Browse Source 
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-06-11 00:45:02 +08:00
committed by GitHub
parent d9ddd1bedc
commit 91f80a5585
5 changed files with 517 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/common/src/metrics.rs
+218
View File
@@ -600,6 +600,8 @@ pub struct Metrics {
current_scan_cycle_throttle_sleep_events_start: AtomicU64,
current_scan_cycle_throttle_sleep_duration_millis_start: AtomicU64,
current_scan_cycle_ilm_actions_start: AtomicU64,
current_scan_cycle_lifecycle_expiry_actions_start: AtomicU64,
current_scan_cycle_lifecycle_transition_actions_start: AtomicU64,
current_scan_cycle_heal_objects_start: AtomicU64,
current_scan_cycle_replication_checks_start: AtomicU64,
current_scan_cycle_usage_saves_start: AtomicU64,
@@ -620,6 +622,8 @@ pub struct Metrics {
last_scan_cycle_throttle_sleep_events: AtomicU64,
last_scan_cycle_throttle_sleep_duration_millis: AtomicU64,
last_scan_cycle_ilm_actions: AtomicU64,
last_scan_cycle_lifecycle_expiry_actions: AtomicU64,
last_scan_cycle_lifecycle_transition_actions: AtomicU64,
last_scan_cycle_heal_objects: AtomicU64,
last_scan_cycle_replication_checks: AtomicU64,
last_scan_cycle_usage_saves: AtomicU64,
@@ -632,6 +636,20 @@ pub struct Metrics {
scanner_yield_duration_millis: AtomicU64,
scanner_throttle_sleep_duration_millis: AtomicU64,
scanner_ilm_actions: AtomicU64,
scanner_lifecycle_expiry_actions: AtomicU64,
scanner_lifecycle_transition_actions: AtomicU64,
scanner_transition_queue_capacity: AtomicU64,
scanner_transition_queued: AtomicU64,
scanner_transition_active: AtomicU64,
scanner_transition_workers: AtomicU64,
scanner_transition_queue_full: AtomicU64,
scanner_transition_queue_send_timeout: AtomicU64,
scanner_transition_compensation_scheduled: AtomicU64,
scanner_transition_compensation_running: AtomicU64,
scanner_transition_queued_total: AtomicU64,
scanner_transition_missed_total: AtomicU64,
scanner_transition_completed: AtomicU64,
scanner_transition_failed: AtomicU64,
scanner_throttle_idle_mode_enabled: AtomicBool,
scanner_throttle_sleep_factor_micros: AtomicU64,
scanner_throttle_max_sleep_millis: AtomicU64,
@@ -710,6 +728,8 @@ pub struct ScanCycleWorkSnapshot {
throttle_sleep_events: u64,
throttle_sleep_duration_millis: u64,
ilm_actions: u64,
lifecycle_expiry_actions: u64,
lifecycle_transition_actions: u64,
heal_objects: u64,
replication_checks: u64,
usage_saves: u64,
@@ -775,6 +795,34 @@ impl Default for ScannerPacingPressureSnapshot {
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ScannerLifecycleTransitionStateUpdate {
pub queue_capacity: u64,
pub queued: u64,
pub active: u64,
pub workers: u64,
pub queue_full: u64,
pub queue_send_timeout: u64,
pub compensation_scheduled: u64,
pub compensation_running: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerLifecycleTransitionSnapshot {
pub current_queue_capacity: u64,
pub current_queued: u64,
pub current_active: u64,
pub current_workers: u64,
pub queue_full: u64,
pub queue_send_timeout: u64,
pub compensation_scheduled: u64,
pub compensation_running: u64,
pub scanner_queued: u64,
pub scanner_missed: u64,
pub completed: u64,
pub failed: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ScannerLastMinute {
pub actions: HashMap<String, ScannerTimedAction>,
@@ -828,6 +876,10 @@ pub struct ScannerMetricsReport {
#[serde(default)]
pub current_cycle_ilm_actions: u64,
#[serde(default)]
pub current_cycle_lifecycle_expiry_actions: u64,
#[serde(default)]
pub current_cycle_lifecycle_transition_actions: u64,
#[serde(default)]
pub current_cycle_heal_objects: u64,
#[serde(default)]
pub current_cycle_replication_checks: u64,
@@ -863,6 +915,10 @@ pub struct ScannerMetricsReport {
#[serde(default)]
pub last_cycle_ilm_actions: u64,
#[serde(default)]
pub last_cycle_lifecycle_expiry_actions: u64,
#[serde(default)]
pub last_cycle_lifecycle_transition_actions: u64,
#[serde(default)]
pub last_cycle_heal_objects: u64,
#[serde(default)]
pub last_cycle_replication_checks: u64,
@@ -882,6 +938,8 @@ pub struct ScannerMetricsReport {
#[serde(default)]
pub pacing_pressure: ScannerPacingPressureSnapshot,
#[serde(default)]
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
#[serde(default)]
pub throttle_idle_mode_enabled: bool,
#[serde(default)]
pub throttle_sleep_factor: f64,
@@ -1134,6 +1192,8 @@ impl Metrics {
current_scan_cycle_throttle_sleep_events_start: AtomicU64::new(0),
current_scan_cycle_throttle_sleep_duration_millis_start: AtomicU64::new(0),
current_scan_cycle_ilm_actions_start: AtomicU64::new(0),
current_scan_cycle_lifecycle_expiry_actions_start: AtomicU64::new(0),
current_scan_cycle_lifecycle_transition_actions_start: AtomicU64::new(0),
current_scan_cycle_heal_objects_start: AtomicU64::new(0),
current_scan_cycle_replication_checks_start: AtomicU64::new(0),
current_scan_cycle_usage_saves_start: AtomicU64::new(0),
@@ -1154,6 +1214,8 @@ impl Metrics {
last_scan_cycle_throttle_sleep_events: AtomicU64::new(0),
last_scan_cycle_throttle_sleep_duration_millis: AtomicU64::new(0),
last_scan_cycle_ilm_actions: AtomicU64::new(0),
last_scan_cycle_lifecycle_expiry_actions: AtomicU64::new(0),
last_scan_cycle_lifecycle_transition_actions: AtomicU64::new(0),
last_scan_cycle_heal_objects: AtomicU64::new(0),
last_scan_cycle_replication_checks: AtomicU64::new(0),
last_scan_cycle_usage_saves: AtomicU64::new(0),
@@ -1166,6 +1228,20 @@ impl Metrics {
scanner_yield_duration_millis: AtomicU64::new(0),
scanner_throttle_sleep_duration_millis: AtomicU64::new(0),
scanner_ilm_actions: AtomicU64::new(0),
scanner_lifecycle_expiry_actions: AtomicU64::new(0),
scanner_lifecycle_transition_actions: AtomicU64::new(0),
scanner_transition_queue_capacity: AtomicU64::new(0),
scanner_transition_queued: AtomicU64::new(0),
scanner_transition_active: AtomicU64::new(0),
scanner_transition_workers: AtomicU64::new(0),
scanner_transition_queue_full: AtomicU64::new(0),
scanner_transition_queue_send_timeout: AtomicU64::new(0),
scanner_transition_compensation_scheduled: AtomicU64::new(0),
scanner_transition_compensation_running: AtomicU64::new(0),
scanner_transition_queued_total: AtomicU64::new(0),
scanner_transition_missed_total: AtomicU64::new(0),
scanner_transition_completed: AtomicU64::new(0),
scanner_transition_failed: AtomicU64::new(0),
scanner_throttle_idle_mode_enabled: AtomicBool::new(false),
scanner_throttle_sleep_factor_micros: AtomicU64::new(0),
scanner_throttle_max_sleep_millis: AtomicU64::new(0),
@@ -1337,6 +1413,19 @@ impl Metrics {
self.record_scanner_source_executed(ScannerWorkSource::Lifecycle, count);
}
pub fn record_scanner_lifecycle_action(&self, action: IlmAction, count: u64) {
self.record_scanner_ilm_action(count);
match action {
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
self.scanner_lifecycle_transition_actions.fetch_add(count, Ordering::Relaxed);
}
_ if action.delete() => {
self.scanner_lifecycle_expiry_actions.fetch_add(count, Ordering::Relaxed);
}
_ => {}
}
}
pub fn record_scanner_ilm_enqueue_result(&self, count: u64, queued: bool) {
if queued {
self.record_scanner_source_queued(ScannerWorkSource::Lifecycle, count);
@@ -1345,6 +1434,38 @@ impl Metrics {
}
}
pub fn record_scanner_transition_enqueue_result(&self, count: u64, queued: bool) {
self.record_scanner_ilm_enqueue_result(count, queued);
if queued {
self.scanner_transition_queued_total.fetch_add(count, Ordering::Relaxed);
} else {
self.scanner_transition_missed_total.fetch_add(count, Ordering::Relaxed);
}
}
pub fn record_scanner_lifecycle_transition_state(&self, state: ScannerLifecycleTransitionStateUpdate) {
self.scanner_transition_queue_capacity
.store(state.queue_capacity, Ordering::Relaxed);
self.scanner_transition_queued.store(state.queued, Ordering::Relaxed);
self.scanner_transition_active.store(state.active, Ordering::Relaxed);
self.scanner_transition_workers.store(state.workers, Ordering::Relaxed);
self.scanner_transition_queue_full.store(state.queue_full, Ordering::Relaxed);
self.scanner_transition_queue_send_timeout
.store(state.queue_send_timeout, Ordering::Relaxed);
self.scanner_transition_compensation_scheduled
.store(state.compensation_scheduled, Ordering::Relaxed);
self.scanner_transition_compensation_running
.store(state.compensation_running, Ordering::Relaxed);
}
pub fn record_scanner_transition_completed(&self, count: u64) {
self.scanner_transition_completed.fetch_add(count, Ordering::Relaxed);
}
pub fn record_scanner_transition_failed(&self, count: u64) {
self.scanner_transition_failed.fetch_add(count, Ordering::Relaxed);
}
pub fn record_scanner_checkpoint_set(&self, version: u16, resume_after: impl Into<String>, reason: impl Into<String>) {
let checkpoint = ScannerCheckpointReport {
version,
@@ -1643,6 +1764,10 @@ impl Metrics {
.store(snapshot.throttle_sleep_duration_millis, Ordering::Relaxed);
self.current_scan_cycle_ilm_actions_start
.store(snapshot.ilm_actions, Ordering::Relaxed);
self.current_scan_cycle_lifecycle_expiry_actions_start
.store(snapshot.lifecycle_expiry_actions, Ordering::Relaxed);
self.current_scan_cycle_lifecycle_transition_actions_start
.store(snapshot.lifecycle_transition_actions, Ordering::Relaxed);
self.current_scan_cycle_heal_objects_start
.store(snapshot.heal_objects, Ordering::Relaxed);
self.current_scan_cycle_replication_checks_start
@@ -1673,6 +1798,8 @@ impl Metrics {
throttle_sleep_events: self.lifetime(Metric::ThrottleSleep),
throttle_sleep_duration_millis: self.scanner_throttle_sleep_duration_millis.load(Ordering::Relaxed),
ilm_actions: self.scanner_ilm_actions.load(Ordering::Relaxed),
lifecycle_expiry_actions: self.scanner_lifecycle_expiry_actions.load(Ordering::Relaxed),
lifecycle_transition_actions: self.scanner_lifecycle_transition_actions.load(Ordering::Relaxed),
heal_objects: self.lifetime(Metric::HealAbandonedObject),
replication_checks: self.lifetime(Metric::CheckReplication),
usage_saves: self.lifetime(Metric::SaveUsage),
@@ -1692,6 +1819,10 @@ impl Metrics {
.current_scan_cycle_throttle_sleep_duration_millis_start
.load(Ordering::Relaxed),
ilm_actions: self.current_scan_cycle_ilm_actions_start.load(Ordering::Relaxed),
lifecycle_expiry_actions: self.current_scan_cycle_lifecycle_expiry_actions_start.load(Ordering::Relaxed),
lifecycle_transition_actions: self
.current_scan_cycle_lifecycle_transition_actions_start
.load(Ordering::Relaxed),
heal_objects: self.current_scan_cycle_heal_objects_start.load(Ordering::Relaxed),
replication_checks: self.current_scan_cycle_replication_checks_start.load(Ordering::Relaxed),
usage_saves: self.current_scan_cycle_usage_saves_start.load(Ordering::Relaxed),
@@ -1712,6 +1843,12 @@ impl Metrics {
.throttle_sleep_duration_millis
.saturating_sub(start.throttle_sleep_duration_millis),
ilm_actions: current.ilm_actions.saturating_sub(start.ilm_actions),
lifecycle_expiry_actions: current
.lifecycle_expiry_actions
.saturating_sub(start.lifecycle_expiry_actions),
lifecycle_transition_actions: current
.lifecycle_transition_actions
.saturating_sub(start.lifecycle_transition_actions),
heal_objects: current.heal_objects.saturating_sub(start.heal_objects),
replication_checks: current.replication_checks.saturating_sub(start.replication_checks),
usage_saves: current.usage_saves.saturating_sub(start.usage_saves),
@@ -1789,6 +1926,10 @@ impl Metrics {
self.last_scan_cycle_throttle_sleep_duration_millis
.store(work.throttle_sleep_duration_millis, Ordering::Relaxed);
self.last_scan_cycle_ilm_actions.store(work.ilm_actions, Ordering::Relaxed);
self.last_scan_cycle_lifecycle_expiry_actions
.store(work.lifecycle_expiry_actions, Ordering::Relaxed);
self.last_scan_cycle_lifecycle_transition_actions
.store(work.lifecycle_transition_actions, Ordering::Relaxed);
self.last_scan_cycle_heal_objects.store(work.heal_objects, Ordering::Relaxed);
self.last_scan_cycle_replication_checks
.store(work.replication_checks, Ordering::Relaxed);
@@ -1880,6 +2021,8 @@ impl Metrics {
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
m.current_cycle_ilm_actions = current_work.ilm_actions;
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
m.current_cycle_heal_objects = current_work.heal_objects;
m.current_cycle_replication_checks = current_work.replication_checks;
m.current_cycle_usage_saves = current_work.usage_saves;
@@ -1908,6 +2051,8 @@ impl Metrics {
m.last_cycle_throttle_sleep_duration_seconds =
self.last_scan_cycle_throttle_sleep_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
m.last_cycle_ilm_actions = self.last_scan_cycle_ilm_actions.load(Ordering::Relaxed);
m.last_cycle_lifecycle_expiry_actions = self.last_scan_cycle_lifecycle_expiry_actions.load(Ordering::Relaxed);
m.last_cycle_lifecycle_transition_actions = self.last_scan_cycle_lifecycle_transition_actions.load(Ordering::Relaxed);
m.last_cycle_heal_objects = self.last_scan_cycle_heal_objects.load(Ordering::Relaxed);
m.last_cycle_replication_checks = self.last_scan_cycle_replication_checks.load(Ordering::Relaxed);
m.last_cycle_usage_saves = self.last_scan_cycle_usage_saves.load(Ordering::Relaxed);
@@ -1928,6 +2073,20 @@ impl Metrics {
})
})
.collect();
m.lifecycle_transition = ScannerLifecycleTransitionSnapshot {
current_queue_capacity: self.scanner_transition_queue_capacity.load(Ordering::Relaxed),
current_queued: self.scanner_transition_queued.load(Ordering::Relaxed),
current_active: self.scanner_transition_active.load(Ordering::Relaxed),
current_workers: self.scanner_transition_workers.load(Ordering::Relaxed),
queue_full: self.scanner_transition_queue_full.load(Ordering::Relaxed),
queue_send_timeout: self.scanner_transition_queue_send_timeout.load(Ordering::Relaxed),
compensation_scheduled: self.scanner_transition_compensation_scheduled.load(Ordering::Relaxed),
compensation_running: self.scanner_transition_compensation_running.load(Ordering::Relaxed),
scanner_queued: self.scanner_transition_queued_total.load(Ordering::Relaxed),
scanner_missed: self.scanner_transition_missed_total.load(Ordering::Relaxed),
completed: self.scanner_transition_completed.load(Ordering::Relaxed),
failed: self.scanner_transition_failed.load(Ordering::Relaxed),
};
m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
m.throttle_max_sleep_seconds = self.scanner_throttle_max_sleep_millis.load(Ordering::Relaxed) as f64 / 1000.0;
@@ -2262,6 +2421,64 @@ mod tests {
assert_eq!(lifecycle.missed, 8);
}
#[tokio::test]
async fn report_splits_scanner_lifecycle_actions_by_expiry_and_transition() {
let metrics = Metrics::new();
let start = metrics.start_scan_cycle_work();
metrics.record_scanner_lifecycle_action(IlmAction::DeleteAction, 2);
metrics.record_scanner_lifecycle_action(IlmAction::DeleteRestoredVersionAction, 3);
metrics.record_scanner_lifecycle_action(IlmAction::TransitionAction, 5);
metrics.record_scanner_lifecycle_action(IlmAction::TransitionVersionAction, 7);
let report = metrics.report().await;
assert_eq!(report.current_cycle_ilm_actions, 17);
assert_eq!(report.current_cycle_lifecycle_expiry_actions, 5);
assert_eq!(report.current_cycle_lifecycle_transition_actions, 12);
metrics.finish_scan_cycle_work(start);
let report = metrics.report().await;
assert_eq!(report.last_cycle_ilm_actions, 17);
assert_eq!(report.last_cycle_lifecycle_expiry_actions, 5);
assert_eq!(report.last_cycle_lifecycle_transition_actions, 12);
}
#[tokio::test]
async fn report_includes_scanner_lifecycle_transition_status() {
let metrics = Metrics::new();
metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate {
queue_capacity: 16,
queued: 5,
active: 2,
workers: 4,
queue_full: 3,
queue_send_timeout: 1,
compensation_scheduled: 2,
compensation_running: 1,
});
metrics.record_scanner_transition_enqueue_result(6, true);
metrics.record_scanner_transition_enqueue_result(2, false);
metrics.record_scanner_transition_completed(4);
metrics.record_scanner_transition_failed(1);
let report = metrics.report().await;
assert_eq!(report.lifecycle_transition.current_queue_capacity, 16);
assert_eq!(report.lifecycle_transition.current_queued, 5);
assert_eq!(report.lifecycle_transition.current_active, 2);
assert_eq!(report.lifecycle_transition.current_workers, 4);
assert_eq!(report.lifecycle_transition.queue_full, 3);
assert_eq!(report.lifecycle_transition.queue_send_timeout, 1);
assert_eq!(report.lifecycle_transition.compensation_scheduled, 2);
assert_eq!(report.lifecycle_transition.compensation_running, 1);
assert_eq!(report.lifecycle_transition.scanner_queued, 6);
assert_eq!(report.lifecycle_transition.scanner_missed, 2);
assert_eq!(report.lifecycle_transition.completed, 4);
assert_eq!(report.lifecycle_transition.failed, 1);
}
#[tokio::test]
async fn scanner_metrics_update_source_work() {
let metrics = Metrics::new();
@@ -2485,6 +2702,7 @@ mod tests {
heal_objects: 2,
replication_checks: 4,
usage_saves: 6,
..Default::default()
});
let report = metrics.report().await;