feat(scanner): expose lifecycle transition status (#3326)

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-06-11 00:45:02 +08:00
committed by GitHub
parent d9ddd1bedc
commit 91f80a5585
5 changed files with 517 additions and 14 deletions
+218
View File
@@ -600,6 +600,8 @@ pub struct Metrics {
current_scan_cycle_throttle_sleep_events_start: AtomicU64, current_scan_cycle_throttle_sleep_events_start: AtomicU64,
current_scan_cycle_throttle_sleep_duration_millis_start: AtomicU64, current_scan_cycle_throttle_sleep_duration_millis_start: AtomicU64,
current_scan_cycle_ilm_actions_start: AtomicU64, current_scan_cycle_ilm_actions_start: AtomicU64,
current_scan_cycle_lifecycle_expiry_actions_start: AtomicU64,
current_scan_cycle_lifecycle_transition_actions_start: AtomicU64,
current_scan_cycle_heal_objects_start: AtomicU64, current_scan_cycle_heal_objects_start: AtomicU64,
current_scan_cycle_replication_checks_start: AtomicU64, current_scan_cycle_replication_checks_start: AtomicU64,
current_scan_cycle_usage_saves_start: AtomicU64, current_scan_cycle_usage_saves_start: AtomicU64,
@@ -620,6 +622,8 @@ pub struct Metrics {
last_scan_cycle_throttle_sleep_events: AtomicU64, last_scan_cycle_throttle_sleep_events: AtomicU64,
last_scan_cycle_throttle_sleep_duration_millis: AtomicU64, last_scan_cycle_throttle_sleep_duration_millis: AtomicU64,
last_scan_cycle_ilm_actions: AtomicU64, last_scan_cycle_ilm_actions: AtomicU64,
last_scan_cycle_lifecycle_expiry_actions: AtomicU64,
last_scan_cycle_lifecycle_transition_actions: AtomicU64,
last_scan_cycle_heal_objects: AtomicU64, last_scan_cycle_heal_objects: AtomicU64,
last_scan_cycle_replication_checks: AtomicU64, last_scan_cycle_replication_checks: AtomicU64,
last_scan_cycle_usage_saves: AtomicU64, last_scan_cycle_usage_saves: AtomicU64,
@@ -632,6 +636,20 @@ pub struct Metrics {
scanner_yield_duration_millis: AtomicU64, scanner_yield_duration_millis: AtomicU64,
scanner_throttle_sleep_duration_millis: AtomicU64, scanner_throttle_sleep_duration_millis: AtomicU64,
scanner_ilm_actions: AtomicU64, scanner_ilm_actions: AtomicU64,
scanner_lifecycle_expiry_actions: AtomicU64,
scanner_lifecycle_transition_actions: AtomicU64,
scanner_transition_queue_capacity: AtomicU64,
scanner_transition_queued: AtomicU64,
scanner_transition_active: AtomicU64,
scanner_transition_workers: AtomicU64,
scanner_transition_queue_full: AtomicU64,
scanner_transition_queue_send_timeout: AtomicU64,
scanner_transition_compensation_scheduled: AtomicU64,
scanner_transition_compensation_running: AtomicU64,
scanner_transition_queued_total: AtomicU64,
scanner_transition_missed_total: AtomicU64,
scanner_transition_completed: AtomicU64,
scanner_transition_failed: AtomicU64,
scanner_throttle_idle_mode_enabled: AtomicBool, scanner_throttle_idle_mode_enabled: AtomicBool,
scanner_throttle_sleep_factor_micros: AtomicU64, scanner_throttle_sleep_factor_micros: AtomicU64,
scanner_throttle_max_sleep_millis: AtomicU64, scanner_throttle_max_sleep_millis: AtomicU64,
@@ -710,6 +728,8 @@ pub struct ScanCycleWorkSnapshot {
throttle_sleep_events: u64, throttle_sleep_events: u64,
throttle_sleep_duration_millis: u64, throttle_sleep_duration_millis: u64,
ilm_actions: u64, ilm_actions: u64,
lifecycle_expiry_actions: u64,
lifecycle_transition_actions: u64,
heal_objects: u64, heal_objects: u64,
replication_checks: u64, replication_checks: u64,
usage_saves: u64, usage_saves: u64,
@@ -775,6 +795,34 @@ impl Default for ScannerPacingPressureSnapshot {
} }
} }
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ScannerLifecycleTransitionStateUpdate {
pub queue_capacity: u64,
pub queued: u64,
pub active: u64,
pub workers: u64,
pub queue_full: u64,
pub queue_send_timeout: u64,
pub compensation_scheduled: u64,
pub compensation_running: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerLifecycleTransitionSnapshot {
pub current_queue_capacity: u64,
pub current_queued: u64,
pub current_active: u64,
pub current_workers: u64,
pub queue_full: u64,
pub queue_send_timeout: u64,
pub compensation_scheduled: u64,
pub compensation_running: u64,
pub scanner_queued: u64,
pub scanner_missed: u64,
pub completed: u64,
pub failed: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)] #[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ScannerLastMinute { pub struct ScannerLastMinute {
pub actions: HashMap<String, ScannerTimedAction>, pub actions: HashMap<String, ScannerTimedAction>,
@@ -828,6 +876,10 @@ pub struct ScannerMetricsReport {
#[serde(default)] #[serde(default)]
pub current_cycle_ilm_actions: u64, pub current_cycle_ilm_actions: u64,
#[serde(default)] #[serde(default)]
pub current_cycle_lifecycle_expiry_actions: u64,
#[serde(default)]
pub current_cycle_lifecycle_transition_actions: u64,
#[serde(default)]
pub current_cycle_heal_objects: u64, pub current_cycle_heal_objects: u64,
#[serde(default)] #[serde(default)]
pub current_cycle_replication_checks: u64, pub current_cycle_replication_checks: u64,
@@ -863,6 +915,10 @@ pub struct ScannerMetricsReport {
#[serde(default)] #[serde(default)]
pub last_cycle_ilm_actions: u64, pub last_cycle_ilm_actions: u64,
#[serde(default)] #[serde(default)]
pub last_cycle_lifecycle_expiry_actions: u64,
#[serde(default)]
pub last_cycle_lifecycle_transition_actions: u64,
#[serde(default)]
pub last_cycle_heal_objects: u64, pub last_cycle_heal_objects: u64,
#[serde(default)] #[serde(default)]
pub last_cycle_replication_checks: u64, pub last_cycle_replication_checks: u64,
@@ -882,6 +938,8 @@ pub struct ScannerMetricsReport {
#[serde(default)] #[serde(default)]
pub pacing_pressure: ScannerPacingPressureSnapshot, pub pacing_pressure: ScannerPacingPressureSnapshot,
#[serde(default)] #[serde(default)]
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
#[serde(default)]
pub throttle_idle_mode_enabled: bool, pub throttle_idle_mode_enabled: bool,
#[serde(default)] #[serde(default)]
pub throttle_sleep_factor: f64, pub throttle_sleep_factor: f64,
@@ -1134,6 +1192,8 @@ impl Metrics {
current_scan_cycle_throttle_sleep_events_start: AtomicU64::new(0), current_scan_cycle_throttle_sleep_events_start: AtomicU64::new(0),
current_scan_cycle_throttle_sleep_duration_millis_start: AtomicU64::new(0), current_scan_cycle_throttle_sleep_duration_millis_start: AtomicU64::new(0),
current_scan_cycle_ilm_actions_start: AtomicU64::new(0), current_scan_cycle_ilm_actions_start: AtomicU64::new(0),
current_scan_cycle_lifecycle_expiry_actions_start: AtomicU64::new(0),
current_scan_cycle_lifecycle_transition_actions_start: AtomicU64::new(0),
current_scan_cycle_heal_objects_start: AtomicU64::new(0), current_scan_cycle_heal_objects_start: AtomicU64::new(0),
current_scan_cycle_replication_checks_start: AtomicU64::new(0), current_scan_cycle_replication_checks_start: AtomicU64::new(0),
current_scan_cycle_usage_saves_start: AtomicU64::new(0), current_scan_cycle_usage_saves_start: AtomicU64::new(0),
@@ -1154,6 +1214,8 @@ impl Metrics {
last_scan_cycle_throttle_sleep_events: AtomicU64::new(0), last_scan_cycle_throttle_sleep_events: AtomicU64::new(0),
last_scan_cycle_throttle_sleep_duration_millis: AtomicU64::new(0), last_scan_cycle_throttle_sleep_duration_millis: AtomicU64::new(0),
last_scan_cycle_ilm_actions: AtomicU64::new(0), last_scan_cycle_ilm_actions: AtomicU64::new(0),
last_scan_cycle_lifecycle_expiry_actions: AtomicU64::new(0),
last_scan_cycle_lifecycle_transition_actions: AtomicU64::new(0),
last_scan_cycle_heal_objects: AtomicU64::new(0), last_scan_cycle_heal_objects: AtomicU64::new(0),
last_scan_cycle_replication_checks: AtomicU64::new(0), last_scan_cycle_replication_checks: AtomicU64::new(0),
last_scan_cycle_usage_saves: AtomicU64::new(0), last_scan_cycle_usage_saves: AtomicU64::new(0),
@@ -1166,6 +1228,20 @@ impl Metrics {
scanner_yield_duration_millis: AtomicU64::new(0), scanner_yield_duration_millis: AtomicU64::new(0),
scanner_throttle_sleep_duration_millis: AtomicU64::new(0), scanner_throttle_sleep_duration_millis: AtomicU64::new(0),
scanner_ilm_actions: AtomicU64::new(0), scanner_ilm_actions: AtomicU64::new(0),
scanner_lifecycle_expiry_actions: AtomicU64::new(0),
scanner_lifecycle_transition_actions: AtomicU64::new(0),
scanner_transition_queue_capacity: AtomicU64::new(0),
scanner_transition_queued: AtomicU64::new(0),
scanner_transition_active: AtomicU64::new(0),
scanner_transition_workers: AtomicU64::new(0),
scanner_transition_queue_full: AtomicU64::new(0),
scanner_transition_queue_send_timeout: AtomicU64::new(0),
scanner_transition_compensation_scheduled: AtomicU64::new(0),
scanner_transition_compensation_running: AtomicU64::new(0),
scanner_transition_queued_total: AtomicU64::new(0),
scanner_transition_missed_total: AtomicU64::new(0),
scanner_transition_completed: AtomicU64::new(0),
scanner_transition_failed: AtomicU64::new(0),
scanner_throttle_idle_mode_enabled: AtomicBool::new(false), scanner_throttle_idle_mode_enabled: AtomicBool::new(false),
scanner_throttle_sleep_factor_micros: AtomicU64::new(0), scanner_throttle_sleep_factor_micros: AtomicU64::new(0),
scanner_throttle_max_sleep_millis: AtomicU64::new(0), scanner_throttle_max_sleep_millis: AtomicU64::new(0),
@@ -1337,6 +1413,19 @@ impl Metrics {
self.record_scanner_source_executed(ScannerWorkSource::Lifecycle, count); self.record_scanner_source_executed(ScannerWorkSource::Lifecycle, count);
} }
pub fn record_scanner_lifecycle_action(&self, action: IlmAction, count: u64) {
self.record_scanner_ilm_action(count);
match action {
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
self.scanner_lifecycle_transition_actions.fetch_add(count, Ordering::Relaxed);
}
_ if action.delete() => {
self.scanner_lifecycle_expiry_actions.fetch_add(count, Ordering::Relaxed);
}
_ => {}
}
}
pub fn record_scanner_ilm_enqueue_result(&self, count: u64, queued: bool) { pub fn record_scanner_ilm_enqueue_result(&self, count: u64, queued: bool) {
if queued { if queued {
self.record_scanner_source_queued(ScannerWorkSource::Lifecycle, count); self.record_scanner_source_queued(ScannerWorkSource::Lifecycle, count);
@@ -1345,6 +1434,38 @@ impl Metrics {
} }
} }
pub fn record_scanner_transition_enqueue_result(&self, count: u64, queued: bool) {
self.record_scanner_ilm_enqueue_result(count, queued);
if queued {
self.scanner_transition_queued_total.fetch_add(count, Ordering::Relaxed);
} else {
self.scanner_transition_missed_total.fetch_add(count, Ordering::Relaxed);
}
}
pub fn record_scanner_lifecycle_transition_state(&self, state: ScannerLifecycleTransitionStateUpdate) {
self.scanner_transition_queue_capacity
.store(state.queue_capacity, Ordering::Relaxed);
self.scanner_transition_queued.store(state.queued, Ordering::Relaxed);
self.scanner_transition_active.store(state.active, Ordering::Relaxed);
self.scanner_transition_workers.store(state.workers, Ordering::Relaxed);
self.scanner_transition_queue_full.store(state.queue_full, Ordering::Relaxed);
self.scanner_transition_queue_send_timeout
.store(state.queue_send_timeout, Ordering::Relaxed);
self.scanner_transition_compensation_scheduled
.store(state.compensation_scheduled, Ordering::Relaxed);
self.scanner_transition_compensation_running
.store(state.compensation_running, Ordering::Relaxed);
}
pub fn record_scanner_transition_completed(&self, count: u64) {
self.scanner_transition_completed.fetch_add(count, Ordering::Relaxed);
}
pub fn record_scanner_transition_failed(&self, count: u64) {
self.scanner_transition_failed.fetch_add(count, Ordering::Relaxed);
}
pub fn record_scanner_checkpoint_set(&self, version: u16, resume_after: impl Into<String>, reason: impl Into<String>) { pub fn record_scanner_checkpoint_set(&self, version: u16, resume_after: impl Into<String>, reason: impl Into<String>) {
let checkpoint = ScannerCheckpointReport { let checkpoint = ScannerCheckpointReport {
version, version,
@@ -1643,6 +1764,10 @@ impl Metrics {
.store(snapshot.throttle_sleep_duration_millis, Ordering::Relaxed); .store(snapshot.throttle_sleep_duration_millis, Ordering::Relaxed);
self.current_scan_cycle_ilm_actions_start self.current_scan_cycle_ilm_actions_start
.store(snapshot.ilm_actions, Ordering::Relaxed); .store(snapshot.ilm_actions, Ordering::Relaxed);
self.current_scan_cycle_lifecycle_expiry_actions_start
.store(snapshot.lifecycle_expiry_actions, Ordering::Relaxed);
self.current_scan_cycle_lifecycle_transition_actions_start
.store(snapshot.lifecycle_transition_actions, Ordering::Relaxed);
self.current_scan_cycle_heal_objects_start self.current_scan_cycle_heal_objects_start
.store(snapshot.heal_objects, Ordering::Relaxed); .store(snapshot.heal_objects, Ordering::Relaxed);
self.current_scan_cycle_replication_checks_start self.current_scan_cycle_replication_checks_start
@@ -1673,6 +1798,8 @@ impl Metrics {
throttle_sleep_events: self.lifetime(Metric::ThrottleSleep), throttle_sleep_events: self.lifetime(Metric::ThrottleSleep),
throttle_sleep_duration_millis: self.scanner_throttle_sleep_duration_millis.load(Ordering::Relaxed), throttle_sleep_duration_millis: self.scanner_throttle_sleep_duration_millis.load(Ordering::Relaxed),
ilm_actions: self.scanner_ilm_actions.load(Ordering::Relaxed), ilm_actions: self.scanner_ilm_actions.load(Ordering::Relaxed),
lifecycle_expiry_actions: self.scanner_lifecycle_expiry_actions.load(Ordering::Relaxed),
lifecycle_transition_actions: self.scanner_lifecycle_transition_actions.load(Ordering::Relaxed),
heal_objects: self.lifetime(Metric::HealAbandonedObject), heal_objects: self.lifetime(Metric::HealAbandonedObject),
replication_checks: self.lifetime(Metric::CheckReplication), replication_checks: self.lifetime(Metric::CheckReplication),
usage_saves: self.lifetime(Metric::SaveUsage), usage_saves: self.lifetime(Metric::SaveUsage),
@@ -1692,6 +1819,10 @@ impl Metrics {
.current_scan_cycle_throttle_sleep_duration_millis_start .current_scan_cycle_throttle_sleep_duration_millis_start
.load(Ordering::Relaxed), .load(Ordering::Relaxed),
ilm_actions: self.current_scan_cycle_ilm_actions_start.load(Ordering::Relaxed), ilm_actions: self.current_scan_cycle_ilm_actions_start.load(Ordering::Relaxed),
lifecycle_expiry_actions: self.current_scan_cycle_lifecycle_expiry_actions_start.load(Ordering::Relaxed),
lifecycle_transition_actions: self
.current_scan_cycle_lifecycle_transition_actions_start
.load(Ordering::Relaxed),
heal_objects: self.current_scan_cycle_heal_objects_start.load(Ordering::Relaxed), heal_objects: self.current_scan_cycle_heal_objects_start.load(Ordering::Relaxed),
replication_checks: self.current_scan_cycle_replication_checks_start.load(Ordering::Relaxed), replication_checks: self.current_scan_cycle_replication_checks_start.load(Ordering::Relaxed),
usage_saves: self.current_scan_cycle_usage_saves_start.load(Ordering::Relaxed), usage_saves: self.current_scan_cycle_usage_saves_start.load(Ordering::Relaxed),
@@ -1712,6 +1843,12 @@ impl Metrics {
.throttle_sleep_duration_millis .throttle_sleep_duration_millis
.saturating_sub(start.throttle_sleep_duration_millis), .saturating_sub(start.throttle_sleep_duration_millis),
ilm_actions: current.ilm_actions.saturating_sub(start.ilm_actions), ilm_actions: current.ilm_actions.saturating_sub(start.ilm_actions),
lifecycle_expiry_actions: current
.lifecycle_expiry_actions
.saturating_sub(start.lifecycle_expiry_actions),
lifecycle_transition_actions: current
.lifecycle_transition_actions
.saturating_sub(start.lifecycle_transition_actions),
heal_objects: current.heal_objects.saturating_sub(start.heal_objects), heal_objects: current.heal_objects.saturating_sub(start.heal_objects),
replication_checks: current.replication_checks.saturating_sub(start.replication_checks), replication_checks: current.replication_checks.saturating_sub(start.replication_checks),
usage_saves: current.usage_saves.saturating_sub(start.usage_saves), usage_saves: current.usage_saves.saturating_sub(start.usage_saves),
@@ -1789,6 +1926,10 @@ impl Metrics {
self.last_scan_cycle_throttle_sleep_duration_millis self.last_scan_cycle_throttle_sleep_duration_millis
.store(work.throttle_sleep_duration_millis, Ordering::Relaxed); .store(work.throttle_sleep_duration_millis, Ordering::Relaxed);
self.last_scan_cycle_ilm_actions.store(work.ilm_actions, Ordering::Relaxed); self.last_scan_cycle_ilm_actions.store(work.ilm_actions, Ordering::Relaxed);
self.last_scan_cycle_lifecycle_expiry_actions
.store(work.lifecycle_expiry_actions, Ordering::Relaxed);
self.last_scan_cycle_lifecycle_transition_actions
.store(work.lifecycle_transition_actions, Ordering::Relaxed);
self.last_scan_cycle_heal_objects.store(work.heal_objects, Ordering::Relaxed); self.last_scan_cycle_heal_objects.store(work.heal_objects, Ordering::Relaxed);
self.last_scan_cycle_replication_checks self.last_scan_cycle_replication_checks
.store(work.replication_checks, Ordering::Relaxed); .store(work.replication_checks, Ordering::Relaxed);
@@ -1880,6 +2021,8 @@ impl Metrics {
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events; m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0; m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
m.current_cycle_ilm_actions = current_work.ilm_actions; m.current_cycle_ilm_actions = current_work.ilm_actions;
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
m.current_cycle_heal_objects = current_work.heal_objects; m.current_cycle_heal_objects = current_work.heal_objects;
m.current_cycle_replication_checks = current_work.replication_checks; m.current_cycle_replication_checks = current_work.replication_checks;
m.current_cycle_usage_saves = current_work.usage_saves; m.current_cycle_usage_saves = current_work.usage_saves;
@@ -1908,6 +2051,8 @@ impl Metrics {
m.last_cycle_throttle_sleep_duration_seconds = m.last_cycle_throttle_sleep_duration_seconds =
self.last_scan_cycle_throttle_sleep_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0; self.last_scan_cycle_throttle_sleep_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
m.last_cycle_ilm_actions = self.last_scan_cycle_ilm_actions.load(Ordering::Relaxed); m.last_cycle_ilm_actions = self.last_scan_cycle_ilm_actions.load(Ordering::Relaxed);
m.last_cycle_lifecycle_expiry_actions = self.last_scan_cycle_lifecycle_expiry_actions.load(Ordering::Relaxed);
m.last_cycle_lifecycle_transition_actions = self.last_scan_cycle_lifecycle_transition_actions.load(Ordering::Relaxed);
m.last_cycle_heal_objects = self.last_scan_cycle_heal_objects.load(Ordering::Relaxed); m.last_cycle_heal_objects = self.last_scan_cycle_heal_objects.load(Ordering::Relaxed);
m.last_cycle_replication_checks = self.last_scan_cycle_replication_checks.load(Ordering::Relaxed); m.last_cycle_replication_checks = self.last_scan_cycle_replication_checks.load(Ordering::Relaxed);
m.last_cycle_usage_saves = self.last_scan_cycle_usage_saves.load(Ordering::Relaxed); m.last_cycle_usage_saves = self.last_scan_cycle_usage_saves.load(Ordering::Relaxed);
@@ -1928,6 +2073,20 @@ impl Metrics {
}) })
}) })
.collect(); .collect();
m.lifecycle_transition = ScannerLifecycleTransitionSnapshot {
current_queue_capacity: self.scanner_transition_queue_capacity.load(Ordering::Relaxed),
current_queued: self.scanner_transition_queued.load(Ordering::Relaxed),
current_active: self.scanner_transition_active.load(Ordering::Relaxed),
current_workers: self.scanner_transition_workers.load(Ordering::Relaxed),
queue_full: self.scanner_transition_queue_full.load(Ordering::Relaxed),
queue_send_timeout: self.scanner_transition_queue_send_timeout.load(Ordering::Relaxed),
compensation_scheduled: self.scanner_transition_compensation_scheduled.load(Ordering::Relaxed),
compensation_running: self.scanner_transition_compensation_running.load(Ordering::Relaxed),
scanner_queued: self.scanner_transition_queued_total.load(Ordering::Relaxed),
scanner_missed: self.scanner_transition_missed_total.load(Ordering::Relaxed),
completed: self.scanner_transition_completed.load(Ordering::Relaxed),
failed: self.scanner_transition_failed.load(Ordering::Relaxed),
};
m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed); m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0; m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
m.throttle_max_sleep_seconds = self.scanner_throttle_max_sleep_millis.load(Ordering::Relaxed) as f64 / 1000.0; m.throttle_max_sleep_seconds = self.scanner_throttle_max_sleep_millis.load(Ordering::Relaxed) as f64 / 1000.0;
@@ -2262,6 +2421,64 @@ mod tests {
assert_eq!(lifecycle.missed, 8); assert_eq!(lifecycle.missed, 8);
} }
#[tokio::test]
async fn report_splits_scanner_lifecycle_actions_by_expiry_and_transition() {
let metrics = Metrics::new();
let start = metrics.start_scan_cycle_work();
metrics.record_scanner_lifecycle_action(IlmAction::DeleteAction, 2);
metrics.record_scanner_lifecycle_action(IlmAction::DeleteRestoredVersionAction, 3);
metrics.record_scanner_lifecycle_action(IlmAction::TransitionAction, 5);
metrics.record_scanner_lifecycle_action(IlmAction::TransitionVersionAction, 7);
let report = metrics.report().await;
assert_eq!(report.current_cycle_ilm_actions, 17);
assert_eq!(report.current_cycle_lifecycle_expiry_actions, 5);
assert_eq!(report.current_cycle_lifecycle_transition_actions, 12);
metrics.finish_scan_cycle_work(start);
let report = metrics.report().await;
assert_eq!(report.last_cycle_ilm_actions, 17);
assert_eq!(report.last_cycle_lifecycle_expiry_actions, 5);
assert_eq!(report.last_cycle_lifecycle_transition_actions, 12);
}
#[tokio::test]
async fn report_includes_scanner_lifecycle_transition_status() {
let metrics = Metrics::new();
metrics.record_scanner_lifecycle_transition_state(ScannerLifecycleTransitionStateUpdate {
queue_capacity: 16,
queued: 5,
active: 2,
workers: 4,
queue_full: 3,
queue_send_timeout: 1,
compensation_scheduled: 2,
compensation_running: 1,
});
metrics.record_scanner_transition_enqueue_result(6, true);
metrics.record_scanner_transition_enqueue_result(2, false);
metrics.record_scanner_transition_completed(4);
metrics.record_scanner_transition_failed(1);
let report = metrics.report().await;
assert_eq!(report.lifecycle_transition.current_queue_capacity, 16);
assert_eq!(report.lifecycle_transition.current_queued, 5);
assert_eq!(report.lifecycle_transition.current_active, 2);
assert_eq!(report.lifecycle_transition.current_workers, 4);
assert_eq!(report.lifecycle_transition.queue_full, 3);
assert_eq!(report.lifecycle_transition.queue_send_timeout, 1);
assert_eq!(report.lifecycle_transition.compensation_scheduled, 2);
assert_eq!(report.lifecycle_transition.compensation_running, 1);
assert_eq!(report.lifecycle_transition.scanner_queued, 6);
assert_eq!(report.lifecycle_transition.scanner_missed, 2);
assert_eq!(report.lifecycle_transition.completed, 4);
assert_eq!(report.lifecycle_transition.failed, 1);
}
#[tokio::test] #[tokio::test]
async fn scanner_metrics_update_source_work() { async fn scanner_metrics_update_source_work() {
let metrics = Metrics::new(); let metrics = Metrics::new();
@@ -2485,6 +2702,7 @@ mod tests {
heal_objects: 2, heal_objects: 2,
replication_checks: 4, replication_checks: 4,
usage_saves: 6, usage_saves: 6,
..Default::default()
}); });
let report = metrics.report().await; let report = metrics.report().await;
@@ -45,7 +45,7 @@ use futures::Future;
use http::HeaderMap; use http::HeaderMap;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use rustfs_common::heal_channel::rep_has_active_rules; use rustfs_common::heal_channel::rep_has_active_rules;
use rustfs_common::metrics::{IlmAction, Metrics, global_metrics}; use rustfs_common::metrics::{IlmAction, Metrics, ScannerLifecycleTransitionStateUpdate, global_metrics};
use rustfs_config::{ use rustfs_config::{
DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_QUEUE_SEND_TIMEOUT_MS, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX, DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_QUEUE_SEND_TIMEOUT_MS, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
DEFAULT_TRANSITION_WORKERS_CAP, ENV_TRANSITION_QUEUE_CAPACITY, ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS, ENV_TRANSITION_WORKERS, DEFAULT_TRANSITION_WORKERS_CAP, ENV_TRANSITION_QUEUE_CAPACITY, ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS, ENV_TRANSITION_WORKERS,
@@ -151,6 +151,20 @@ fn record_scanner_lifecycle_enqueue_result(src: &LcEventSrc, count: u64, queued:
} }
} }
fn record_scanner_transition_enqueue_result(src: &LcEventSrc, count: u64, queued: bool) {
if matches!(src, LcEventSrc::Scanner) {
global_metrics().record_scanner_transition_enqueue_result(count, queued);
}
}
fn nonnegative_i64_to_u64(value: i64) -> u64 {
u64::try_from(value).unwrap_or_default()
}
fn usize_to_u64_saturated(value: usize) -> u64 {
u64::try_from(value).unwrap_or(u64::MAX)
}
#[cfg(any(test, debug_assertions))] #[cfg(any(test, debug_assertions))]
fn should_force_immediate_transition_enqueue_timeout() -> bool { fn should_force_immediate_transition_enqueue_timeout() -> bool {
env::var(rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT) env::var(rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT)
@@ -664,14 +678,17 @@ impl TransitionState {
return false; return false;
} }
Self::inc_counter(&self.compensation_scheduled_tasks); Self::inc_counter(&self.compensation_scheduled_tasks);
self.record_scanner_transition_state();
let bucket = bucket.to_string(); let bucket = bucket.to_string();
let scheduled = Arc::clone(&self.compensation_buckets); let scheduled = Arc::clone(&self.compensation_buckets);
let state = Arc::clone(self); let state = Arc::clone(self);
tokio::spawn(async move { tokio::spawn(async move {
Self::inc_counter(&state.compensation_running_tasks); Self::inc_counter(&state.compensation_running_tasks);
state.record_scanner_transition_state();
let Some(api) = crate::new_object_layer_fn() else { let Some(api) = crate::new_object_layer_fn() else {
scheduled.lock().unwrap().remove(&bucket); scheduled.lock().unwrap().remove(&bucket);
Self::add_counter(&state.compensation_running_tasks, -1); Self::add_counter(&state.compensation_running_tasks, -1);
state.record_scanner_transition_state();
warn!(bucket = %bucket, "transition compensation skipped because object layer is unavailable"); warn!(bucket = %bucket, "transition compensation skipped because object layer is unavailable");
return; return;
}; };
@@ -684,6 +701,7 @@ impl TransitionState {
scheduled.lock().unwrap().remove(&bucket); scheduled.lock().unwrap().remove(&bucket);
Self::add_counter(&state.compensation_running_tasks, -1); Self::add_counter(&state.compensation_running_tasks, -1);
state.record_scanner_transition_state();
}); });
true true
} }
@@ -703,6 +721,23 @@ impl TransitionState {
counter.load(Ordering::Relaxed) counter.load(Ordering::Relaxed)
} }
fn scanner_transition_state_update(&self) -> ScannerLifecycleTransitionStateUpdate {
ScannerLifecycleTransitionStateUpdate {
queue_capacity: usize_to_u64_saturated(self.transition_queue_capacity),
queued: usize_to_u64_saturated(self.transition_rx.len()),
active: nonnegative_i64_to_u64(Self::counter_value(&self.active_tasks)),
workers: nonnegative_i64_to_u64(Self::counter_value(&self.num_workers)),
queue_full: nonnegative_i64_to_u64(Self::counter_value(&self.queue_full_tasks)),
queue_send_timeout: nonnegative_i64_to_u64(Self::counter_value(&self.queue_send_timeout_tasks)),
compensation_scheduled: nonnegative_i64_to_u64(Self::counter_value(&self.compensation_scheduled_tasks)),
compensation_running: nonnegative_i64_to_u64(Self::counter_value(&self.compensation_running_tasks)),
}
}
fn record_scanner_transition_state(&self) {
global_metrics().record_scanner_lifecycle_transition_state(self.scanner_transition_state_update());
}
fn handle_immediate_enqueue_failure(self: &Arc<Self>, oi: &ObjectInfo, src: &LcEventSrc, failure: ImmediateEnqueueFailure) { fn handle_immediate_enqueue_failure(self: &Arc<Self>, oi: &ObjectInfo, src: &LcEventSrc, failure: ImmediateEnqueueFailure) {
Self::inc_counter(&self.missed_immediate_tasks); Self::inc_counter(&self.missed_immediate_tasks);
let scheduled = self.schedule_bucket_compensation(&oi.bucket); let scheduled = self.schedule_bucket_compensation(&oi.bucket);
@@ -755,7 +790,8 @@ impl TransitionState {
pub async fn queue_transition_task(self: &Arc<Self>, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) -> bool { pub async fn queue_transition_task(self: &Arc<Self>, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) -> bool {
if is_immediate_transition_source(src) && should_force_immediate_transition_enqueue_timeout() { if is_immediate_transition_source(src) && should_force_immediate_transition_enqueue_timeout() {
self.handle_immediate_enqueue_failure(oi, src, ImmediateEnqueueFailure::ForcedTimeout); self.handle_immediate_enqueue_failure(oi, src, ImmediateEnqueueFailure::ForcedTimeout);
record_scanner_lifecycle_enqueue_result(src, 1, false); record_scanner_transition_enqueue_result(src, 1, false);
self.record_scanner_transition_state();
return false; return false;
} }
@@ -797,13 +833,15 @@ impl TransitionState {
self.handle_immediate_enqueue_failure(oi, src, ImmediateEnqueueFailure::QueueClosed { timeout_ms: None }); self.handle_immediate_enqueue_failure(oi, src, ImmediateEnqueueFailure::QueueClosed { timeout_ms: None });
} }
} }
record_scanner_lifecycle_enqueue_result(src, 1, queued); record_scanner_transition_enqueue_result(src, 1, queued);
self.record_scanner_transition_state();
return queued; return queued;
} }
match self.transition_tx.try_send(Some(task)) { match self.transition_tx.try_send(Some(task)) {
Ok(()) => queued = true, Ok(()) => queued = true,
Err(async_channel::TrySendError::Full(_)) => { Err(async_channel::TrySendError::Full(_)) => {
Self::inc_counter(&self.queue_full_tasks);
debug!( debug!(
bucket = %oi.bucket, bucket = %oi.bucket,
object = %oi.name, object = %oi.name,
@@ -820,7 +858,8 @@ impl TransitionState {
); );
} }
} }
record_scanner_lifecycle_enqueue_result(src, 1, queued); record_scanner_transition_enqueue_result(src, 1, queued);
self.record_scanner_transition_state();
queued queued
} }
@@ -893,6 +932,7 @@ impl TransitionState {
let task = task.as_any().downcast_ref::<TransitionTask>().expect("TransitionTask downcast failed"); let task = task.as_any().downcast_ref::<TransitionTask>().expect("TransitionTask downcast failed");
TransitionState::inc_counter(&GLOBAL_TransitionState.active_tasks); TransitionState::inc_counter(&GLOBAL_TransitionState.active_tasks);
GLOBAL_TransitionState.record_scanner_transition_state();
let obj_info_for_event = ObjectInfo { let obj_info_for_event = ObjectInfo {
bucket: task.obj_info.bucket.clone(), bucket: task.obj_info.bucket.clone(),
@@ -903,6 +943,7 @@ impl TransitionState {
}; };
if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await { if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
global_metrics().record_scanner_transition_failed(1);
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") { if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
error!("Transition to {} failed for {}/{} version:{} with {}", error!("Transition to {} failed for {}/{} version:{} with {}",
task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.map(|v| v.to_string()).unwrap_or_default(), err.to_string()); task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.map(|v| v.to_string()).unwrap_or_default(), err.to_string());
@@ -917,6 +958,7 @@ impl TransitionState {
..Default::default() ..Default::default()
}); });
} else { } else {
global_metrics().record_scanner_transition_completed(1);
let mut ts = TierStats { let mut ts = TierStats {
total_size: task.obj_info.size as u64, total_size: task.obj_info.size as u64,
num_versions: 1, num_versions: 1,
@@ -938,6 +980,7 @@ impl TransitionState {
}); });
} }
TransitionState::add_counter(&GLOBAL_TransitionState.active_tasks, -1); TransitionState::add_counter(&GLOBAL_TransitionState.active_tasks, -1);
GLOBAL_TransitionState.record_scanner_transition_state();
} }
} }
else => () else => ()
@@ -1006,6 +1049,7 @@ impl TransitionState {
let current_workers = workers.len() as i64; let current_workers = workers.len() as i64;
GLOBAL_TransitionState.num_workers.store(current_workers, Ordering::SeqCst); GLOBAL_TransitionState.num_workers.store(current_workers, Ordering::SeqCst);
GLOBAL_TransitionState.record_scanner_transition_state();
info!( info!(
requested_transition_workers = requested, requested_transition_workers = requested,
@@ -2333,7 +2377,7 @@ mod tests {
BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectInfo, ObjectOptions, PutObjReader, BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectInfo, ObjectOptions, PutObjReader,
}; };
use futures::FutureExt; use futures::FutureExt;
use rustfs_common::metrics::IlmAction; use rustfs_common::metrics::{IlmAction, global_metrics};
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX; use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType}; use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType};
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Timestamp}; use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Timestamp};
@@ -2372,6 +2416,7 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
#[serial]
async fn scanner_transition_enqueue_reports_full_queue() { async fn scanner_transition_enqueue_reports_full_queue() {
let state = TransitionState::new_with_capacity(1); let state = TransitionState::new_with_capacity(1);
let object = ObjectInfo { let object = ObjectInfo {
@@ -2392,6 +2437,35 @@ mod tests {
assert_eq!(state.transition_rx.len(), 1); assert_eq!(state.transition_rx.len(), 1);
} }
#[tokio::test]
#[serial]
async fn scanner_transition_enqueue_updates_transition_status() {
let before = global_metrics().report().await.lifecycle_transition;
let state = TransitionState::new_with_capacity(1);
let object = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
};
let event = crate::bucket::lifecycle::lifecycle::Event {
action: IlmAction::TransitionAction,
..Default::default()
};
let first = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
let second = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
assert!(first);
assert!(!second);
let after = global_metrics().report().await.lifecycle_transition;
assert_eq!(after.scanner_queued.saturating_sub(before.scanner_queued), 1);
assert_eq!(after.scanner_missed.saturating_sub(before.scanner_missed), 1);
assert_eq!(after.queue_full, 1);
assert_eq!(after.current_queue_capacity, 1);
assert_eq!(after.current_queued, 1);
assert_eq!(after.current_active, 0);
}
#[test] #[test]
fn mark_delete_opts_skip_decommissioned_on_remote_success_sets_flag_on_success() { fn mark_delete_opts_skip_decommissioned_on_remote_success_sets_flag_on_success() {
let mut opts = ObjectOptions::default(); let mut opts = ObjectOptions::default();
+62 -1
View File
@@ -18,7 +18,8 @@ use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::Dr
use rustfs_io_metrics::internode_metrics::global_internode_metrics; use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_madmin::metrics::{ use rustfs_madmin::metrics::{
DiskIOStats, DiskMetric, LastMinute as MadminLastMinute, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics, DiskIOStats, DiskMetric, LastMinute as MadminLastMinute, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics,
ScannerMetrics as MadminScannerMetrics, ScannerPacingPressureSnapshot as MadminScannerPacingPressureSnapshot, ScannerLifecycleTransitionSnapshot as MadminScannerLifecycleTransitionSnapshot, ScannerMetrics as MadminScannerMetrics,
ScannerPacingPressureSnapshot as MadminScannerPacingPressureSnapshot,
ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, TimedAction as MadminTimedAction, ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, TimedAction as MadminTimedAction,
}; };
use rustfs_utils::os::get_drive_stats; use rustfs_utils::os::get_drive_stats;
@@ -100,6 +101,10 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
.collect(), .collect(),
}, },
active_paths: metrics.active_paths, active_paths: metrics.active_paths,
current_cycle_lifecycle_expiry_actions: metrics.current_cycle_lifecycle_expiry_actions,
current_cycle_lifecycle_transition_actions: metrics.current_cycle_lifecycle_transition_actions,
last_cycle_lifecycle_expiry_actions: metrics.last_cycle_lifecycle_expiry_actions,
last_cycle_lifecycle_transition_actions: metrics.last_cycle_lifecycle_transition_actions,
last_cycle_partial_source: metrics.last_cycle_partial_source, last_cycle_partial_source: metrics.last_cycle_partial_source,
last_cycle_partial_source_code: metrics.last_cycle_partial_source_code, last_cycle_partial_source_code: metrics.last_cycle_partial_source_code,
pacing_pressure: MadminScannerPacingPressureSnapshot { pacing_pressure: MadminScannerPacingPressureSnapshot {
@@ -112,6 +117,20 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
last_cycle_yield_ratio: metrics.pacing_pressure.last_cycle_yield_ratio, last_cycle_yield_ratio: metrics.pacing_pressure.last_cycle_yield_ratio,
last_cycle_total_pause_ratio: metrics.pacing_pressure.last_cycle_total_pause_ratio, last_cycle_total_pause_ratio: metrics.pacing_pressure.last_cycle_total_pause_ratio,
}, },
lifecycle_transition: MadminScannerLifecycleTransitionSnapshot {
current_queue_capacity: metrics.lifecycle_transition.current_queue_capacity,
current_queued: metrics.lifecycle_transition.current_queued,
current_active: metrics.lifecycle_transition.current_active,
current_workers: metrics.lifecycle_transition.current_workers,
queue_full: metrics.lifecycle_transition.queue_full,
queue_send_timeout: metrics.lifecycle_transition.queue_send_timeout,
compensation_scheduled: metrics.lifecycle_transition.compensation_scheduled,
compensation_running: metrics.lifecycle_transition.compensation_running,
scanner_queued: metrics.lifecycle_transition.scanner_queued,
scanner_missed: metrics.lifecycle_transition.scanner_missed,
completed: metrics.lifecycle_transition.completed,
failed: metrics.lifecycle_transition.failed,
},
partial_cycles_by_source: metrics partial_cycles_by_source: metrics
.partial_cycles_by_source .partial_cycles_by_source
.into_iter() .into_iter()
@@ -412,4 +431,46 @@ mod test {
assert_eq!(scanner.pacing_pressure.last_cycle_yield_ratio, 0.05); assert_eq!(scanner.pacing_pressure.last_cycle_yield_ratio, 0.05);
assert_eq!(scanner.pacing_pressure.last_cycle_total_pause_ratio, 0.3); assert_eq!(scanner.pacing_pressure.last_cycle_total_pause_ratio, 0.3);
} }
#[test]
fn scanner_metrics_mapping_preserves_lifecycle_transition_status() {
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
current_cycle_lifecycle_expiry_actions: 2,
current_cycle_lifecycle_transition_actions: 3,
last_cycle_lifecycle_expiry_actions: 5,
last_cycle_lifecycle_transition_actions: 7,
lifecycle_transition: rustfs_common::metrics::ScannerLifecycleTransitionSnapshot {
current_queue_capacity: 16,
current_queued: 5,
current_active: 2,
current_workers: 4,
queue_full: 3,
queue_send_timeout: 1,
compensation_scheduled: 2,
compensation_running: 1,
scanner_queued: 6,
scanner_missed: 2,
completed: 4,
failed: 1,
},
..Default::default()
});
assert_eq!(scanner.lifecycle_transition.current_queue_capacity, 16);
assert_eq!(scanner.lifecycle_transition.current_queued, 5);
assert_eq!(scanner.lifecycle_transition.current_active, 2);
assert_eq!(scanner.lifecycle_transition.current_workers, 4);
assert_eq!(scanner.lifecycle_transition.queue_full, 3);
assert_eq!(scanner.lifecycle_transition.queue_send_timeout, 1);
assert_eq!(scanner.lifecycle_transition.compensation_scheduled, 2);
assert_eq!(scanner.lifecycle_transition.compensation_running, 1);
assert_eq!(scanner.lifecycle_transition.scanner_queued, 6);
assert_eq!(scanner.lifecycle_transition.scanner_missed, 2);
assert_eq!(scanner.lifecycle_transition.completed, 4);
assert_eq!(scanner.lifecycle_transition.failed, 1);
assert_eq!(scanner.current_cycle_lifecycle_expiry_actions, 2);
assert_eq!(scanner.current_cycle_lifecycle_transition_actions, 3);
assert_eq!(scanner.last_cycle_lifecycle_expiry_actions, 5);
assert_eq!(scanner.last_cycle_lifecycle_transition_actions, 7);
}
} }
+135
View File
@@ -203,6 +203,51 @@ impl ScannerPacingPressureSnapshot {
} }
} }
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerLifecycleTransitionSnapshot {
#[serde(rename = "current_queue_capacity", default)]
pub current_queue_capacity: u64,
#[serde(rename = "current_queued", default)]
pub current_queued: u64,
#[serde(rename = "current_active", default)]
pub current_active: u64,
#[serde(rename = "current_workers", default)]
pub current_workers: u64,
#[serde(rename = "queue_full", default)]
pub queue_full: u64,
#[serde(rename = "queue_send_timeout", default)]
pub queue_send_timeout: u64,
#[serde(rename = "compensation_scheduled", default)]
pub compensation_scheduled: u64,
#[serde(rename = "compensation_running", default)]
pub compensation_running: u64,
#[serde(rename = "scanner_queued", default)]
pub scanner_queued: u64,
#[serde(rename = "scanner_missed", default)]
pub scanner_missed: u64,
#[serde(rename = "completed", default)]
pub completed: u64,
#[serde(rename = "failed", default)]
pub failed: u64,
}
impl ScannerLifecycleTransitionSnapshot {
fn merge(&mut self, other: &Self) {
self.current_queue_capacity = self.current_queue_capacity.saturating_add(other.current_queue_capacity);
self.current_queued = self.current_queued.saturating_add(other.current_queued);
self.current_active = self.current_active.saturating_add(other.current_active);
self.current_workers = self.current_workers.saturating_add(other.current_workers);
self.queue_full = self.queue_full.saturating_add(other.queue_full);
self.queue_send_timeout = self.queue_send_timeout.saturating_add(other.queue_send_timeout);
self.compensation_scheduled = self.compensation_scheduled.saturating_add(other.compensation_scheduled);
self.compensation_running = self.compensation_running.saturating_add(other.compensation_running);
self.scanner_queued = self.scanner_queued.saturating_add(other.scanner_queued);
self.scanner_missed = self.scanner_missed.saturating_add(other.scanner_missed);
self.completed = self.completed.saturating_add(other.completed);
self.failed = self.failed.saturating_add(other.failed);
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)] #[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ScannerMetrics { pub struct ScannerMetrics {
#[serde(rename = "collected")] #[serde(rename = "collected")]
@@ -223,6 +268,14 @@ pub struct ScannerMetrics {
pub last_minute: LastMinute, pub last_minute: LastMinute,
#[serde(rename = "active")] #[serde(rename = "active")]
pub active_paths: Vec<String>, pub active_paths: Vec<String>,
#[serde(rename = "current_cycle_lifecycle_expiry_actions", default)]
pub current_cycle_lifecycle_expiry_actions: u64,
#[serde(rename = "current_cycle_lifecycle_transition_actions", default)]
pub current_cycle_lifecycle_transition_actions: u64,
#[serde(rename = "last_cycle_lifecycle_expiry_actions", default)]
pub last_cycle_lifecycle_expiry_actions: u64,
#[serde(rename = "last_cycle_lifecycle_transition_actions", default)]
pub last_cycle_lifecycle_transition_actions: u64,
#[serde(rename = "last_cycle_partial_source", default)] #[serde(rename = "last_cycle_partial_source", default)]
pub last_cycle_partial_source: String, pub last_cycle_partial_source: String,
#[serde(rename = "last_cycle_partial_source_code", default)] #[serde(rename = "last_cycle_partial_source_code", default)]
@@ -231,6 +284,8 @@ pub struct ScannerMetrics {
pub partial_cycles_by_source: Vec<ScannerSourceCycleSnapshot>, pub partial_cycles_by_source: Vec<ScannerSourceCycleSnapshot>,
#[serde(rename = "pacing_pressure", default)] #[serde(rename = "pacing_pressure", default)]
pub pacing_pressure: ScannerPacingPressureSnapshot, pub pacing_pressure: ScannerPacingPressureSnapshot,
#[serde(rename = "lifecycle_transition", default)]
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
} }
impl ScannerMetrics { impl ScannerMetrics {
@@ -243,6 +298,19 @@ impl ScannerMetrics {
} }
self.pacing_pressure.merge(&other.pacing_pressure); self.pacing_pressure.merge(&other.pacing_pressure);
self.lifecycle_transition.merge(&other.lifecycle_transition);
self.current_cycle_lifecycle_expiry_actions = self
.current_cycle_lifecycle_expiry_actions
.saturating_add(other.current_cycle_lifecycle_expiry_actions);
self.current_cycle_lifecycle_transition_actions = self
.current_cycle_lifecycle_transition_actions
.saturating_add(other.current_cycle_lifecycle_transition_actions);
self.last_cycle_lifecycle_expiry_actions = self
.last_cycle_lifecycle_expiry_actions
.saturating_add(other.last_cycle_lifecycle_expiry_actions);
self.last_cycle_lifecycle_transition_actions = self
.last_cycle_lifecycle_transition_actions
.saturating_add(other.last_cycle_lifecycle_transition_actions);
if self.ongoing_buckets < other.ongoing_buckets { if self.ongoing_buckets < other.ongoing_buckets {
self.ongoing_buckets = other.ongoing_buckets; self.ongoing_buckets = other.ongoing_buckets;
@@ -893,4 +961,71 @@ mod tests {
assert_eq!(pacing_pressure.primary_pressure, "none"); assert_eq!(pacing_pressure.primary_pressure, "none");
assert_eq!(pacing_pressure.current_active_scans, 1); assert_eq!(pacing_pressure.current_active_scans, 1);
} }
#[test]
fn scanner_metrics_merge_aggregates_lifecycle_transition_status() {
let collected_at = Utc::now();
let mut scanner = ScannerMetrics {
collected_at,
current_cycle_lifecycle_expiry_actions: 2,
current_cycle_lifecycle_transition_actions: 3,
last_cycle_lifecycle_expiry_actions: 5,
last_cycle_lifecycle_transition_actions: 7,
lifecycle_transition: ScannerLifecycleTransitionSnapshot {
current_queue_capacity: 8,
current_queued: 2,
current_active: 1,
current_workers: 2,
queue_full: 3,
queue_send_timeout: 1,
compensation_scheduled: 1,
compensation_running: 1,
scanner_queued: 5,
scanner_missed: 2,
completed: 7,
failed: 1,
},
..Default::default()
};
scanner.merge(&ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
current_cycle_lifecycle_expiry_actions: 11,
current_cycle_lifecycle_transition_actions: 13,
last_cycle_lifecycle_expiry_actions: 17,
last_cycle_lifecycle_transition_actions: 19,
lifecycle_transition: ScannerLifecycleTransitionSnapshot {
current_queue_capacity: 4,
current_queued: 3,
current_active: 2,
current_workers: 1,
queue_full: 2,
queue_send_timeout: 3,
compensation_scheduled: 4,
compensation_running: 0,
scanner_queued: 6,
scanner_missed: 4,
completed: 8,
failed: 2,
},
..Default::default()
});
assert_eq!(scanner.lifecycle_transition.current_queue_capacity, 12);
assert_eq!(scanner.lifecycle_transition.current_queued, 5);
assert_eq!(scanner.lifecycle_transition.current_active, 3);
assert_eq!(scanner.lifecycle_transition.current_workers, 3);
assert_eq!(scanner.lifecycle_transition.queue_full, 5);
assert_eq!(scanner.lifecycle_transition.queue_send_timeout, 4);
assert_eq!(scanner.lifecycle_transition.compensation_scheduled, 5);
assert_eq!(scanner.lifecycle_transition.compensation_running, 1);
assert_eq!(scanner.lifecycle_transition.scanner_queued, 11);
assert_eq!(scanner.lifecycle_transition.scanner_missed, 6);
assert_eq!(scanner.lifecycle_transition.completed, 15);
assert_eq!(scanner.lifecycle_transition.failed, 3);
assert_eq!(scanner.current_cycle_lifecycle_expiry_actions, 13);
assert_eq!(scanner.current_cycle_lifecycle_transition_actions, 16);
assert_eq!(scanner.last_cycle_lifecycle_expiry_actions, 22);
assert_eq!(scanner.last_cycle_lifecycle_transition_actions, 26);
}
} }
+23 -8
View File
@@ -188,9 +188,9 @@ fn should_log_failed_object(failed_objects: usize) -> bool {
failed_objects <= SCANNER_FAILED_OBJECT_LOG_INITIAL_LIMIT || failed_objects.is_multiple_of(SCANNER_FAILED_OBJECT_LOG_EVERY) failed_objects <= SCANNER_FAILED_OBJECT_LOG_INITIAL_LIMIT || failed_objects.is_multiple_of(SCANNER_FAILED_OBJECT_LOG_EVERY)
} }
fn record_scanner_ilm_action_if_queued(metrics: &Metrics, count: u64, queued: bool) -> bool { fn record_scanner_ilm_action_if_queued(metrics: &Metrics, action: IlmAction, count: u64, queued: bool) -> bool {
if queued { if queued {
metrics.record_scanner_ilm_action(count); metrics.record_scanner_lifecycle_action(action, count);
} }
queued queued
} }
@@ -679,7 +679,7 @@ impl ScannerItem {
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action); debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
let done_ilm = Metrics::time_ilm(event.action); let done_ilm = Metrics::time_ilm(event.action);
let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
if record_scanner_ilm_action_if_queued(global_metrics(), 1, queued) { if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
done_ilm(1)(); done_ilm(1)();
remaining_versions = 0; remaining_versions = 0;
} else { } else {
@@ -712,7 +712,7 @@ impl ScannerItem {
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action); debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
let done_ilm = Metrics::time_ilm(event.action); let done_ilm = Metrics::time_ilm(event.action);
let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
if record_scanner_ilm_action_if_queued(global_metrics(), 1, queued) { if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
done_ilm(1)(); done_ilm(1)();
if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction { if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction {
remaining_versions -= 1; remaining_versions -= 1;
@@ -740,7 +740,7 @@ impl ScannerItem {
debug!("apply_actions: applying transition rule for object: {} {}", oi.name, event.action); debug!("apply_actions: applying transition rule for object: {} {}", oi.name, event.action);
let done_ilm = Metrics::time_ilm(event.action); let done_ilm = Metrics::time_ilm(event.action);
let queued = apply_transition_rule(event, &LcEventSrc::Scanner, oi).await; let queued = apply_transition_rule(event, &LcEventSrc::Scanner, oi).await;
if record_scanner_ilm_action_if_queued(global_metrics(), 1, queued) { if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
done_ilm(1)(); done_ilm(1)();
} }
} }
@@ -768,7 +768,7 @@ impl ScannerItem {
.await .await
.enqueue_by_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner) .enqueue_by_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner)
.await; .await;
if record_scanner_ilm_action_if_queued(global_metrics(), count, queued) { if record_scanner_ilm_action_if_queued(global_metrics(), action, count, queued) {
done_ilm(count)(); done_ilm(count)();
remaining_versions = remaining_versions.saturating_sub(noncurrent_accounting.len()); remaining_versions = remaining_versions.saturating_sub(noncurrent_accounting.len());
} }
@@ -2169,8 +2169,9 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_scanner_ilm_action_accounting_requires_enqueue_success() { async fn test_scanner_ilm_action_accounting_requires_enqueue_success() {
let metrics = Metrics::new(); let metrics = Metrics::new();
let _start = metrics.start_scan_cycle_work();
record_scanner_ilm_action_if_queued(&metrics, 2, false); record_scanner_ilm_action_if_queued(&metrics, IlmAction::DeleteAction, 2, false);
let report = metrics.report().await; let report = metrics.report().await;
let lifecycle = report let lifecycle = report
.source_work .source_work
@@ -2178,8 +2179,9 @@ mod tests {
.find(|work| work.source == ScannerWorkSource::Lifecycle.as_str()) .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
.expect("lifecycle source work should be visible"); .expect("lifecycle source work should be visible");
assert_eq!(lifecycle.executed, 0); assert_eq!(lifecycle.executed, 0);
assert_eq!(report.current_cycle_lifecycle_expiry_actions, 0);
record_scanner_ilm_action_if_queued(&metrics, 3, true); record_scanner_ilm_action_if_queued(&metrics, IlmAction::DeleteAction, 3, true);
let report = metrics.report().await; let report = metrics.report().await;
let lifecycle = report let lifecycle = report
.source_work .source_work
@@ -2187,6 +2189,19 @@ mod tests {
.find(|work| work.source == ScannerWorkSource::Lifecycle.as_str()) .find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
.expect("lifecycle source work should be visible"); .expect("lifecycle source work should be visible");
assert_eq!(lifecycle.executed, 3); assert_eq!(lifecycle.executed, 3);
assert_eq!(report.current_cycle_lifecycle_expiry_actions, 3);
assert_eq!(report.current_cycle_lifecycle_transition_actions, 0);
record_scanner_ilm_action_if_queued(&metrics, IlmAction::TransitionAction, 4, true);
let report = metrics.report().await;
let lifecycle = report
.source_work
.iter()
.find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
.expect("lifecycle source work should be visible");
assert_eq!(lifecycle.executed, 7);
assert_eq!(report.current_cycle_lifecycle_expiry_actions, 3);
assert_eq!(report.current_cycle_lifecycle_transition_actions, 4);
} }
#[test] #[test]