fix: harden lifecycle transition compensation and regression coverage (#2995)

* fix(ecstore): honor transition worker configuration

* fix(ecstore): add transition queue backpressure metrics

* fix(ecstore): schedule transition compensation on enqueue pressure

* fix(ecstore): log transition compensation scheduling

* test(rustfs): add transition compensation fault-injection coverage

* test(rustfs): cover delete after transition compensation

* test(scanner): cover cleanup after transition compensation

* test(rustfs): extend compensation transition coverage

* test(scanner): cover backfill idempotency after compensation

* test(scanner): cover noncurrent expiry after compensation

* test(rustfs): cover versioned delete after compensation

* test(rustfs): cover delete marker lifecycle after compensation

* test(scanner): extend versioned lifecycle compensation coverage

* test(scanner): model versioned delete after compensation

* test(scanner): clarify modeled versioned delete helper

* refactor(ecstore): optimize transition enqueue hot path

* refactor(ecstore): centralize transition runtime constants

* style(ecstore): apply rustfmt for transition timeout helper

* fix(ilm): align queue-full metric semantics

* refactor(ecstore): unify immediate enqueue failure handling

* refactor(ecstore): reuse transition worker env constant

* ci(actions): update setup action inputs

crates/obs/src/metrics/collectors/ilm.rs

@@ -36,6 +36,14 @@ pub struct IlmStats {
     pub transition_pending_tasks: u64,
     /// Number of missed immediate ILM transition tasks
     pub transition_missed_immediate_tasks: u64,
+    /// Number of ILM transition tasks that initially hit full queue backpressure
+    pub transition_queue_full_tasks: u64,
+    /// Number of ILM transition tasks that timed out waiting for queue capacity
+    pub transition_queue_send_timeout_tasks: u64,
+    /// Number of bucket-level compensation tasks scheduled after immediate enqueue failure
+    pub transition_compensation_scheduled_tasks: u64,
+    /// Number of bucket-level compensation tasks currently running
+    pub transition_compensation_running_tasks: u64,
     /// Total number of object versions scanned for ILM
     pub versions_scanned: u64,
 }
@@ -53,6 +61,19 @@ pub fn collect_ilm_metrics(stats: &IlmStats) -> Vec<PrometheusMetric> {
             &ILM_TRANSITION_MISSED_IMMEDIATE_TASKS_MD,
             stats.transition_missed_immediate_tasks as f64,
         ),
+        PrometheusMetric::from_descriptor(&ILM_TRANSITION_QUEUE_FULL_TASKS_MD, stats.transition_queue_full_tasks as f64),
+        PrometheusMetric::from_descriptor(
+            &ILM_TRANSITION_QUEUE_SEND_TIMEOUT_TASKS_MD,
+            stats.transition_queue_send_timeout_tasks as f64,
+        ),
+        PrometheusMetric::from_descriptor(
+            &ILM_TRANSITION_COMPENSATION_SCHEDULED_TASKS_MD,
+            stats.transition_compensation_scheduled_tasks as f64,
+        ),
+        PrometheusMetric::from_descriptor(
+            &ILM_TRANSITION_COMPENSATION_RUNNING_TASKS_MD,
+            stats.transition_compensation_running_tasks as f64,
+        ),
         PrometheusMetric::from_descriptor(&ILM_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
     ]
 }
@@ -68,12 +89,16 @@ mod tests {
             transition_active_tasks: 5,
             transition_pending_tasks: 50,
             transition_missed_immediate_tasks: 10,
+            transition_queue_full_tasks: 2,
+            transition_queue_send_timeout_tasks: 3,
+            transition_compensation_scheduled_tasks: 4,
+            transition_compensation_running_tasks: 1,
             versions_scanned: 1000000,
         };
 
         let metrics = collect_ilm_metrics(&stats);
 
-        assert_eq!(metrics.len(), 5);
+        assert_eq!(metrics.len(), 9);
 
         let pending = metrics.iter().find(|m| m.value == 100.0);
         assert!(pending.is_some());
@@ -87,7 +112,7 @@ mod tests {
         let stats = IlmStats::default();
         let metrics = collect_ilm_metrics(&stats);
 
-        assert_eq!(metrics.len(), 5);
+        assert_eq!(metrics.len(), 9);
         for metric in &metrics {
             assert_eq!(metric.value, 0.0);
             assert!(metric.labels.is_empty());

crates/obs/src/metrics/schema/entry/metric_name.rs

@@ -248,6 +248,10 @@ pub enum MetricName {
     IlmTransitionActiveTasks,
     IlmTransitionPendingTasks,
     IlmTransitionMissedImmediateTasks,
+    IlmTransitionQueueFullTasks,
+    IlmTransitionQueueSendTimeoutTasks,
+    IlmTransitionCompensationScheduledTasks,
+    IlmTransitionCompensationRunningTasks,
     IlmVersionsScanned,
 
     // Copy the relevant metrics
@@ -584,6 +588,10 @@ impl MetricName {
             Self::IlmTransitionActiveTasks => "transition_active_tasks".to_string(),
             Self::IlmTransitionPendingTasks => "transition_pending_tasks".to_string(),
             Self::IlmTransitionMissedImmediateTasks => "transition_missed_immediate_tasks".to_string(),
+            Self::IlmTransitionQueueFullTasks => "transition_queue_full_tasks".to_string(),
+            Self::IlmTransitionQueueSendTimeoutTasks => "transition_queue_send_timeout_tasks".to_string(),
+            Self::IlmTransitionCompensationScheduledTasks => "transition_compensation_scheduled_tasks".to_string(),
+            Self::IlmTransitionCompensationRunningTasks => "transition_compensation_running_tasks".to_string(),
             Self::IlmVersionsScanned => "versions_scanned".to_string(),
 
             // Copy the relevant metrics

crates/obs/src/metrics/schema/ilm.rs

@@ -53,6 +53,42 @@ pub static ILM_TRANSITION_MISSED_IMMEDIATE_TASKS_MD: LazyLock<MetricDescriptor>
     )
 });
 
+pub static ILM_TRANSITION_QUEUE_FULL_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::IlmTransitionQueueFullTasks,
+        "Number of ILM transition tasks that initially hit full transition queue backpressure",
+        &[],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_TRANSITION_QUEUE_SEND_TIMEOUT_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::IlmTransitionQueueSendTimeoutTasks,
+        "Number of ILM transition tasks that timed out waiting for queue capacity",
+        &[],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_TRANSITION_COMPENSATION_SCHEDULED_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::IlmTransitionCompensationScheduledTasks,
+        "Number of bucket-level ILM transition compensation tasks scheduled after enqueue failure",
+        &[],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_TRANSITION_COMPENSATION_RUNNING_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::IlmTransitionCompensationRunningTasks,
+        "Number of bucket-level ILM transition compensation tasks currently running",
+        &[],
+        subsystems::ILM,
+    )
+});
+
 pub static ILM_VERSIONS_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(
         MetricName::IlmVersionsScanned,

crates/obs/src/metrics/stats_collector.rs

@@ -840,6 +840,10 @@ pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
     let transition_active_tasks = GLOBAL_TransitionState.active_tasks().max(0) as u64;
     let transition_pending_tasks = GLOBAL_TransitionState.pending_tasks() as u64;
     let transition_missed_immediate_tasks = GLOBAL_TransitionState.missed_immediate_tasks().max(0) as u64;
+    let transition_queue_full_tasks = GLOBAL_TransitionState.queue_full_tasks().max(0) as u64;
+    let transition_queue_send_timeout_tasks = GLOBAL_TransitionState.queue_send_timeout_tasks().max(0) as u64;
+    let transition_compensation_scheduled_tasks = GLOBAL_TransitionState.compensation_scheduled_tasks().max(0) as u64;
+    let transition_compensation_running_tasks = GLOBAL_TransitionState.compensation_running_tasks().max(0) as u64;
     let metrics = global_metrics().report().await;
     let versions_scanned = metrics.life_time_ilm.values().copied().sum();
 
@@ -848,6 +852,10 @@ pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
         transition_active_tasks,
         transition_pending_tasks,
         transition_missed_immediate_tasks,
+        transition_queue_full_tasks,
+        transition_queue_send_timeout_tasks,
+        transition_compensation_scheduled_tasks,
+        transition_compensation_running_tasks,
         versions_scanned,
     })
 }