feat(ecstore): add the on-demand migration backfill job (#7087)

* feat(ecstore): add on-demand migration backfill job core

Add the background backfill job for on-demand migration
(rustfs/backlog#2159): a durable checkpoint under
buckets/<bucket>/on-demand-migration-backfill.json saved by If-Match
compare-and-set every 1000 keys or 10 s, a 60 s owner lease renewed by
every save, a recovery pass that takes over expired leases (or jobs this
node owned before a restart) and cancels jobs whose config changed, and a
main loop over the source ListObjectsV2 pages with the skip_existing
policy, dry runs, bounded outstanding pulls and wait-on-full enqueueing.

The pull queue gains per-job completion reports so the job can count
pulled/failed keys (hashes only), and pull permits become two-tier so an
online miss is never queued behind a backfill pull.

* feat(admin): expose on-demand migration backfill job

Wire the ODM-12 backfill job (rustfs/backlog#2159) to its operators:
POST /v3/on-demand-migration/{bucket}/backfill?op=start|cancel and
GET .../backfill return the checkpoint document, GET .../status gains a
backfill summary, and the recovery loop plus the process-wide runner are
installed at startup. Backfill control reuses
Set/GetBucketOnDemandMigrationAction and is recorded in the route policy,
the registration matrix and the admin route snapshot.

Add the rustfs-madmin wire types and client methods with golden fixtures
shared by the server tests, the backfill_* metric descriptors and their
collector, and three e2e scenarios: a full backfill across list pages,
cancellation, and resuming from the persisted continuation token after a
server restart.

crates/obs/src/metrics/collectors/mod.rs

@@ -70,7 +70,8 @@ pub use notification::{NotificationStats, collect_notification_metrics};
 pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
 pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};
 pub use on_demand_migration::{
-    OnDemandMigrationBreakerState, OnDemandMigrationBucketStats, collect_on_demand_migration_metrics, source_latency_le_label,
+    OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBreakerState, OnDemandMigrationBucketStats,
+    collect_on_demand_migration_backfill_metrics, collect_on_demand_migration_metrics, source_latency_le_label,
 };
 pub use replication::{ReplicationMetricsSnapshot, collect_replication_metrics};
 pub(crate) use replication::{ReplicationRuntimeStats, collect_replication_runtime_metrics};

crates/obs/src/metrics/collectors/on_demand_migration.rs

@@ -21,10 +21,12 @@
 
 use crate::metrics::report::PrometheusMetric;
 use crate::metrics::schema::on_demand_migration::{
-    BREAKER_STATE_CLOSED, BREAKER_STATE_HALF_OPEN, BREAKER_STATE_OPEN, BUCKET_L, LE_L, ODM_BREAKER_STATE_MD,
-    ODM_INFLIGHT_PULLS_MD, ODM_PULL_FAILURES_TOTAL_MD, ODM_PULLED_BYTES_TOTAL_MD, ODM_PULLED_OBJECTS_TOTAL_MD,
-    ODM_QUEUE_DEPTH_MD, ODM_REQUESTS_TOTAL_MD, ODM_SOURCE_LATENCY_SECONDS_COUNT_MD, ODM_SOURCE_LATENCY_SECONDS_DISTRIBUTION_MD,
-    ODM_SOURCE_LATENCY_SECONDS_SUM_MD, OP_L, OUTCOME_L, PATH_L, REASON_L,
+    BREAKER_STATE_CLOSED, BREAKER_STATE_HALF_OPEN, BREAKER_STATE_OPEN, BUCKET_L, LE_L, ODM_BACKFILL_BYTES_MD,
+    ODM_BACKFILL_ENQUEUED_MD, ODM_BACKFILL_FAILED_MD, ODM_BACKFILL_JOBS_MD, ODM_BACKFILL_LISTED_MD, ODM_BACKFILL_PULLED_MD,
+    ODM_BACKFILL_SKIPPED_EXISTING_MD, ODM_BREAKER_STATE_MD, ODM_INFLIGHT_PULLS_MD, ODM_PULL_FAILURES_TOTAL_MD,
+    ODM_PULLED_BYTES_TOTAL_MD, ODM_PULLED_OBJECTS_TOTAL_MD, ODM_QUEUE_DEPTH_MD, ODM_REQUESTS_TOTAL_MD,
+    ODM_SOURCE_LATENCY_SECONDS_COUNT_MD, ODM_SOURCE_LATENCY_SECONDS_DISTRIBUTION_MD, ODM_SOURCE_LATENCY_SECONDS_SUM_MD, OP_L,
+    OUTCOME_L, PATH_L, REASON_L, SERVER_L, STATE_L,
 };
 use std::borrow::Cow;
 use std::collections::BTreeMap;
@@ -170,6 +172,48 @@ pub fn collect_on_demand_migration_metrics(stats: &[OnDemandMigrationBucketStats
     metrics
 }
 
+/// Counters of one bucket's latest backfill job (ODM-12,
+/// rustfs/backlog#2159).
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct OdmBackfillBucketStats {
+    pub bucket: String,
+    /// Checkpoint state label (`running`, `completed`, ...).
+    pub state: String,
+    pub listed: u64,
+    pub enqueued: u64,
+    pub pulled: u64,
+    pub skipped_existing: u64,
+    pub failed: u64,
+    pub bytes: u64,
+}
+
+/// Backfill stats of every bucket with a checkpoint, labelled by node.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct OdmBackfillRuntimeStats {
+    pub server: String,
+    pub buckets: Vec<OdmBackfillBucketStats>,
+}
+
+/// Seven series per bucket: the state gauge and six counters.
+pub fn collect_on_demand_migration_backfill_metrics(stats: &OdmBackfillRuntimeStats) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(stats.buckets.len() * 7);
+    for bucket in &stats.buckets {
+        let labelled = |descriptor: &'static std::sync::LazyLock<crate::MetricDescriptor>, value: u64| {
+            PrometheusMetric::from_descriptor(descriptor, value as f64)
+                .with_label_owned(SERVER_L, stats.server.clone())
+                .with_label_owned(BUCKET_L, bucket.bucket.clone())
+        };
+        metrics.push(labelled(&ODM_BACKFILL_JOBS_MD, 1).with_label_owned(STATE_L, bucket.state.clone()));
+        metrics.push(labelled(&ODM_BACKFILL_LISTED_MD, bucket.listed));
+        metrics.push(labelled(&ODM_BACKFILL_ENQUEUED_MD, bucket.enqueued));
+        metrics.push(labelled(&ODM_BACKFILL_PULLED_MD, bucket.pulled));
+        metrics.push(labelled(&ODM_BACKFILL_SKIPPED_EXISTING_MD, bucket.skipped_existing));
+        metrics.push(labelled(&ODM_BACKFILL_FAILED_MD, bucket.failed));
+        metrics.push(labelled(&ODM_BACKFILL_BYTES_MD, bucket.bytes));
+    }
+    metrics
+}
+
 #[cfg(test)]
 pub(crate) mod tests {
     use super::*;
@@ -354,4 +398,81 @@ pub(crate) mod tests {
     fn empty_snapshot_emits_nothing() {
         assert!(collect_on_demand_migration_metrics(&[]).is_empty());
     }
+
+    /// One running and one completed job, as the scheduler would see them
+    /// from the local backfill runner.
+    pub(crate) fn backfill_golden_stats(server: &str) -> OdmBackfillRuntimeStats {
+        OdmBackfillRuntimeStats {
+            server: server.to_string(),
+            buckets: vec![
+                OdmBackfillBucketStats {
+                    bucket: "photos".to_string(),
+                    state: "running".to_string(),
+                    listed: 2000,
+                    enqueued: 1500,
+                    pulled: 1400,
+                    skipped_existing: 500,
+                    failed: 3,
+                    bytes: 73_400_320,
+                },
+                OdmBackfillBucketStats {
+                    bucket: "docs".to_string(),
+                    state: "completed".to_string(),
+                    ..Default::default()
+                },
+            ],
+        }
+    }
+
+    fn backfill_series<'a>(metrics: &'a [PrometheusMetric], name: &str, bucket: &str) -> Option<&'a PrometheusMetric> {
+        metrics.iter().find(|metric| {
+            metric.name == name
+                && metric
+                    .labels
+                    .iter()
+                    .any(|(label, value)| *label == BUCKET_L && value.as_ref() == bucket)
+        })
+    }
+
+    #[test]
+    fn backfill_collects_seven_series_per_bucket_with_the_odm_subsystem_prefix() {
+        let stats = backfill_golden_stats("node1:9000");
+        let metrics = collect_on_demand_migration_backfill_metrics(&stats);
+        assert_eq!(metrics.len(), 14);
+        assert!(
+            metrics
+                .iter()
+                .all(|metric| metric.name.starts_with("rustfs_on_demand_migration_backfill_"))
+        );
+        assert!(metrics.iter().all(|metric| {
+            metric
+                .labels
+                .iter()
+                .any(|(label, value)| *label == SERVER_L && value.as_ref() == "node1:9000")
+        }));
+
+        let jobs = backfill_series(&metrics, &ODM_BACKFILL_JOBS_MD.get_full_metric_name(), "photos").expect("jobs gauge");
+        assert_eq!(jobs.value, 1.0);
+        assert!(
+            jobs.labels
+                .iter()
+                .any(|(label, value)| *label == STATE_L && value.as_ref() == "running")
+        );
+        let listed = backfill_series(&metrics, &ODM_BACKFILL_LISTED_MD.get_full_metric_name(), "photos").expect("listed");
+        assert_eq!(listed.value, 2000.0);
+        let bytes = backfill_series(&metrics, &ODM_BACKFILL_BYTES_MD.get_full_metric_name(), "photos").expect("bytes");
+        assert_eq!(bytes.value, 73_400_320.0);
+        let docs_failed = backfill_series(&metrics, &ODM_BACKFILL_FAILED_MD.get_full_metric_name(), "docs").expect("docs failed");
+        assert_eq!(docs_failed.value, 0.0);
+        assert_eq!(
+            ODM_BACKFILL_LISTED_MD.get_full_metric_name(),
+            "rustfs_on_demand_migration_backfill_listed_total"
+        );
+    }
+
+    #[test]
+    fn backfill_no_buckets_means_no_series() {
+        let metrics = collect_on_demand_migration_backfill_metrics(&OdmBackfillRuntimeStats::default());
+        assert!(metrics.is_empty());
+    }
 }

crates/obs/src/metrics/mod.rs

@@ -34,6 +34,7 @@ pub(crate) use storage_api::metrics::{
     BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
     StorageAdminApi, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor,
     obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
-    obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_on_demand_migration_snapshot,
-    obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, obs_transition_state_handle,
+    obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_on_demand_migration_backfill_snapshot,
+    obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
+    obs_transition_state_handle,
 };

crates/obs/src/metrics/scheduler.rs

@@ -33,6 +33,7 @@ use crate::metrics::collectors::{
     NotificationStats,
     NotificationTargetRuntimeStats,
     NotificationTargetStats,
+    OdmBackfillRuntimeStats,
     OnDemandMigrationBucketStats,
     // System monitoring collectors (migrated from rustfs-obs::system)
     ProcessAttributeError,
@@ -65,6 +66,7 @@ use crate::metrics::collectors::{
     collect_node_metrics,
     collect_notification_runtime_metrics,
     collect_notification_target_runtime_metrics,
+    collect_on_demand_migration_backfill_metrics,
     collect_on_demand_migration_metrics,
     collect_process_attributes,
     collect_process_cpu_metrics,
@@ -121,12 +123,15 @@ use crate::metrics::schema::notification_target::{
     TARGET_ID as NOTIFICATION_TARGET_ID_LABEL, TARGET_TYPE as NOTIFICATION_TARGET_TYPE_LABEL,
 };
 use crate::metrics::schema::on_demand_migration::{
-    BUCKET_L as ODM_BUCKET_L, LE_L as ODM_LE_L, ODM_BREAKER_STATE_MD, ODM_INFLIGHT_PULLS_MD, ODM_PULL_FAILURES_TOTAL_MD,
+    BACKFILL_STATES as ODM_BACKFILL_STATES, BUCKET_L as ODM_BUCKET_L, LE_L as ODM_LE_L, ODM_BACKFILL_BYTES_MD,
+    ODM_BACKFILL_ENQUEUED_MD, ODM_BACKFILL_FAILED_MD, ODM_BACKFILL_JOBS_MD, ODM_BACKFILL_LISTED_MD, ODM_BACKFILL_PULLED_MD,
+    ODM_BACKFILL_SKIPPED_EXISTING_MD, ODM_BREAKER_STATE_MD, ODM_INFLIGHT_PULLS_MD, ODM_PULL_FAILURES_TOTAL_MD,
     ODM_PULLED_BYTES_TOTAL_MD, ODM_PULLED_OBJECTS_TOTAL_MD, ODM_QUEUE_DEPTH_MD, ODM_REQUESTS_TOTAL_MD,
     ODM_SOURCE_LATENCY_SECONDS_COUNT_MD, ODM_SOURCE_LATENCY_SECONDS_DISTRIBUTION_MD, ODM_SOURCE_LATENCY_SECONDS_SUM_MD,
     OP_L as ODM_OP_L, OUTCOME_L as ODM_OUTCOME_L, PATH_L as ODM_PATH_L, PULL_FAILURE_REASONS as ODM_PULL_FAILURE_REASONS,
     PULL_PATHS as ODM_PULL_PATHS, REASON_L as ODM_REASON_L, REQUEST_OPS as ODM_REQUEST_OPS,
-    REQUEST_OUTCOMES as ODM_REQUEST_OUTCOMES, SOURCE_LATENCY_LE as ODM_SOURCE_LATENCY_LE,
+    REQUEST_OUTCOMES as ODM_REQUEST_OUTCOMES, SERVER_L as ODM_SERVER_L, SOURCE_LATENCY_LE as ODM_SOURCE_LATENCY_LE,
+    STATE_L as ODM_STATE_L,
 };
 use crate::metrics::schema::scanner::{
     BUCKET_LABEL as SCANNER_BUCKET_LABEL, CYCLE_SCOPE_LABEL as SCANNER_CYCLE_SCOPE_LABEL, DRIVE_LABEL as SCANNER_DRIVE_LABEL,
@@ -144,9 +149,9 @@ use crate::metrics::stats_collector::{
     collect_bucket_replication_stats_bundle, collect_bucket_stats, collect_cluster_and_health_stats,
     collect_cluster_config_stats, collect_cluster_usage_metric_stats, collect_compression_cluster_stats,
     collect_disk_and_system_drive_runtime_stats, collect_erasure_set_stats, collect_host_network_stats, collect_iam_stats,
-    collect_ilm_runtime_metric_stats, collect_internode_network_stats, collect_on_demand_migration_stats,
-    collect_process_metric_bundle_with, collect_replication_stats, collect_scanner_runtime_metric_stats,
-    collect_system_cpu_and_memory_stats_with,
+    collect_ilm_runtime_metric_stats, collect_internode_network_stats, collect_on_demand_migration_backfill_stats,
+    collect_on_demand_migration_stats, collect_process_metric_bundle_with, collect_replication_stats,
+    collect_scanner_runtime_metric_stats, collect_system_cpu_and_memory_stats_with,
 };
 use crate::node_identity::{SERVER_LABEL, current_local_node_identity};
 use crate::telemetry::retire_metric_series;
@@ -914,6 +919,10 @@ fn on_demand_migration_bucket_live_keys(stats: &[OnDemandMigrationBucketStats])
     stats.iter().map(|stat| stat.bucket.clone()).collect()
 }
 
+fn on_demand_migration_backfill_bucket_live_keys(stats: &OdmBackfillRuntimeStats) -> HashSet<BucketKey> {
+    stats.buckets.iter().map(|stat| stat.bucket.clone()).collect()
+}
+
 fn update_series_zero_tombstones<T: Clone + Eq + std::hash::Hash>(
     has_seen_valid_snapshot: &mut bool,
     prev_live_keys: &mut HashSet<T>,
@@ -1646,6 +1655,42 @@ fn retire_on_demand_migration_metric_series(bucket: &str) -> usize {
         .sum()
 }
 
+/// Every on-demand migration backfill series one node's job can own. The
+/// `state` gauge is enumerated over the checkpoint's fixed lifecycle values
+/// because only one of them is emitted per cycle.
+fn on_demand_migration_backfill_metric_series(server: &str, bucket: &str) -> Vec<MetricSeriesKey> {
+    let job_labels = || {
+        vec![
+            (ODM_SERVER_L, Cow::Owned(server.to_string())),
+            (ODM_BUCKET_L, Cow::Owned(bucket.to_string())),
+        ]
+    };
+    let mut series = Vec::new();
+    for state in ODM_BACKFILL_STATES {
+        let mut labels = job_labels();
+        labels.push((ODM_STATE_L, Cow::Borrowed(state)));
+        series.push((ODM_BACKFILL_JOBS_MD.get_full_metric_name(), labels));
+    }
+    for descriptor in [
+        &*ODM_BACKFILL_LISTED_MD,
+        &*ODM_BACKFILL_ENQUEUED_MD,
+        &*ODM_BACKFILL_PULLED_MD,
+        &*ODM_BACKFILL_SKIPPED_EXISTING_MD,
+        &*ODM_BACKFILL_FAILED_MD,
+        &*ODM_BACKFILL_BYTES_MD,
+    ] {
+        series.push((descriptor.get_full_metric_name(), job_labels()));
+    }
+    series
+}
+
+fn retire_on_demand_migration_backfill_metric_series(server: &str, bucket: &str) -> usize {
+    on_demand_migration_backfill_metric_series(server, bucket)
+        .iter()
+        .map(|(name, labels)| retire_metric_series(name, labels))
+        .sum()
+}
+
 fn retire_repl_backlog_target_metric_series(bucket: &str, target_arn: &str) -> usize {
     let labels = [
         (BUCKET_L, Cow::Owned(bucket.to_string())),
@@ -2037,6 +2082,8 @@ pub fn init_metrics_runtime(token: CancellationToken) {
         let mut has_seen_proxy_bucket_snapshot = false;
         let mut prev_on_demand_migration_live_keys: HashSet<BucketKey> = HashSet::new();
         let mut has_seen_on_demand_migration_snapshot = false;
+        let mut prev_on_demand_migration_backfill_live_keys: HashSet<BucketKey> = HashSet::new();
+        let mut has_seen_on_demand_migration_backfill_snapshot = false;
         loop {
             tokio::select! {
                 _ = interval.tick() => {
@@ -2125,6 +2172,24 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                             prev_on_demand_migration_live_keys = current_on_demand_migration_live_keys;
                             has_seen_on_demand_migration_snapshot = true;
                             metrics.extend(collect_on_demand_migration_metrics(&on_demand_migration));
+                            // Backfill jobs come and go independently of the bucket's config,
+                            // so their series are retired on their own key set.
+                            let on_demand_migration_backfill = collect_on_demand_migration_backfill_stats();
+                            let current_on_demand_migration_backfill_live_keys =
+                                on_demand_migration_backfill_bucket_live_keys(&on_demand_migration_backfill);
+                            let retire_on_demand_migration_backfill_buckets = if has_seen_on_demand_migration_backfill_snapshot
+                            {
+                                prev_on_demand_migration_backfill_live_keys
+                                    .difference(&current_on_demand_migration_backfill_live_keys)
+                                    .cloned()
+                                    .collect::<Vec<_>>()
+                            } else {
+                                Vec::new()
+                            };
+                            prev_on_demand_migration_backfill_live_keys = current_on_demand_migration_backfill_live_keys;
+                            has_seen_on_demand_migration_backfill_snapshot = true;
+                            let on_demand_migration_backfill_server = on_demand_migration_backfill.server.clone();
+                            metrics.extend(collect_on_demand_migration_backfill_metrics(&on_demand_migration_backfill));
                             let replication = collect_replication_stats().await;
                             metrics.extend(collect_replication_runtime_metrics(&ReplicationRuntimeStats {
                                 server: current_local_node_identity(),
@@ -2154,6 +2219,12 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                             for bucket in retire_on_demand_migration_buckets {
                                 let _ = retire_on_demand_migration_metric_series(&bucket);
                             }
+                            for bucket in retire_on_demand_migration_backfill_buckets {
+                                let _ = retire_on_demand_migration_backfill_metric_series(
+                                    &on_demand_migration_backfill_server,
+                                    &bucket,
+                                );
+                            }
                         },
                     ).await;
                 }
@@ -2980,6 +3051,66 @@ mod tests {
         assert_eq!(on_demand_migration_metric_series("photos").len(), emitted.len());
     }
 
+    #[test]
+    fn on_demand_migration_backfill_bucket_keys_detect_finished_jobs() {
+        let stats = crate::metrics::collectors::on_demand_migration::tests::backfill_golden_stats("node1:9000");
+        let previous = on_demand_migration_backfill_bucket_live_keys(&stats);
+        let current = on_demand_migration_backfill_bucket_live_keys(&OdmBackfillRuntimeStats {
+            server: stats.server.clone(),
+            buckets: stats.buckets[..1].to_vec(),
+        });
+        let retired = previous.difference(&current).cloned().collect::<HashSet<_>>();
+
+        assert_eq!(retired, bucket_keys(&["docs"]));
+        assert_eq!(current, bucket_keys(&["photos"]));
+        assert!(on_demand_migration_backfill_bucket_live_keys(&OdmBackfillRuntimeStats::default()).is_empty());
+    }
+
+    /// Every emitted backfill series must be named by the retirement walk.
+    /// The walk is wider than one cycle's emission on purpose: only the
+    /// job's current `state` gauge is emitted, so retirement enumerates all
+    /// lifecycle values to clear whichever one is live.
+    #[test]
+    fn on_demand_migration_backfill_retirement_covers_every_emitted_series() {
+        let stats = crate::metrics::collectors::on_demand_migration::tests::backfill_golden_stats("node1:9000");
+        let emitted = collect_on_demand_migration_backfill_metrics(&stats)
+            .into_iter()
+            .map(|metric| {
+                (
+                    metric.name.to_string(),
+                    metric
+                        .labels
+                        .into_iter()
+                        .map(|(key, value)| (key, value.to_string()))
+                        .collect::<Vec<_>>(),
+                )
+            })
+            .collect::<HashSet<_>>();
+        let retired = stats
+            .buckets
+            .iter()
+            .flat_map(|bucket| on_demand_migration_backfill_metric_series(&stats.server, &bucket.bucket))
+            .map(|(name, labels)| {
+                (
+                    name,
+                    labels
+                        .into_iter()
+                        .map(|(key, value)| (key, value.to_string()))
+                        .collect::<Vec<_>>(),
+                )
+            })
+            .collect::<HashSet<_>>();
+
+        assert!(emitted.is_subset(&retired), "emitted: {emitted:?}, retired: {retired:?}");
+        assert_eq!(
+            on_demand_migration_backfill_metric_series("node1:9000", "photos").len(),
+            ODM_BACKFILL_STATES.len() + 6
+        );
+        // Without a process-global recorder there is nothing to retire; the
+        // walk itself must still cover every series.
+        assert_eq!(retire_on_demand_migration_backfill_metric_series("node1:9000", "photos"), 0);
+    }
+
     #[test]
     fn metrics_runtime_status_reports_disabled_state() {
         let snapshot = build_metrics_runtime_status_snapshot(false, false, fixed_metrics_runtime_config(), false);

crates/obs/src/metrics/schema/on_demand_migration.rs

@@ -35,6 +35,10 @@ pub const PATH_L: &str = "path";
 pub const REASON_L: &str = "reason";
 /// Upper bound (seconds) of a source latency bucket.
 pub const LE_L: &str = "le";
+/// Node the backfill job runs on.
+pub const SERVER_L: &str = "server";
+/// Lifecycle state of a backfill job.
+pub const STATE_L: &str = "state";
 
 /// Fixed `op` label values.
 pub const REQUEST_OPS: [&str; 2] = ["get", "head"];
@@ -71,6 +75,17 @@ pub const PULL_FAILURE_REASONS: [&str; 12] = [
 pub const SOURCE_LATENCY_LE: [&str; 15] = [
     "0.005", "0.01", "0.02", "0.05", "0.1", "0.2", "0.5", "1", "2", "5", "10", "20", "30", "60", "+Inf",
 ];
+/// Fixed `state` label values of `backfill_jobs`; mirrors the checkpoint's
+/// `BackfillState` variants in ecstore.
+pub const BACKFILL_STATES: [&str; 7] = [
+    "pending",
+    "running",
+    "paused",
+    "cancelled",
+    "completed",
+    "completed_with_failures",
+    "failed",
+];
 
 /// `breaker_state` gauge value: the breaker admits every request.
 pub const BREAKER_STATE_CLOSED: f64 = 0.0;
@@ -183,8 +198,80 @@ pub static ODM_BREAKER_STATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     )
 });
 
-// backfill_* descriptors (`listed_total`, `pulled_total`, `skipped_total`,
-// `failed_total`, `state`) are added here by ODM-12 (rustfs/backlog#2159).
+// backfill_* descriptors (ODM-12, rustfs/backlog#2159). Unlike the request
+// path above, a backfill job runs on exactly one node at a time, so every
+// backfill series carries the owning node in `server` on top of `bucket`.
+
+const BACKFILL_JOBS: &str = "backfill_jobs";
+const BACKFILL_LISTED_TOTAL: &str = "backfill_listed_total";
+const BACKFILL_ENQUEUED_TOTAL: &str = "backfill_enqueued_total";
+const BACKFILL_PULLED_TOTAL: &str = "backfill_pulled_total";
+const BACKFILL_SKIPPED_EXISTING_TOTAL: &str = "backfill_skipped_existing_total";
+const BACKFILL_FAILED_TOTAL: &str = "backfill_failed_total";
+const BACKFILL_BYTES_TOTAL: &str = "backfill_bytes_total";
+
+pub static ODM_BACKFILL_JOBS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(BACKFILL_JOBS),
+        "On-demand migration backfill jobs by server, bucket and state (1 for the bucket's current state)",
+        &[SERVER_L, BUCKET_L, STATE_L],
+        subsystems::ON_DEMAND_MIGRATION,
+    )
+});
+
+pub static ODM_BACKFILL_LISTED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::from(BACKFILL_LISTED_TOTAL),
+        "Source keys listed by the on-demand migration backfill job, by server and bucket",
+        &[SERVER_L, BUCKET_L],
+        subsystems::ON_DEMAND_MIGRATION,
+    )
+});
+
+pub static ODM_BACKFILL_ENQUEUED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::from(BACKFILL_ENQUEUED_TOTAL),
+        "Keys queued for pulling by the on-demand migration backfill job, by server and bucket",
+        &[SERVER_L, BUCKET_L],
+        subsystems::ON_DEMAND_MIGRATION,
+    )
+});
+
+pub static ODM_BACKFILL_PULLED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::from(BACKFILL_PULLED_TOTAL),
+        "Objects stored locally by the on-demand migration backfill job, by server and bucket",
+        &[SERVER_L, BUCKET_L],
+        subsystems::ON_DEMAND_MIGRATION,
+    )
+});
+
+pub static ODM_BACKFILL_SKIPPED_EXISTING_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::from(BACKFILL_SKIPPED_EXISTING_TOTAL),
+        "Keys the on-demand migration backfill job skipped because a local object already existed, by server and bucket",
+        &[SERVER_L, BUCKET_L],
+        subsystems::ON_DEMAND_MIGRATION,
+    )
+});
+
+pub static ODM_BACKFILL_FAILED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::from(BACKFILL_FAILED_TOTAL),
+        "Keys the on-demand migration backfill job could not pull, by server and bucket",
+        &[SERVER_L, BUCKET_L],
+        subsystems::ON_DEMAND_MIGRATION,
+    )
+});
+
+pub static ODM_BACKFILL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::from(BACKFILL_BYTES_TOTAL),
+        "Bytes stored locally by the on-demand migration backfill job, by server and bucket",
+        &[SERVER_L, BUCKET_L],
+        subsystems::ON_DEMAND_MIGRATION,
+    )
+});
 
 #[cfg(test)]
 mod tests {
@@ -208,6 +295,13 @@ mod tests {
             (&*ODM_SOURCE_LATENCY_SECONDS_SUM_MD, "source_latency_seconds_sum"),
             (&*ODM_SOURCE_LATENCY_SECONDS_COUNT_MD, "source_latency_seconds_count"),
             (&*ODM_BREAKER_STATE_MD, "breaker_state"),
+            (&*ODM_BACKFILL_JOBS_MD, "backfill_jobs"),
+            (&*ODM_BACKFILL_LISTED_MD, "backfill_listed_total"),
+            (&*ODM_BACKFILL_ENQUEUED_MD, "backfill_enqueued_total"),
+            (&*ODM_BACKFILL_PULLED_MD, "backfill_pulled_total"),
+            (&*ODM_BACKFILL_SKIPPED_EXISTING_MD, "backfill_skipped_existing_total"),
+            (&*ODM_BACKFILL_FAILED_MD, "backfill_failed_total"),
+            (&*ODM_BACKFILL_BYTES_MD, "backfill_bytes_total"),
         ] {
             assert_eq!(descriptor.get_full_metric_name(), format!("rustfs_on_demand_migration_{suffix}"));
             assert_eq!(descriptor.subsystem, subsystems::ON_DEMAND_MIGRATION);
@@ -259,6 +353,23 @@ mod tests {
         assert_eq!(SOURCE_LATENCY_LE[14], "+Inf");
     }
 
+    #[test]
+    fn backfill_series_are_server_and_bucket_scoped() {
+        assert_eq!(ODM_BACKFILL_JOBS_MD.metric_type, MetricType::Gauge);
+        assert_eq!(labels(&ODM_BACKFILL_JOBS_MD), vec!["server", "bucket", "state"]);
+        for descriptor in [
+            &*ODM_BACKFILL_LISTED_MD,
+            &*ODM_BACKFILL_ENQUEUED_MD,
+            &*ODM_BACKFILL_PULLED_MD,
+            &*ODM_BACKFILL_SKIPPED_EXISTING_MD,
+            &*ODM_BACKFILL_FAILED_MD,
+            &*ODM_BACKFILL_BYTES_MD,
+        ] {
+            assert_eq!(descriptor.metric_type, MetricType::Counter);
+            assert_eq!(labels(descriptor), vec!["server", "bucket"]);
+        }
+    }
+
     #[test]
     fn fixed_label_values_are_unique() {
         for values in [
@@ -267,6 +378,7 @@ mod tests {
             PULL_PATHS.as_slice(),
             PULL_FAILURE_REASONS.as_slice(),
             SOURCE_LATENCY_LE.as_slice(),
+            BACKFILL_STATES.as_slice(),
         ] {
             let unique: std::collections::BTreeSet<_> = values.iter().collect();
             assert_eq!(unique.len(), values.len(), "{values:?}");

crates/obs/src/metrics/stats_collector.rs

@@ -26,15 +26,16 @@ use crate::metrics::collectors::{
     ClusterHealthStats, ClusterStats, ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats,
     DriveDetailedStats, DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats,
     IlmBackpressureStats, IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats,
-    OnDemandMigrationBucketStats, ProcessStats, ProcessStatusType, ReplicationMetricsSnapshot, ResourceStats,
-    ScannerRuntimeStats, ScannerStats,
+    OdmBackfillRuntimeStats, OnDemandMigrationBucketStats, ProcessStats, ProcessStatusType, ReplicationMetricsSnapshot,
+    ResourceStats, ScannerRuntimeStats, ScannerStats,
 };
 use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
 use crate::metrics::{
     BucketOperations, BucketOptions, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore, StorageAdminApi,
     obs_bucket_replication_stats_snapshot, obs_get_quota_config, obs_get_total_usable_capacity,
     obs_get_total_usable_capacity_free, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend,
-    obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
+    obs_on_demand_migration_backfill_snapshot, obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot,
+    obs_resolve_object_store_handle,
 };
 use crate::node_identity::current_local_node_identity;
 use jiff::Timestamp;
@@ -667,6 +668,11 @@ pub fn collect_on_demand_migration_stats() -> Vec<OnDemandMigrationBucketStats>
     obs_on_demand_migration_snapshot()
 }
 
+/// Collect this node's on-demand migration backfill job progress.
+pub fn collect_on_demand_migration_backfill_stats() -> OdmBackfillRuntimeStats {
+    obs_on_demand_migration_backfill_snapshot(current_local_node_identity())
+}
+
 /// Collect site-level replication stats from the global replication runtime.
 pub async fn collect_replication_stats() -> ReplicationMetricsSnapshot {
     obs_site_replication_stats().await

crates/obs/src/metrics/storage_api.rs

@@ -17,6 +17,9 @@ use std::time::Duration;
 
 pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
 pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
+use rustfs_ecstore::api::bucket::on_demand_migration::backfill::{
+    BackfillCheckpoint as SourceBackfillCheckpoint, global_backfill_runner as source_global_backfill_runner,
+};
 use rustfs_ecstore::api::bucket::on_demand_migration::{
     BreakerState as SourceOdmBreakerState, OdmBucketSnapshot as SourceOdmBucketSnapshot,
     OnDemandMigrationSys as SourceOnDemandMigrationSys,
@@ -41,7 +44,9 @@ pub(crate) use rustfs_ecstore::api::runtime::{
 pub(crate) use rustfs_ecstore::api::storage::ECStore as ObsStore;
 use rustfs_storage_api as storage_contracts;
 
-use crate::metrics::collectors::{OnDemandMigrationBreakerState, OnDemandMigrationBucketStats};
+use crate::metrics::collectors::{
+    OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBreakerState, OnDemandMigrationBucketStats,
+};
 
 #[derive(Debug, Clone, PartialEq)]
 pub(crate) struct ObsBucketReplicationTargetStatsSnapshot {
@@ -496,6 +501,38 @@ pub(crate) fn obs_on_demand_migration_snapshot() -> Vec<OnDemandMigrationBucketS
         .collect()
 }
 
+fn on_demand_migration_backfill_stats_from_checkpoint(
+    bucket: String,
+    checkpoint: SourceBackfillCheckpoint,
+) -> OdmBackfillBucketStats {
+    OdmBackfillBucketStats {
+        bucket,
+        state: checkpoint.state.as_str().to_string(),
+        listed: checkpoint.listed,
+        enqueued: checkpoint.enqueued,
+        pulled: checkpoint.pulled,
+        skipped_existing: checkpoint.skipped_existing,
+        failed: checkpoint.failed,
+        bytes: checkpoint.bytes,
+    }
+}
+
+/// Backfill jobs running on this node, sorted by bucket. Empty until the
+/// runner is installed, and empty again once a job finishes: the series are
+/// per-node job progress, not a cluster-wide history.
+pub(crate) fn obs_on_demand_migration_backfill_snapshot(server: String) -> OdmBackfillRuntimeStats {
+    let buckets = source_global_backfill_runner()
+        .map(|runner| {
+            runner
+                .local_job_snapshots()
+                .into_iter()
+                .map(|(bucket, checkpoint)| on_demand_migration_backfill_stats_from_checkpoint(bucket, checkpoint))
+                .collect()
+        })
+        .unwrap_or_default();
+    OdmBackfillRuntimeStats { server, buckets }
+}
+
 pub(crate) async fn obs_replication_site_stats_snapshot(current_data_transfer_rate: f64) -> ObsReplicationSiteStatsSnapshot {
     let Some(stats) = get_global_replication_stats() else {
         return ObsReplicationSiteStatsSnapshot::default();
@@ -808,7 +845,8 @@ pub(crate) mod metrics {
         ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
         obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config,
         obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
-        obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_on_demand_migration_snapshot,
-        obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, obs_transition_state_handle,
+        obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_on_demand_migration_backfill_snapshot,
+        obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
+        obs_transition_state_handle,
     };
 }