feat(ecstore): add the on-demand migration backfill job (#7087)

* feat(ecstore): add on-demand migration backfill job core

Add the background backfill job for on-demand migration
(rustfs/backlog#2159): a durable checkpoint under
buckets/<bucket>/on-demand-migration-backfill.json saved by If-Match
compare-and-set every 1000 keys or 10 s, a 60 s owner lease renewed by
every save, a recovery pass that takes over expired leases (or jobs this
node owned before a restart) and cancels jobs whose config changed, and a
main loop over the source ListObjectsV2 pages with the skip_existing
policy, dry runs, bounded outstanding pulls and wait-on-full enqueueing.

The pull queue gains per-job completion reports so the job can count
pulled/failed keys (hashes only), and pull permits become two-tier so an
online miss is never queued behind a backfill pull.

* feat(admin): expose on-demand migration backfill job

Wire the ODM-12 backfill job (rustfs/backlog#2159) to its operators:
POST /v3/on-demand-migration/{bucket}/backfill?op=start|cancel and
GET .../backfill return the checkpoint document, GET .../status gains a
backfill summary, and the recovery loop plus the process-wide runner are
installed at startup. Backfill control reuses
Set/GetBucketOnDemandMigrationAction and is recorded in the route policy,
the registration matrix and the admin route snapshot.

Add the rustfs-madmin wire types and client methods with golden fixtures
shared by the server tests, the backfill_* metric descriptors and their
collector, and three e2e scenarios: a full backfill across list pages,
cancellation, and resuming from the persisted continuation token after a
server restart.

crates/ecstore/src/api/mod.rs

@@ -161,9 +161,20 @@ pub mod bucket {
         };
         pub use crate::bucket::on_demand_migration::{
             EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS,
-            PullCompletion, PullQueue, PullReason, PullSource, SourceBody, WriteBackBody, WriteBackError, WriteBackOutcome,
-            WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with,
+            PullCompletion, PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, WriteBackBody, WriteBackError,
+            WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with,
         };
+        pub mod backfill {
+            pub use crate::bucket::on_demand_migration::backfill::{
+                BACKFILL_CHECKPOINT_FILE, BACKFILL_CHECKPOINT_FORMAT_VERSION, BACKFILL_FAILED_KEYS_CAPACITY, BACKFILL_LEASE,
+                BACKFILL_LEASE_LOCK_PREFIX, BACKFILL_LIST_PAGE_SIZE, BACKFILL_RECOVERY_INTERVAL, BACKFILL_SAVE_EVERY_KEYS,
+                BACKFILL_SAVE_INTERVAL, BackfillCheckpoint, BackfillContext, BackfillContextFactory, BackfillError,
+                BackfillLastError, BackfillOwner, BackfillRecoveryStats, BackfillRequest, BackfillRunner, BackfillState,
+                BucketBackfillContext, LocalBackfillObject, PriorityPullPermits, PullPermit, PullPriority, SkipExisting,
+                StoredCheckpoint, SysBackfillContexts, global_backfill_runner, install_global_backfill_runner, key_hash,
+                read_checkpoint, run_backfill_recovery_loop, spawn_backfill_recovery_loop,
+            };
+        }
         pub mod source_client {
             pub use crate::bucket::on_demand_migration::source_client::{
                 SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceObject, SourcePage, SourceProbe,

crates/ecstore/src/bucket/on_demand_migration/mod.rs

@@ -20,6 +20,7 @@
 //! clients guarded by a breaker, a negative cache, singleflight and a pull
 //! concurrency limit (rustfs/backlog#2147).
 
+pub mod backfill;
 pub mod breaker;
 pub mod config;
 pub mod negative_cache;
@@ -40,8 +41,8 @@ pub use config::{
 pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
 pub use pull::{
     EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS, PullCompletion,
-    PullQueue, PullReason, PullSource, SourceBody, WriteBackBody, WriteBackError, WriteBackOutcome, WriteBackPart,
-    WriteBackRequest, commit_inline, commit_inline_with,
+    PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, WriteBackBody, WriteBackError, WriteBackOutcome,
+    WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with,
 };
 pub use stats::{
     GaugeGuard, LastSourceError, LatencyBucketSnapshot, OdmOp, OdmOutcome, OdmStats, OdmStatsSnapshot, PullFailureReason,

crates/ecstore/src/bucket/on_demand_migration/pull.rs

@@ -40,6 +40,7 @@
 //! oversized body therefore fails the local write before it can commit,
 //! independently of the digest check the write-back performs.
 
+use super::backfill::PullPriority;
 use super::source_client::{SourceClient, SourceError, SourceHead};
 use super::stats::{PullFailureReason, PullPath};
 use super::sys::{BucketOdmState, OnDemandMigrationSys, PullError, PullOutcome, PullSlot};
@@ -57,7 +58,7 @@ use std::task::{Context, Poll};
 use std::time::{Duration, Instant};
 use time::OffsetDateTime;
 use tokio::sync::mpsc::{self, error::TrySendError};
-use tokio::sync::watch;
+use tokio::sync::{oneshot, watch};
 use tokio::task::JoinSet;
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, trace};
@@ -108,6 +109,27 @@ impl PullReason {
             PullReason::Backfill => PullPath::Backfill,
         }
     }
+
+    /// Backfill pulls yield pull permits to online requests.
+    pub fn priority(self) -> PullPriority {
+        match self {
+            PullReason::RangeGet | PullReason::LargeObject => PullPriority::Online,
+            PullReason::Backfill => PullPriority::Backfill,
+        }
+    }
+}
+
+/// How one queued pull ended, delivered to the requester that asked for a
+/// report (the backfill job counts these).
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum QueuedPullOutcome {
+    /// A new local object of `size` bytes was written.
+    Stored {
+        size: u64,
+    },
+    /// A current local version already existed; nothing was pulled.
+    AlreadyPresent,
+    Failed(PullError),
 }
 
 /// Result of [`PullQueue::enqueue`].
@@ -731,6 +753,8 @@ pub async fn commit_inline_with(
 struct PullJob {
     key: String,
     reason: PullReason,
+    /// Dropped without a send when the job is cancelled before it runs.
+    report: Option<oneshot::Sender<QueuedPullOutcome>>,
 }
 
 /// Bounded per-bucket queue of background pulls. Keys are unique while
@@ -809,26 +833,39 @@ impl PullQueue {
     }
 
     pub fn enqueue(&self, key: &str, reason: PullReason) -> EnqueueOutcome {
+        self.enqueue_with_report(key, reason).0
+    }
+
+    /// [`Self::enqueue`] that also hands back the job's report channel when
+    /// a new job was queued (`Coalesced` pulls report to their first
+    /// requester only).
+    pub fn enqueue_with_report(
+        &self,
+        key: &str,
+        reason: PullReason,
+    ) -> (EnqueueOutcome, Option<oneshot::Receiver<QueuedPullOutcome>>) {
         if self.cancel.is_cancelled() {
-            return EnqueueOutcome::Unavailable;
+            return (EnqueueOutcome::Unavailable, None);
         }
         let mut pending = self.pending.lock();
         if pending.contains(key) {
-            return EnqueueOutcome::Coalesced;
+            return (EnqueueOutcome::Coalesced, None);
         }
+        let (report_tx, report_rx) = oneshot::channel();
         match self.tx.try_send(PullJob {
             key: key.to_string(),
             reason,
+            report: Some(report_tx),
         }) {
             Ok(()) => {
                 pending.insert(key.to_string());
-                EnqueueOutcome::Enqueued
+                (EnqueueOutcome::Enqueued, Some(report_rx))
             }
             Err(TrySendError::Full(_)) => {
                 self.stats.record_pull_failure(PullFailureReason::QueueFull);
-                EnqueueOutcome::QueueFull
+                (EnqueueOutcome::QueueFull, None)
             }
-            Err(TrySendError::Closed(_)) => EnqueueOutcome::Unavailable,
+            Err(TrySendError::Closed(_)) => (EnqueueOutcome::Unavailable, None),
         }
     }
 }
@@ -860,7 +897,7 @@ async fn dispatch(
             queue: Arc::clone(&queue),
             key: job.key.clone(),
         };
-        let slot = match state.acquire_pull_slot(&job.key).await {
+        let slot = match state.acquire_pull_slot_with_priority(&job.key, job.reason.priority()).await {
             Ok(slot) => slot,
             Err(err) => {
                 let result: Result<PullCompletion, PullError> = Err(err);
@@ -905,11 +942,14 @@ async fn run_job(
 ) {
     let _pending = pending;
     let cancel = state.cancel_token();
-    match slot {
+    let report = match slot {
         PullSlot::Follower(follower) => {
             // Someone else (inline GET or an earlier job) is pulling the key;
             // its result makes this job redundant.
-            let _ = follower.wait().await;
+            match follower.wait().await {
+                Ok(outcome) => QueuedPullOutcome::Stored { size: outcome.size },
+                Err(err) => QueuedPullOutcome::Failed(err),
+            }
         }
         PullSlot::Leader(leader) => {
             let ctx = PullContext {
@@ -921,8 +961,17 @@ async fn run_job(
             };
             let result = pull_object(&ctx).await;
             record_completion(&state, &job.key, job.reason.path(), &result);
+            let report = match &result {
+                Ok(PullCompletion::Stored(outcome)) => QueuedPullOutcome::Stored { size: outcome.size },
+                Ok(PullCompletion::AlreadyPresent(_)) => QueuedPullOutcome::AlreadyPresent,
+                Err(err) => QueuedPullOutcome::Failed(err.clone()),
+            };
             leader.complete(result.map(|completion| completion.outcome()));
+            report
         }
+    };
+    if let Some(tx) = job.report {
+        let _ = tx.send(report);
     }
 }
 
@@ -948,9 +997,18 @@ impl BucketOdmState {
 
     /// Queues a background pull of `key`; see [`EnqueueOutcome`].
     pub fn enqueue_pull(self: &Arc<Self>, key: &str, reason: PullReason) -> EnqueueOutcome {
+        self.enqueue_pull_with_report(key, reason).0
+    }
+
+    /// [`Self::enqueue_pull`] with the job's report channel.
+    pub fn enqueue_pull_with_report(
+        self: &Arc<Self>,
+        key: &str,
+        reason: PullReason,
+    ) -> (EnqueueOutcome, Option<oneshot::Receiver<QueuedPullOutcome>>) {
         match self.pull_queue() {
-            Some(queue) => queue.enqueue(key, reason),
-            None => EnqueueOutcome::Unavailable,
+            Some(queue) => queue.enqueue_with_report(key, reason),
+            None => (EnqueueOutcome::Unavailable, None),
         }
     }
 }

crates/ecstore/src/bucket/on_demand_migration/sys.rs

@@ -39,6 +39,7 @@
 //! (`pull.rs`) stores objects with; each bucket state captures it at build
 //! time together with its lazily started [`PullQueue`].
 
+use super::backfill::{PriorityPullPermits, PullPermit, PullPriority};
 use super::breaker::{Breaker, BreakerState, BreakerTransition, BreakerVerdict};
 use super::config::{
     ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig, PathStyle as ConfigPathStyle, Provider, SourceConfig,
@@ -57,7 +58,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, OnceLock};
 use std::time::Duration;
 use time::OffsetDateTime;
-use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch};
+use tokio::sync::watch;
 use tokio::time::Instant;
 use tokio_util::sync::CancellationToken;
 use tracing::{info, warn};
@@ -182,7 +183,7 @@ pub struct PullLeader {
     state: Arc<BucketOdmState>,
     key: String,
     tx: watch::Sender<Option<PullResult>>,
-    _permit: OwnedSemaphorePermit,
+    _permit: PullPermit,
     _inflight: GaugeGuard,
     completed: bool,
 }
@@ -269,7 +270,8 @@ pub struct BucketOdmState {
     breaker: Breaker,
     negative_cache: NegativeCache,
     inflight: Mutex<HashMap<String, watch::Receiver<Option<PullResult>>>>,
-    pull_semaphore: Arc<Semaphore>,
+    /// Online misses first, backfill pulls when nobody waits (ODM-12).
+    pull_permits: Arc<PriorityPullPermits>,
     stats: Arc<OdmStats>,
     cancel: CancellationToken,
     last_source_error_logged_at: Mutex<Option<Instant>>,
@@ -318,7 +320,7 @@ impl BucketOdmState {
             breaker: Breaker::new(),
             negative_cache: NegativeCache::new(Duration::from_secs(policy.negative_cache_ttl_secs)),
             inflight: Mutex::new(HashMap::new()),
-            pull_semaphore: Arc::new(Semaphore::new(policy.max_concurrent_pulls.max(1) as usize)),
+            pull_permits: PriorityPullPermits::new(policy.max_concurrent_pulls.max(1) as usize),
             stats,
             cancel: CancellationToken::new(),
             last_source_error_logged_at: Mutex::new(None),
@@ -477,6 +479,16 @@ impl BucketOdmState {
     /// later callers for the same key become followers and never touch the
     /// semaphore. Fails with `Canceled` when the state is torn down.
     pub async fn acquire_pull_slot(self: &Arc<Self>, key: &str) -> Result<PullSlot, PullError> {
+        self.acquire_pull_slot_with_priority(key, PullPriority::Online).await
+    }
+
+    /// [`Self::acquire_pull_slot`] at the given permit priority; the pull
+    /// queue passes `Backfill` for backfill jobs.
+    pub async fn acquire_pull_slot_with_priority(
+        self: &Arc<Self>,
+        key: &str,
+        priority: PullPriority,
+    ) -> Result<PullSlot, PullError> {
         if self.cancel.is_cancelled() {
             return Err(PullError::canceled("bucket on-demand migration state was removed"));
         }
@@ -499,7 +511,7 @@ impl BucketOdmState {
         let permit = {
             let _queued = self.stats.queue_guard();
             tokio::select! {
-                permit = Arc::clone(&self.pull_semaphore).acquire_owned() => permit,
+                permit = self.pull_permits.acquire(priority) => permit,
                 _ = self.cancel.cancelled() => {
                     return Err(PullError::canceled("bucket on-demand migration state was removed"));
                 }
@@ -524,6 +536,10 @@ impl BucketOdmState {
         self.inflight.lock().len()
     }
 
+    pub fn pull_permits(&self) -> &Arc<PriorityPullPermits> {
+        &self.pull_permits
+    }
+
     pub fn snapshot(&self) -> OdmBucketSnapshot {
         OdmBucketSnapshot {
             bucket: self.bucket.clone(),