diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index d8895815d..87a0a166b 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=09bc29b3e0ed7c5779331740e18e1d5ca05cb240c1363d1ecc95c1ef1e337038 +sha256-darwin=efd7357af1e998134df7d01e0a10d7186898e47abbe86ae6ecb6aef1f8c58357 sha256-linux=86e69337ad1440252a2ee20a12063c989ed12442d3b1ddf9e9233acf0f2ec089 diff --git a/crates/e2e_test/src/on_demand_migration/backfill_test.rs b/crates/e2e_test/src/on_demand_migration/backfill_test.rs new file mode 100644 index 000000000..18d84b56b --- /dev/null +++ b/crates/e2e_test/src/on_demand_migration/backfill_test.rs @@ -0,0 +1,254 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! On-demand migration backfill job scenarios (ODM-12, rustfs/backlog#2159): +//! a full backfill of a small-object source, cancellation, and resuming from +//! the persisted continuation token after a server restart. + +use super::common::{BackfillOp, BackfillRequest, ODM_SERVER_ENV, OdmSourceSpec, OdmTestEnv, SeedObject}; +use crate::fake_s3_target::Operation; +use bytes::Bytes; +use std::time::Duration; + +type TestResult = Result<(), Box>; + +const SOURCE_BUCKET: &str = "odm-backfill-source"; +const LOCAL_BUCKET: &str = "odm-backfill-local"; +const KEY_PREFIX: &str = "cold/"; +/// Keys per scenario. The fake source retains at most 4,096 object versions +/// and 4,096 journal entries, and one pull is a HEAD plus a GET, so a +/// scenario that asserts on the journal stays below ~2,000 keys. The job +/// lists 1,000 keys per page, so this still spans several pages and exercises +/// the continuation token, which is what the scenarios are about. +const SEEDED_KEYS: usize = 1500; + +fn key(i: usize) -> String { + format!("{KEY_PREFIX}{i:05}") +} + +/// Content that identifies the key so a mis-stored object is caught. +fn body(i: usize) -> Bytes { + Bytes::from(format!("object-{i:05}-payload")) +} + +fn seed(env: &OdmTestEnv, count: usize) { + let objects: Vec = (0..count).map(|i| SeedObject::new(key(i), body(i))).collect(); + let etags = env.seed_source(SOURCE_BUCKET, &objects); + assert_eq!(etags.len(), count); +} + +async fn configure(env: &OdmTestEnv, spec: &OdmSourceSpec) -> TestResult { + let response = env.configure_source(LOCAL_BUCKET, spec).await?; + assert_eq!(response.status, 200, "configure: {}", response.body); + // The probe issued a one-key listing; count only the job's traffic. + env.source.take_requests(); + Ok(()) +} + +fn source_lists(env: &OdmTestEnv) -> Vec> { + env.source + .requests() + .into_iter() + .filter(|record| record.operation == Operation::ListObjectsV2) + .map(|record| record.continuation_token) + .collect() +} + +fn source_gets(env: &OdmTestEnv) -> usize { + env.source + .requests() + .into_iter() + .filter(|record| record.operation == Operation::GetObject) + .count() +} + +fn counter(job: &serde_json::Value, name: &str) -> u64 { + job[name].as_u64().unwrap_or_else(|| panic!("{name} missing in {job}")) +} + +#[tokio::test] +async fn backfill_pulls_every_source_object_across_list_pages() -> TestResult { + const COUNT: usize = SEEDED_KEYS; + let env = OdmTestEnv::start().await?; + env.source.create_bucket(SOURCE_BUCKET); + env.rustfs.create_test_bucket(LOCAL_BUCKET).await?; + seed(&env, COUNT); + configure(&env, &env.fake_source_spec(SOURCE_BUCKET)).await?; + + let started = env.start_backfill(LOCAL_BUCKET, BackfillRequest::default()).await?; + assert_eq!(started.status, 200, "start: {}", started.body); + let job = started.json()?["job"].clone(); + assert_eq!(job["state"], "running"); + assert_eq!(job["skip_existing"], "always"); + assert_eq!(job["dry_run"], false); + let job_id = job["job_id"].as_str().expect("job id").to_string(); + + // A second start while the job holds its lease is a conflict. + let again = env + .backfill(LOCAL_BUCKET, BackfillOp::Start(BackfillRequest::default())) + .await?; + assert_eq!(again.status, 409, "second start: {}", again.body); + assert!(again.body.contains("OnDemandMigrationBackfillRunning"), "{}", again.body); + + let done = env + .wait_for_backfill(LOCAL_BUCKET, Duration::from_secs(240), |job| job["state"] == "completed") + .await?; + assert_eq!(done["job_id"], job_id.as_str()); + assert_eq!(counter(&done, "listed"), COUNT as u64); + assert_eq!(counter(&done, "enqueued"), COUNT as u64); + assert_eq!(counter(&done, "pulled"), COUNT as u64); + assert_eq!(counter(&done, "failed"), 0); + assert_eq!(counter(&done, "skipped_existing"), 0); + assert!(done["continuation_token"].is_null(), "a finished job carries no cursor"); + assert_eq!(done["last_key"], key(COUNT - 1)); + assert!(done["failed_keys"].as_array().is_some_and(Vec::is_empty)); + let expected_bytes: u64 = (0..COUNT).map(|i| body(i).len() as u64).sum(); + assert_eq!(counter(&done, "bytes"), expected_bytes); + + assert_eq!(env.local_key_count(LOCAL_BUCKET, KEY_PREFIX).await?, COUNT); + for i in [0, 999, 1000, 1200, COUNT - 1] { + env.assert_local_present(LOCAL_BUCKET, &key(i), &body(i)).await; + } + + let lists = source_lists(&env); + assert_eq!(lists.len(), COUNT.div_ceil(1000), "{COUNT} keys at 1000 per page: {lists:?}"); + assert!(lists[0].is_none(), "the first page starts without a cursor"); + assert!(lists[1..].iter().all(Option::is_some), "every later page carries the cursor"); + assert_eq!(source_gets(&env), COUNT, "every object is fetched exactly once"); + + // The status endpoint summarises the same job. + let status = env.status(LOCAL_BUCKET).await?; + assert_eq!(status.status, 200); + let summary = status.json()?["backfill"].clone(); + assert_eq!(summary["job_id"], job_id.as_str()); + assert_eq!(summary["state"], "completed"); + assert_eq!(counter(&summary, "pulled"), COUNT as u64); + Ok(()) +} + +#[tokio::test] +async fn backfill_cancel_stops_enqueueing_and_persists_cancelled() -> TestResult { + const COUNT: usize = SEEDED_KEYS; + let env = OdmTestEnv::start().await?; + env.source.create_bucket(SOURCE_BUCKET); + env.rustfs.create_test_bucket(LOCAL_BUCKET).await?; + seed(&env, COUNT); + let mut spec = env.fake_source_spec(SOURCE_BUCKET); + spec.policy.max_concurrent_pulls = 1; + configure(&env, &spec).await?; + + // Cancelling before any job exists is a 404, not a silent success. + let nothing = env.backfill(LOCAL_BUCKET, BackfillOp::Cancel).await?; + assert_eq!(nothing.status, 404, "cancel without a job: {}", nothing.body); + assert!(nothing.body.contains("NoSuchBackfillJob"), "{}", nothing.body); + let unread = env.backfill(LOCAL_BUCKET, BackfillOp::Status).await?; + assert_eq!(unread.status, 404, "status without a job: {}", unread.body); + + let started = env.start_backfill(LOCAL_BUCKET, BackfillRequest::default()).await?; + assert_eq!(started.status, 200, "start: {}", started.body); + env.wait_for_backfill(LOCAL_BUCKET, Duration::from_secs(60), |job| { + job["state"] == "running" && counter(job, "enqueued") > 0 + }) + .await?; + + let cancelled = env.backfill(LOCAL_BUCKET, BackfillOp::Cancel).await?; + assert_eq!(cancelled.status, 200, "cancel: {}", cancelled.body); + let job = cancelled.json()?["job"].clone(); + assert_eq!(job["state"], "cancelled"); + let enqueued_at_cancel = counter(&job, "enqueued"); + assert!(enqueued_at_cancel < COUNT as u64, "the job was cancelled mid-way: {job}"); + + // Nothing is queued after the cancel: the checkpoint and the source + // traffic both stop moving once the few in-flight pulls drain. + tokio::time::sleep(Duration::from_secs(2)).await; + let persisted = env.backfill_job(LOCAL_BUCKET).await?.expect("checkpoint kept for inspection"); + assert_eq!(persisted["state"], "cancelled"); + assert_eq!(counter(&persisted, "enqueued"), enqueued_at_cancel); + let gets_after_drain = source_gets(&env); + tokio::time::sleep(Duration::from_secs(1)).await; + assert_eq!(source_gets(&env), gets_after_drain, "no source GET after the cancel drained"); + assert!(env.local_key_count(LOCAL_BUCKET, KEY_PREFIX).await? < COUNT); + + // Cancel is idempotent and the status endpoint reports the final state. + let again = env.backfill(LOCAL_BUCKET, BackfillOp::Cancel).await?; + assert_eq!(again.status, 200, "second cancel: {}", again.body); + assert_eq!(again.json()?["job"]["state"], "cancelled"); + let status = env.status(LOCAL_BUCKET).await?; + assert_eq!(status.json()?["backfill"]["state"], "cancelled"); + + // A cancelled job releases the bucket: a new job can start. + let restarted = env.start_backfill(LOCAL_BUCKET, BackfillRequest::default()).await?; + assert_eq!(restarted.status, 200, "restart after cancel: {}", restarted.body); + assert_ne!(restarted.json()?["job"]["job_id"], job["job_id"]); + let _ = env.backfill(LOCAL_BUCKET, BackfillOp::Cancel).await?; + Ok(()) +} + +#[tokio::test] +async fn backfill_resumes_from_continuation_token_after_restart() -> TestResult { + const COUNT: usize = SEEDED_KEYS; + let mut env = OdmTestEnv::start().await?; + env.source.create_bucket(SOURCE_BUCKET); + env.rustfs.create_test_bucket(LOCAL_BUCKET).await?; + seed(&env, COUNT); + let mut spec = env.fake_source_spec(SOURCE_BUCKET); + spec.policy.max_concurrent_pulls = 2; + configure(&env, &spec).await?; + + let started = env.start_backfill(LOCAL_BUCKET, BackfillRequest::default()).await?; + assert_eq!(started.status, 200, "start: {}", started.body); + let job_id = started.json()?["job"]["job_id"].as_str().expect("job id").to_string(); + + // Wait for the first page to be committed (cursor persisted), then kill + // the server while the job is still running. + let mid = env + .wait_for_backfill(LOCAL_BUCKET, Duration::from_secs(120), |job| { + job["state"] == "running" && job["continuation_token"].is_string() + }) + .await?; + assert!(counter(&mid, "listed") >= 1000 && counter(&mid, "listed") < COUNT as u64, "{mid}"); + env.source.take_requests(); + env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?; + + let done = env + .wait_for_backfill(LOCAL_BUCKET, Duration::from_secs(300), |job| job["state"] == "completed") + .await?; + assert_eq!(done["job_id"], job_id.as_str(), "the same job continues after the restart"); + assert_eq!(counter(&done, "failed"), 0); + assert!( + counter(&done, "listed") >= COUNT as u64, + "the resumed job listed the rest (the interrupted page is listed twice): {done}" + ); + // Keys pulled before the crash are re-listed and skipped, never re-pulled; + // a pull whose report died with the old process is counted neither way, + // so only the lower bound and the queue accounting are exact. + assert!(counter(&done, "pulled") + counter(&done, "skipped_existing") >= COUNT as u64, "{done}"); + assert!(counter(&done, "pulled") <= counter(&done, "enqueued"), "{done}"); + assert_eq!(env.local_key_count(LOCAL_BUCKET, KEY_PREFIX).await?, COUNT); + for i in [0, 500, 999, 1000, COUNT - 1] { + env.assert_local_present(LOCAL_BUCKET, &key(i), &body(i)).await; + } + + let lists = source_lists(&env); + assert!(!lists.is_empty(), "the resumed job listed the source"); + assert!( + lists.iter().all(Option::is_some), + "after the restart every source listing carries a continuation-token: {lists:?}" + ); + assert!( + lists.len() <= COUNT.div_ceil(1000), + "the listing did not start over from the first page: {lists:?}" + ); + Ok(()) +} diff --git a/crates/e2e_test/src/on_demand_migration/common.rs b/crates/e2e_test/src/on_demand_migration/common.rs index 8886e64a5..d1f542f10 100644 --- a/crates/e2e_test/src/on_demand_migration/common.rs +++ b/crates/e2e_test/src/on_demand_migration/common.rs @@ -40,8 +40,12 @@ pub type BoxError = Box; /// turns it on so scenario tests exercise the feature without repeating it. pub const ODM_MODULE_SWITCH_ENV: &str = "RUSTFS_ON_DEMAND_MIGRATION_ENABLED"; /// The source client shares the replication egress guard, which rejects the -/// fake source's loopback endpoint unless this switch is set. +/// fake source's loopback endpoint unless this switch is set. This is the +/// documented harness opt-in; see +/// `docs/operations/outbound-connection-policy.md`. pub const ALLOW_LOOPBACK_SOURCE_ENV: &str = "RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET"; +/// Server environment every ODM scenario starts (and restarts) with. +pub const ODM_SERVER_ENV: &[(&str, &str)] = &[(ODM_MODULE_SWITCH_ENV, "true"), (ALLOW_LOOPBACK_SOURCE_ENV, "true")]; /// Admin route prefix; the bucket name is appended as one path segment. pub const ODM_ADMIN_ROUTE: &str = "/rustfs/admin/v3/on-demand-migration"; /// Region the fake source is addressed with (it accepts any SigV4 region). @@ -324,7 +328,7 @@ impl OdmTestEnv { pub async fn start_with(options: OdmEnvOptions<'_>) -> Result { let source = FakeS3Target::start_with_options(options.source).await?; let mut rustfs = RustFSTestEnvironment::new().await?; - let mut env: Vec<(&str, &str)> = vec![(ODM_MODULE_SWITCH_ENV, "true"), (ALLOW_LOOPBACK_SOURCE_ENV, "true")]; + let mut env: Vec<(&str, &str)> = ODM_SERVER_ENV.to_vec(); for (name, value) in &options.env { match env.iter_mut().find(|(existing, _)| existing == name) { Some(entry) => entry.1 = value, @@ -412,6 +416,81 @@ impl OdmTestEnv { } } + /// `POST .../{bucket}/backfill?op=start`, retried while the server + /// answers `OnDemandMigrationDisabled`: the bucket state is built + /// asynchronously right after the config `PUT`, so an immediate start can + /// race it. Any other answer is returned as is. + pub async fn start_backfill(&self, bucket: &str, request: BackfillRequest) -> Result { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let response = self.backfill(bucket, BackfillOp::Start(request.clone())).await?; + let state_not_ready = response.status == 400 && response.body.contains("OnDemandMigrationDisabled"); + if !state_not_ready || Instant::now() >= deadline { + return Ok(response); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + + /// The `job` document of `GET .../{bucket}/backfill`, `None` on 404. + pub async fn backfill_job(&self, bucket: &str) -> Result, BoxError> { + let response = self.backfill(bucket, BackfillOp::Status).await?; + match response.status { + 200 => Ok(Some(response.json()?["job"].clone())), + 404 => Ok(None), + status => Err(format!("GET backfill answered {status}: {}", response.body).into()), + } + } + + /// Polls the backfill checkpoint until `accept` returns true or `timeout` + /// elapses (an error naming the last observed document). + pub async fn wait_for_backfill( + &self, + bucket: &str, + timeout: Duration, + accept: impl Fn(&serde_json::Value) -> bool, + ) -> Result { + let deadline = Instant::now() + timeout; + let mut last = serde_json::Value::Null; + loop { + if let Some(job) = self.backfill_job(bucket).await? { + if accept(&job) { + return Ok(job); + } + last = job; + } + if Instant::now() >= deadline { + return Err( + format!("backfill of {bucket} did not reach the expected state within {timeout:?}; last: {last}").into(), + ); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + + /// Number of keys under `prefix` listed by the RustFS under test (local + /// state only, no migration side effects). + pub async fn local_key_count(&self, bucket: &str, prefix: &str) -> Result { + let mut count = 0; + let mut token: Option = None; + loop { + let page = self + .client + .list_objects_v2() + .bucket(bucket) + .prefix(prefix) + .max_keys(1000) + .set_continuation_token(token.take()) + .send() + .await?; + count += page.contents().len(); + match page.next_continuation_token() { + Some(next) if page.is_truncated().unwrap_or(false) => token = Some(next.to_string()), + _ => return Ok(count), + } + } + } + async fn admin( &self, method: http::Method, diff --git a/crates/e2e_test/src/on_demand_migration/mod.rs b/crates/e2e_test/src/on_demand_migration/mod.rs index 54686c386..f13a11409 100644 --- a/crates/e2e_test/src/on_demand_migration/mod.rs +++ b/crates/e2e_test/src/on_demand_migration/mod.rs @@ -17,13 +17,15 @@ //! `common` is the shared environment: one RustFS under test, one programmable //! fake S3 source, admin-API wrappers, seeding and local-state assertions. //! `harness_self_test` proves the harness itself; `get_basic_test` covers the -//! GET read-through (rustfs/backlog#2156). The fault, concurrency, +//! GET read-through (rustfs/backlog#2156) and `backfill_test` the background +//! backfill job (ODM-12, rustfs/backlog#2159). The fault, concurrency, //! interaction and real-source matrix is rustfs/backlog#2158; its lane split //! lives in `.config/nextest.toml` (fault / concurrency / real source run //! nightly, the rest in the merge lane). pub mod common; +mod backfill_test; mod concurrency_test; mod fault_test; mod get_basic_test; diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index e5c10d288..7f265c564 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -161,9 +161,20 @@ pub mod bucket { }; pub use crate::bucket::on_demand_migration::{ EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS, - PullCompletion, PullQueue, PullReason, PullSource, SourceBody, WriteBackBody, WriteBackError, WriteBackOutcome, - WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with, + PullCompletion, PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, WriteBackBody, WriteBackError, + WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with, }; + pub mod backfill { + pub use crate::bucket::on_demand_migration::backfill::{ + BACKFILL_CHECKPOINT_FILE, BACKFILL_CHECKPOINT_FORMAT_VERSION, BACKFILL_FAILED_KEYS_CAPACITY, BACKFILL_LEASE, + BACKFILL_LEASE_LOCK_PREFIX, BACKFILL_LIST_PAGE_SIZE, BACKFILL_RECOVERY_INTERVAL, BACKFILL_SAVE_EVERY_KEYS, + BACKFILL_SAVE_INTERVAL, BackfillCheckpoint, BackfillContext, BackfillContextFactory, BackfillError, + BackfillLastError, BackfillOwner, BackfillRecoveryStats, BackfillRequest, BackfillRunner, BackfillState, + BucketBackfillContext, LocalBackfillObject, PriorityPullPermits, PullPermit, PullPriority, SkipExisting, + StoredCheckpoint, SysBackfillContexts, global_backfill_runner, install_global_backfill_runner, key_hash, + read_checkpoint, run_backfill_recovery_loop, spawn_backfill_recovery_loop, + }; + } pub mod source_client { pub use crate::bucket::on_demand_migration::source_client::{ SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceObject, SourcePage, SourceProbe, diff --git a/crates/ecstore/src/bucket/on_demand_migration/backfill.rs b/crates/ecstore/src/bucket/on_demand_migration/backfill.rs new file mode 100644 index 000000000..14bb716d0 --- /dev/null +++ b/crates/ecstore/src/bucket/on_demand_migration/backfill.rs @@ -0,0 +1,2222 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! On-demand migration backfill job (ODM-12, rustfs/backlog#2159). +//! +//! Read-through only migrates objects somebody asked for; the backfill job +//! walks the source bucket with `ListObjectsV2` and queues every remaining +//! key through the write-back pipeline (`pull.rs`) with +//! [`PullReason::Backfill`]. One job per bucket, one runner per job: +//! +//! - The checkpoint `buckets//on-demand-migration-backfill.json` +//! under the metadata bucket is the durable job record: cursor, counters, +//! owner and lease. It is saved every [`BACKFILL_SAVE_EVERY_KEYS`] keys or +//! [`BACKFILL_SAVE_INTERVAL`], and at every page end, with an `If-Match` +//! compare-and-set so a concurrent cancel or takeover is never overwritten. +//! - The `continuation_token` only advances once every pull queued from the +//! page before it has reported back, so a crash re-lists at most one page +//! (already-present keys are then skipped, never re-pulled). +//! - The owner holds a lease of [`BACKFILL_LEASE`] renewed by every save. The +//! recovery loop ([`run_backfill_recovery_loop`]) scans the buckets this +//! node has an ODM state for every [`BACKFILL_RECOVERY_INTERVAL`] and takes +//! over a running job whose lease expired (or whose owner is this very node, +//! which after a restart cannot still be running it). Start and takeover +//! are serialized cluster-wide by the namespace lock `odm-backfill/`. +//! - A config change or removal fires the bucket state's cancellation token; +//! the job records `cancelled` and keeps the checkpoint for inspection. A +//! takeover compares `config_updated_at` for the same reason. +//! - Backfill pulls take permits after online misses: [`PriorityPullPermits`] +//! never hands a permit to a backfill waiter while an online request waits, +//! and the job keeps at most `2 * max_concurrent_pulls` pulls outstanding. +//! +//! Failed keys are recorded as hashes only; object keys appear in logs at +//! `trace` and nowhere else. + +use super::pull::{EnqueueOutcome, PullReason, QueuedPullOutcome}; +use super::source_client::{SourceError, SourcePage}; +use super::sys::{BucketOdmState, OnDemandMigrationSys}; +use crate::bucket::metadata_sys::bucket_metadata_sys_of; +use crate::config::com::{read_config_with_metadata, save_config_with_opts}; +use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; +use crate::error::Error as StorageError; +use crate::object_api::ObjectOptions; +use crate::runtime::sources::local_node_name; +use crate::set_disk::get_lock_acquire_timeout; +use crate::storage_api_contracts::{namespace::NamespaceLocking as _, object::HTTPPreconditions, object::ObjectOperations as _}; +use crate::store::ECStore; +use async_trait::async_trait; +use futures::StreamExt; +use futures::stream::FuturesUnordered; +use parking_lot::Mutex; +use rustfs_utils::http::metadata_compat::{SUFFIX_ODM_SOURCE_ETAG, has_internal_suffix}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashMap}; +use std::fmt; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; +use time::OffsetDateTime; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, TryAcquireError, oneshot, watch}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; +use tracing::{debug, info, trace, warn}; +use uuid::Uuid; + +const EVENT_ODM_BACKFILL_STATE: &str = "odm_backfill_state"; +const EVENT_ODM_BACKFILL_LEASE_TAKEOVER: &str = "odm_backfill_lease_takeover"; +const EVENT_ODM_BACKFILL_CHECKPOINT: &str = "odm_backfill_checkpoint"; +const EVENT_ODM_BACKFILL_RECOVERY: &str = "odm_backfill_recovery"; +const EVENT_ODM_BACKFILL_KEY: &str = "odm_backfill_key"; +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_ON_DEMAND_MIGRATION: &str = "on_demand_migration"; + +/// Checkpoint file name under `buckets//` in the metadata bucket. +pub const BACKFILL_CHECKPOINT_FILE: &str = "on-demand-migration-backfill.json"; +/// Checkpoint document version this build reads and writes. +pub const BACKFILL_CHECKPOINT_FORMAT_VERSION: u32 = 1; +/// Namespace lock prefix serializing start and takeover per bucket. +pub const BACKFILL_LEASE_LOCK_PREFIX: &str = "odm-backfill/"; +/// Owner lease length; every checkpoint save renews it. +pub const BACKFILL_LEASE: Duration = Duration::from_secs(60); +/// Longest gap between two checkpoint saves of a running job. +pub const BACKFILL_SAVE_INTERVAL: Duration = Duration::from_secs(10); +/// Keys processed between two checkpoint saves. +pub const BACKFILL_SAVE_EVERY_KEYS: u64 = 1000; +/// Interval of the recovery scan. +pub const BACKFILL_RECOVERY_INTERVAL: Duration = Duration::from_secs(60); +/// Retry interval of the recovery scan while a takeover had to be deferred +/// (the bucket state was not built yet, or the lock was busy). +pub const BACKFILL_RECOVERY_RETRY_INTERVAL: Duration = Duration::from_secs(5); +/// `max-keys` of every source listing. +pub const BACKFILL_LIST_PAGE_SIZE: i32 = 1000; +/// Ring capacity of `failed_keys`. +pub const BACKFILL_FAILED_KEYS_CAPACITY: usize = 1000; +/// Retries after the first attempt of a retryable listing failure. +const LIST_MAX_RETRIES: usize = 3; +const LIST_RETRY_BASE_DELAYS: [Duration; LIST_MAX_RETRIES] = + [Duration::from_secs(1), Duration::from_secs(4), Duration::from_secs(16)]; +/// Pause between polls while the pull queue is full and nothing is +/// outstanding, or while the breaker rejects source traffic. +const BACKFILL_IDLE_POLL: Duration = Duration::from_millis(200); +/// Lock wait of a takeover attempt; a busy lock defers to the next scan. +const TAKEOVER_LOCK_TIMEOUT: Duration = Duration::from_secs(5); +/// How long an admin cancel waits for the local job to write its final state. +const CANCEL_SETTLE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Lifecycle of a backfill job as persisted in the checkpoint. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BackfillState { + Pending, + Running, + Paused, + Cancelled, + Completed, + CompletedWithFailures, + Failed, +} + +impl BackfillState { + pub fn as_str(self) -> &'static str { + match self { + BackfillState::Pending => "pending", + BackfillState::Running => "running", + BackfillState::Paused => "paused", + BackfillState::Cancelled => "cancelled", + BackfillState::Completed => "completed", + BackfillState::CompletedWithFailures => "completed_with_failures", + BackfillState::Failed => "failed", + } + } + + /// Whether a runner owns (or should own) the job. + pub fn is_active(self) -> bool { + matches!(self, BackfillState::Pending | BackfillState::Running) + } +} + +impl fmt::Display for BackfillState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// What to do with a listed key that already exists locally. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SkipExisting { + /// A current local version means the key is done. + #[default] + Always, + /// Re-pull when the recorded source ETag or the size differs. + EtagOrSize, +} + +impl SkipExisting { + pub fn as_str(self) -> &'static str { + match self { + SkipExisting::Always => "always", + SkipExisting::EtagOrSize => "etag_or_size", + } + } + + pub fn parse(label: &str) -> Option { + match label { + "always" => Some(SkipExisting::Always), + "etag_or_size" => Some(SkipExisting::EtagOrSize), + _ => None, + } + } +} + +/// Last failure recorded by the job. `key_hash` is the xxh3 of the key, +/// never the key itself. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BackfillLastError { + pub class: String, + #[serde(default)] + pub key_hash: Option, + #[serde(with = "time::serde::rfc3339")] + pub at: OffsetDateTime, +} + +/// Node running the job and the lease it holds. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BackfillOwner { + pub node: String, + #[serde(with = "time::serde::rfc3339")] + pub lease_until: OffsetDateTime, +} + +/// Durable job record (see the module docs). Unknown fields are kept and +/// written back so a newer build's fields survive an older node's save. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BackfillCheckpoint { + pub format_version: u32, + pub job_id: Uuid, + pub state: BackfillState, + /// `updated_at` of the ODM config the job was started for; a different + /// value at takeover cancels the job. + #[serde(with = "time::serde::rfc3339")] + pub config_updated_at: OffsetDateTime, + #[serde(default)] + pub prefix: Option, + #[serde(default)] + pub skip_existing: SkipExisting, + #[serde(default)] + pub dry_run: bool, + #[serde(default)] + pub continuation_token: Option, + #[serde(default)] + pub listed: u64, + #[serde(default)] + pub enqueued: u64, + #[serde(default)] + pub pulled: u64, + #[serde(default)] + pub skipped_existing: u64, + #[serde(default)] + pub failed: u64, + #[serde(default)] + pub bytes: u64, + #[serde(default)] + pub last_key: Option, + #[serde(default)] + pub last_error: Option, + /// Ring of at most [`BACKFILL_FAILED_KEYS_CAPACITY`] key hashes. + #[serde(default)] + pub failed_keys: Vec, + #[serde(with = "time::serde::rfc3339")] + pub started_at: OffsetDateTime, + #[serde(with = "time::serde::rfc3339")] + pub updated_at: OffsetDateTime, + #[serde(default)] + pub owner: Option, + #[serde(flatten)] + pub extra: BTreeMap, +} + +impl BackfillCheckpoint { + /// Decode a persisted checkpoint; a foreign `format_version` is a typed + /// error rather than a best-effort read. + pub fn from_json(bytes: &[u8]) -> Result { + #[derive(Deserialize)] + struct Header { + format_version: u32, + } + let header: Header = serde_json::from_slice(bytes).map_err(|err| BackfillError::Malformed(err.to_string()))?; + if header.format_version != BACKFILL_CHECKPOINT_FORMAT_VERSION { + return Err(BackfillError::UnsupportedFormatVersion { + found: header.format_version, + supported: BACKFILL_CHECKPOINT_FORMAT_VERSION, + }); + } + serde_json::from_slice(bytes).map_err(|err| BackfillError::Malformed(err.to_string())) + } + + pub fn to_json(&self) -> Result, BackfillError> { + serde_json::to_vec(self).map_err(|err| BackfillError::Malformed(err.to_string())) + } + + /// Whether the owner's lease still excludes a takeover at `now`. + pub fn lease_valid_at(&self, now: OffsetDateTime) -> bool { + self.owner.as_ref().is_some_and(|owner| owner.lease_until > now) + } + + fn new(request: &BackfillRequest, config_updated_at: OffsetDateTime, node: &str, now: OffsetDateTime) -> Self { + Self { + format_version: BACKFILL_CHECKPOINT_FORMAT_VERSION, + job_id: Uuid::new_v4(), + state: BackfillState::Running, + config_updated_at, + prefix: request.prefix.clone(), + skip_existing: request.skip_existing, + dry_run: request.dry_run, + continuation_token: None, + listed: 0, + enqueued: 0, + pulled: 0, + skipped_existing: 0, + failed: 0, + bytes: 0, + last_key: None, + last_error: None, + failed_keys: Vec::new(), + started_at: now, + updated_at: now, + owner: Some(BackfillOwner { + node: node.to_string(), + lease_until: now + BACKFILL_LEASE, + }), + extra: BTreeMap::new(), + } + } + + fn record_failure(&mut self, class: &str, key: Option<&str>, now: OffsetDateTime) { + let key_hash = key.map(key_hash); + if let Some(hash) = &key_hash { + if self.failed_keys.len() >= BACKFILL_FAILED_KEYS_CAPACITY { + self.failed_keys.remove(0); + } + self.failed_keys.push(hash.clone()); + } + self.last_error = Some(BackfillLastError { + class: class.to_string(), + key_hash, + at: now, + }); + } +} + +/// Stable, non-reversible identifier of a key for checkpoints and logs. +pub fn key_hash(key: &str) -> String { + format!("{:016x}", xxhash_rust::xxh3::xxh3_64(key.as_bytes())) +} + +/// Admin `start` parameters. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BackfillRequest { + pub prefix: Option, + pub skip_existing: SkipExisting, + /// List and count only; nothing is queued. + pub dry_run: bool, +} + +#[derive(Debug, thiserror::Error)] +pub enum BackfillError { + #[error("backfill checkpoint is malformed: {0}")] + Malformed(String), + #[error("unsupported backfill checkpoint format version {found} (this build supports {supported})")] + UnsupportedFormatVersion { found: u32, supported: u32 }, + #[error("a backfill job is already running for bucket {bucket} (job {job_id}, owner {owner})")] + AlreadyRunning { bucket: String, job_id: Uuid, owner: String }, + #[error("no backfill job recorded for bucket {0}")] + NotFound(String), + #[error("bucket {0} has no usable on-demand migration state on this node")] + Unavailable(String), + #[error("bucket {0} has no on-demand migration config")] + NotConfigured(String), + #[error("backfill lease lock for bucket {0} is busy")] + LeaseBusy(String), + #[error("backfill checkpoint for bucket {0} changed concurrently")] + Conflict(String), + #[error("backfill runner is not installed")] + RunnerNotInstalled, + #[error(transparent)] + Storage(#[from] StorageError), +} + +/// Current local version of a key as the skip policy sees it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LocalBackfillObject { + pub size: u64, + /// The `odm-source-etag` provenance value when present, else the local ETag. + pub source_etag: Option, +} + +/// Receiver of one queued pull's report; `None` when the pull was coalesced +/// into one already running. +pub type PullReport = Option>; + +/// Everything the job needs from its bucket, so the loop can run against a +/// mock in unit tests. Production: [`BucketBackfillContext`]. +#[async_trait] +pub trait BackfillContext: Send + Sync { + /// One source page in the local key namespace. + async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result; + /// Whether the breaker admits source traffic right now. + fn source_available(&self) -> bool; + /// Current local version of `key`; `None` when absent or a delete marker. + async fn local_object(&self, key: &str) -> Result, StorageError>; + /// Queue a backfill pull of `key`. + fn enqueue(&self, key: &str) -> (EnqueueOutcome, PullReport); + /// Backfill pulls in flight or queued before the job waits for a report. + /// Bounding this keeps the queue free for online misses and makes a + /// cancel take effect within a few pulls. + fn max_outstanding(&self) -> usize { + 16 + } + /// Fires when the bucket's ODM state is replaced or removed. + fn cancel_token(&self) -> CancellationToken; + /// `updated_at` of the bucket's ODM config; `None` when unconfigured. + async fn config_updated_at(&self) -> Result, StorageError>; +} + +/// Resolves the buckets a runner may run jobs for. +pub trait BackfillContextFactory: Send + Sync { + fn context(&self, bucket: &str) -> Option>; + /// Buckets with a usable ODM state on this node, for the recovery scan. + fn buckets(&self) -> Vec; +} + +/// [`BackfillContext`] over a live [`BucketOdmState`]. +pub struct BucketBackfillContext { + api: Arc, + state: Arc, +} + +impl BucketBackfillContext { + pub fn new(api: Arc, state: Arc) -> Self { + Self { api, state } + } +} + +#[async_trait] +impl BackfillContext for BucketBackfillContext { + async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result { + let client = self.state.client().map_err(|err| SourceError::Unsupported(err.to_string()))?; + let started = Instant::now(); + let result = client.list_objects_v2(prefix, token, max_keys).await; + // A 404 on a listing is the bucket, not a key: keep it out of the + // negative cache but let the breaker see everything else. + match &result { + Err(SourceError::NotFound) => {} + other => self.state.observe_source(started.elapsed(), "", other.as_ref().err()), + } + result + } + + fn source_available(&self) -> bool { + self.state.breaker().allow_request() + } + + async fn local_object(&self, key: &str) -> Result, StorageError> { + let info = match self + .api + .get_object_info(self.state.bucket(), key, &ObjectOptions::default()) + .await + { + Ok(info) => info, + Err(err) if err.is_not_found() => return Ok(None), + Err(err) => return Err(err), + }; + if info.delete_marker { + return Ok(None); + } + let source_etag = info + .user_defined + .iter() + .find(|(name, _)| has_internal_suffix(name, SUFFIX_ODM_SOURCE_ETAG)) + .map(|(_, value)| value.clone()) + .filter(|value| !value.is_empty()) + .or_else(|| info.etag.clone()); + Ok(Some(LocalBackfillObject { + size: u64::try_from(info.size).unwrap_or(0), + source_etag, + })) + } + + fn enqueue(&self, key: &str) -> (EnqueueOutcome, PullReport) { + self.state.enqueue_pull_with_report(key, PullReason::Backfill) + } + + fn max_outstanding(&self) -> usize { + usize::try_from(self.state.config().policy.max_concurrent_pulls) + .unwrap_or(usize::MAX) + .saturating_mul(2) + .max(2) + } + + fn cancel_token(&self) -> CancellationToken { + self.state.cancel_token() + } + + async fn config_updated_at(&self) -> Result, StorageError> { + let sys = bucket_metadata_sys_of(&self.api.ctx)?; + let guard = sys.read().await; + Ok(guard + .get_on_demand_migration_config(self.state.bucket()) + .await? + .map(|(_, updated_at)| updated_at)) + } +} + +/// Factory over the process-wide [`OnDemandMigrationSys`]. +pub struct SysBackfillContexts { + api: Arc, + sys: &'static OnDemandMigrationSys, +} + +impl SysBackfillContexts { + pub fn new(api: Arc, sys: &'static OnDemandMigrationSys) -> Self { + Self { api, sys } + } +} + +impl BackfillContextFactory for SysBackfillContexts { + fn context(&self, bucket: &str) -> Option> { + let state = self.sys.state(bucket)?; + state.client().ok()?; + Some(Arc::new(BucketBackfillContext::new(Arc::clone(&self.api), state))) + } + + fn buckets(&self) -> Vec { + self.sys.bucket_names() + } +} + +/// Two-tier pull permits: online misses queue on the semaphore, backfill +/// waiters only try for a permit while no online request is waiting, and +/// re-check on every release or online arrival/departure. +pub struct PriorityPullPermits { + semaphore: Arc, + online_waiters: AtomicUsize, + /// Bumped on every permit release and every online waiter change. + epoch: watch::Sender, +} + +impl fmt::Debug for PriorityPullPermits { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PriorityPullPermits") + .field("available", &self.semaphore.available_permits()) + .field("online_waiters", &self.online_waiters.load(Ordering::Relaxed)) + .finish() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PullPriority { + /// A request is waiting on this pull. + Online, + /// Nobody waits; yield to online requests. + Backfill, +} + +/// A held pull permit; dropping it wakes backfill waiters. +pub struct PullPermit { + inner: Option, + permits: Arc, +} + +impl fmt::Debug for PullPermit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PullPermit").finish_non_exhaustive() + } +} + +impl Drop for PullPermit { + fn drop(&mut self) { + // Release before waking so a woken backfill waiter finds the permit. + drop(self.inner.take()); + self.permits.bump(); + } +} + +struct OnlineWaiterGuard<'a>(&'a PriorityPullPermits); + +impl Drop for OnlineWaiterGuard<'_> { + fn drop(&mut self) { + self.0.online_waiters.fetch_sub(1, Ordering::AcqRel); + self.0.bump(); + } +} + +impl PriorityPullPermits { + pub fn new(permits: usize) -> Arc { + Arc::new(Self { + semaphore: Arc::new(Semaphore::new(permits.max(1))), + online_waiters: AtomicUsize::new(0), + epoch: watch::channel(0).0, + }) + } + + pub fn available_permits(&self) -> usize { + self.semaphore.available_permits() + } + + pub fn online_waiters(&self) -> usize { + self.online_waiters.load(Ordering::Acquire) + } + + fn bump(&self) { + self.epoch.send_modify(|epoch| *epoch = epoch.wrapping_add(1)); + } + + /// Resolves with a permit; `Err` only when the semaphore is closed. + pub async fn acquire(self: &Arc, priority: PullPriority) -> Result { + match priority { + PullPriority::Online => { + self.online_waiters.fetch_add(1, Ordering::AcqRel); + self.bump(); + let _waiting = OnlineWaiterGuard(self); + let inner = Arc::clone(&self.semaphore).acquire_owned().await?; + Ok(PullPermit { + inner: Some(inner), + permits: Arc::clone(self), + }) + } + PullPriority::Backfill => { + let mut epoch = self.epoch.subscribe(); + loop { + epoch.borrow_and_update(); + if self.online_waiters.load(Ordering::Acquire) == 0 { + match Arc::clone(&self.semaphore).try_acquire_owned() { + Ok(inner) => { + return Ok(PullPermit { + inner: Some(inner), + permits: Arc::clone(self), + }); + } + Err(TryAcquireError::Closed) => { + // Surface the same error an online waiter gets. + return Arc::clone(&self.semaphore).acquire_owned().await.map(|inner| PullPermit { + inner: Some(inner), + permits: Arc::clone(self), + }); + } + Err(TryAcquireError::NoPermits) => {} + } + } + if epoch.changed().await.is_err() { + return Arc::clone(&self.semaphore).acquire_owned().await.map(|inner| PullPermit { + inner: Some(inner), + permits: Arc::clone(self), + }); + } + } + } + } + } +} + +fn checkpoint_path(bucket: &str) -> String { + format!("{BUCKET_META_PREFIX}/{bucket}/{BACKFILL_CHECKPOINT_FILE}") +} + +fn lease_lock_key(bucket: &str) -> String { + format!("{BACKFILL_LEASE_LOCK_PREFIX}{bucket}") +} + +/// A checkpoint together with the ETag the next save must match. +#[derive(Clone, Debug)] +pub struct StoredCheckpoint { + pub checkpoint: BackfillCheckpoint, + pub etag: String, +} + +pub async fn read_checkpoint(api: &Arc, bucket: &str) -> Result, BackfillError> { + match read_config_with_metadata(Arc::clone(api), &checkpoint_path(bucket), &ObjectOptions::default()).await { + Ok((data, info)) => { + let etag = info + .etag + .ok_or_else(|| BackfillError::Malformed("checkpoint has no entity tag".to_string()))?; + Ok(Some(StoredCheckpoint { + checkpoint: BackfillCheckpoint::from_json(&data)?, + etag, + })) + } + Err(StorageError::ConfigNotFound) | Err(StorageError::FileNotFound) => Ok(None), + Err(err) => Err(err.into()), + } +} + +/// Compare-and-set save: `expected_etag` `None` requires the file to be +/// absent. Returns the ETag of the saved document. +async fn write_checkpoint( + api: &Arc, + bucket: &str, + checkpoint: &BackfillCheckpoint, + expected_etag: Option<&str>, +) -> Result { + let data = checkpoint.to_json()?; + let preconditions = match expected_etag { + Some(etag) => HTTPPreconditions { + if_match: Some(etag.to_string()), + ..Default::default() + }, + None => HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }, + }; + let opts = ObjectOptions { + max_parity: true, + http_preconditions: Some(preconditions), + ..Default::default() + }; + match save_config_with_opts(Arc::clone(api), &checkpoint_path(bucket), data, &opts).await { + Ok(()) => {} + Err(StorageError::PreconditionFailed) => return Err(BackfillError::Conflict(bucket.to_string())), + Err(err) => return Err(err.into()), + } + let stored = read_checkpoint(api, bucket) + .await? + .ok_or_else(|| BackfillError::Conflict(bucket.to_string()))?; + if stored.checkpoint.job_id != checkpoint.job_id || stored.checkpoint.updated_at != checkpoint.updated_at { + return Err(BackfillError::Conflict(bucket.to_string())); + } + Ok(stored.etag) +} + +/// Runtime handle of a job running in this process. +struct JobHandle { + job_id: Uuid, + cancel: CancellationToken, + snapshot: Mutex, + done: watch::Sender, +} + +impl JobHandle { + fn is_done(&self) -> bool { + *self.done.borrow() + } + + async fn wait_done(&self) { + let mut rx = self.done.subscribe(); + let _ = rx.wait_for(|done| *done).await; + } +} + +/// Per-node backfill coordinator: starts, cancels, reports and recovers jobs. +pub struct BackfillRunner { + api: Arc, + node: String, + contexts: Arc, + jobs: Mutex>>, +} + +impl fmt::Debug for BackfillRunner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BackfillRunner") + .field("node", &self.node) + .field("jobs", &self.jobs.lock().keys().collect::>()) + .finish() + } +} + +static GLOBAL_BACKFILL_RUNNER: OnceLock> = OnceLock::new(); + +/// Publishes the process-wide runner; `false` when one was already installed. +pub fn install_global_backfill_runner(runner: Arc) -> bool { + GLOBAL_BACKFILL_RUNNER.set(runner).is_ok() +} + +pub fn global_backfill_runner() -> Option> { + GLOBAL_BACKFILL_RUNNER.get().cloned() +} + +/// Outcome of one recovery scan. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BackfillRecoveryStats { + pub scanned: usize, + pub taken_over: usize, + pub cancelled: usize, + /// Expired jobs this pass could not take over yet (lock busy, no state). + pub deferred: usize, +} + +impl BackfillRunner { + pub fn new(api: Arc, node: impl Into, contexts: Arc) -> Arc { + Arc::new(Self { + api, + node: node.into(), + contexts, + jobs: Mutex::new(HashMap::new()), + }) + } + + /// Runner named after this node's endpoint. + pub async fn for_local_node(api: Arc, contexts: Arc) -> Arc { + let node = local_node_name().await; + Self::new(api, node, contexts) + } + + pub fn node(&self) -> &str { + &self.node + } + + /// Whether a job for `bucket` runs in this process. + pub fn is_running_locally(&self, bucket: &str) -> bool { + self.live_job(bucket).is_some() + } + + /// Last persisted progress of every job running in this process, by + /// bucket name. Observability reads this instead of the checkpoint + /// documents so a metrics cycle never touches storage; a job that has + /// finished is dropped here exactly as [`Self::is_running_locally`] + /// drops it. Lock order: `jobs` before a handle's `snapshot`. + pub fn local_job_snapshots(&self) -> Vec<(String, BackfillCheckpoint)> { + let mut jobs = self.jobs.lock(); + jobs.retain(|_, handle| !handle.is_done()); + let mut snapshots: Vec<_> = jobs + .iter() + .map(|(bucket, handle)| (bucket.clone(), handle.snapshot.lock().clone())) + .collect(); + snapshots.sort_by(|left, right| left.0.cmp(&right.0)); + snapshots + } + + fn live_job(&self, bucket: &str) -> Option> { + let mut jobs = self.jobs.lock(); + match jobs.get(bucket) { + Some(handle) if !handle.is_done() => Some(Arc::clone(handle)), + Some(_) => { + jobs.remove(bucket); + None + } + None => None, + } + } + + /// Resolves once no job for `bucket` runs in this process. + pub async fn wait_until_idle(&self, bucket: &str) { + if let Some(handle) = self.live_job(bucket) { + handle.wait_done().await; + } + } + + async fn lease_lock(&self, bucket: &str, timeout: Duration) -> Result { + let lock = self.api.new_ns_lock(RUSTFS_META_BUCKET, &lease_lock_key(bucket)).await?; + lock.get_write_lock_quiet(timeout) + .await + .map_err(|_| BackfillError::LeaseBusy(bucket.to_string())) + } + + /// Starts a job for `bucket`; `AlreadyRunning` while one holds a lease. + pub async fn start(&self, bucket: &str, request: BackfillRequest) -> Result { + let context = self + .contexts + .context(bucket) + .ok_or_else(|| BackfillError::Unavailable(bucket.to_string()))?; + let config_updated_at = context + .config_updated_at() + .await? + .ok_or_else(|| BackfillError::NotConfigured(bucket.to_string()))?; + + let _lock = self.lease_lock(bucket, get_lock_acquire_timeout()).await?; + let now = OffsetDateTime::now_utc(); + if let Some(handle) = self.live_job(bucket) { + return Err(BackfillError::AlreadyRunning { + bucket: bucket.to_string(), + job_id: handle.job_id, + owner: self.node.clone(), + }); + } + let stored = read_checkpoint(&self.api, bucket).await?; + if let Some(stored) = &stored + && stored.checkpoint.state.is_active() + && stored.checkpoint.lease_valid_at(now) + { + return Err(BackfillError::AlreadyRunning { + bucket: bucket.to_string(), + job_id: stored.checkpoint.job_id, + owner: stored.checkpoint.owner.as_ref().map(|o| o.node.clone()).unwrap_or_default(), + }); + } + let checkpoint = BackfillCheckpoint::new(&request, config_updated_at, &self.node, now); + let etag = write_checkpoint(&self.api, bucket, &checkpoint, stored.as_ref().map(|s| s.etag.as_str())).await?; + info!( + event = EVENT_ODM_BACKFILL_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + state = checkpoint.state.as_str(), + result = "started", + bucket = %bucket, + job_id = %checkpoint.job_id, + dry_run = checkpoint.dry_run, + skip_existing = checkpoint.skip_existing.as_str(), + "On-demand migration backfill job started" + ); + self.spawn_job( + bucket, + context, + StoredCheckpoint { + checkpoint: checkpoint.clone(), + etag, + }, + ); + Ok(checkpoint) + } + + /// Cancels the job for `bucket` wherever it runs. Idempotent on a + /// finished job; `NotFound` when no checkpoint exists. + pub async fn cancel(&self, bucket: &str) -> Result { + if let Some(handle) = self.live_job(bucket) { + handle.cancel.cancel(); + let _ = tokio::time::timeout(CANCEL_SETTLE_TIMEOUT, handle.wait_done()).await; + if let Some(stored) = read_checkpoint(&self.api, bucket).await? { + return Ok(stored.checkpoint); + } + return Ok(handle.snapshot.lock().clone()); + } + let _lock = self.lease_lock(bucket, get_lock_acquire_timeout()).await?; + let Some(stored) = read_checkpoint(&self.api, bucket).await? else { + return Err(BackfillError::NotFound(bucket.to_string())); + }; + if !stored.checkpoint.state.is_active() { + return Ok(stored.checkpoint); + } + let mut checkpoint = stored.checkpoint; + let now = OffsetDateTime::now_utc(); + checkpoint.state = BackfillState::Cancelled; + checkpoint.updated_at = now; + write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?; + info!( + event = EVENT_ODM_BACKFILL_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + state = checkpoint.state.as_str(), + result = "cancelled", + bucket = %bucket, + job_id = %checkpoint.job_id, + owner = %checkpoint.owner.as_ref().map(|o| o.node.as_str()).unwrap_or_default(), + "On-demand migration backfill job cancelled remotely" + ); + Ok(checkpoint) + } + + /// Latest checkpoint: the in-memory progress of a local job, else the + /// persisted document. + pub async fn status(&self, bucket: &str) -> Result, BackfillError> { + if let Some(handle) = self.live_job(bucket) { + return Ok(Some(handle.snapshot.lock().clone())); + } + Ok(read_checkpoint(&self.api, bucket).await?.map(|stored| stored.checkpoint)) + } + + /// One recovery pass over the buckets this node has a state for. + pub async fn recover_once(&self) -> BackfillRecoveryStats { + let mut stats = BackfillRecoveryStats::default(); + for bucket in self.contexts.buckets() { + if self.is_running_locally(&bucket) { + continue; + } + stats.scanned += 1; + match self.try_take_over(&bucket).await { + Ok(TakeoverOutcome::NotNeeded) => {} + Ok(TakeoverOutcome::TakenOver) => stats.taken_over += 1, + Ok(TakeoverOutcome::Cancelled) => stats.cancelled += 1, + Ok(TakeoverOutcome::Deferred) => stats.deferred += 1, + Err(err) => { + stats.deferred += 1; + warn!( + event = EVENT_ODM_BACKFILL_RECOVERY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + bucket = %bucket, + error = %err, + "On-demand migration backfill recovery failed for bucket" + ); + } + } + } + debug!( + event = EVENT_ODM_BACKFILL_RECOVERY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + scanned = stats.scanned, + taken_over = stats.taken_over, + cancelled = stats.cancelled, + deferred = stats.deferred, + "On-demand migration backfill recovery pass finished" + ); + stats + } + + fn takeover_due(&self, checkpoint: &BackfillCheckpoint, now: OffsetDateTime) -> bool { + if !checkpoint.state.is_active() { + return false; + } + match &checkpoint.owner { + // A job this node owned before a restart is dead by construction. + Some(owner) => owner.lease_until <= now || owner.node == self.node, + None => true, + } + } + + async fn try_take_over(&self, bucket: &str) -> Result { + let now = OffsetDateTime::now_utc(); + let Some(stored) = read_checkpoint(&self.api, bucket).await? else { + return Ok(TakeoverOutcome::NotNeeded); + }; + if !self.takeover_due(&stored.checkpoint, now) { + return Ok(TakeoverOutcome::NotNeeded); + } + let Some(context) = self.contexts.context(bucket) else { + return Ok(TakeoverOutcome::Deferred); + }; + let _lock = match self.lease_lock(bucket, TAKEOVER_LOCK_TIMEOUT).await { + Ok(lock) => lock, + Err(BackfillError::LeaseBusy(_)) => return Ok(TakeoverOutcome::Deferred), + Err(err) => return Err(err), + }; + // Re-read under the lock: another node may have taken over meanwhile. + let now = OffsetDateTime::now_utc(); + let Some(stored) = read_checkpoint(&self.api, bucket).await? else { + return Ok(TakeoverOutcome::NotNeeded); + }; + if !self.takeover_due(&stored.checkpoint, now) || self.is_running_locally(bucket) { + return Ok(TakeoverOutcome::NotNeeded); + } + let mut checkpoint = stored.checkpoint.clone(); + let previous_owner = checkpoint.owner.as_ref().map(|o| o.node.clone()).unwrap_or_default(); + let config_updated_at = context.config_updated_at().await?; + if config_updated_at.map(|at| at.unix_timestamp_nanos()) != Some(checkpoint.config_updated_at.unix_timestamp_nanos()) { + checkpoint.state = BackfillState::Cancelled; + checkpoint.updated_at = now; + checkpoint.record_failure("config_changed", None, now); + write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?; + info!( + event = EVENT_ODM_BACKFILL_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + state = checkpoint.state.as_str(), + result = "config_changed", + bucket = %bucket, + job_id = %checkpoint.job_id, + "On-demand migration backfill job cancelled: config changed since it started" + ); + return Ok(TakeoverOutcome::Cancelled); + } + checkpoint.state = BackfillState::Running; + checkpoint.updated_at = now; + checkpoint.owner = Some(BackfillOwner { + node: self.node.clone(), + lease_until: now + BACKFILL_LEASE, + }); + let etag = write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?; + warn!( + event = EVENT_ODM_BACKFILL_LEASE_TAKEOVER, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + bucket = %bucket, + job_id = %checkpoint.job_id, + previous_owner = %previous_owner, + listed = checkpoint.listed, + resumed_from_token = checkpoint.continuation_token.is_some(), + "On-demand migration backfill lease taken over" + ); + self.spawn_job(bucket, context, StoredCheckpoint { checkpoint, etag }); + Ok(TakeoverOutcome::TakenOver) + } + + fn spawn_job(&self, bucket: &str, context: Arc, stored: StoredCheckpoint) { + let handle = Arc::new(JobHandle { + job_id: stored.checkpoint.job_id, + cancel: CancellationToken::new(), + snapshot: Mutex::new(stored.checkpoint.clone()), + done: watch::channel(false).0, + }); + self.jobs.lock().insert(bucket.to_string(), Arc::clone(&handle)); + let api = Arc::clone(&self.api); + let node = self.node.clone(); + let bucket = bucket.to_string(); + tokio::spawn(async move { + let mut job = Job { + api, + bucket, + node, + context, + handle: Arc::clone(&handle), + checkpoint: stored.checkpoint, + etag: stored.etag, + keys_since_save: 0, + last_save: Instant::now(), + outstanding: FuturesUnordered::new(), + }; + job.run().await; + handle.done.send_replace(true); + }); + } +} + +enum TakeoverOutcome { + NotNeeded, + TakenOver, + Cancelled, + Deferred, +} + +/// Why the loop stopped before the listing was exhausted. +enum Stop { + /// Admin cancel (local token) or config change (state token). + Cancelled(&'static str), + /// The checkpoint on disk no longer belongs to this job. + Lost, + /// The listing failed for good. + Failed(SourceError), + /// The bucket state went away under us. + Unavailable, +} + +type OutstandingPull = Pin)> + Send>>; + +struct Job { + api: Arc, + bucket: String, + node: String, + context: Arc, + handle: Arc, + checkpoint: BackfillCheckpoint, + etag: String, + keys_since_save: u64, + last_save: Instant, + outstanding: FuturesUnordered, +} + +impl Job { + async fn run(&mut self) { + let outcome = self.main_loop().await; + let now = OffsetDateTime::now_utc(); + let (state, result) = match outcome { + Ok(()) => { + if self.checkpoint.failed > 0 { + (BackfillState::CompletedWithFailures, "completed_with_failures") + } else { + (BackfillState::Completed, "completed") + } + } + Err(Stop::Cancelled(reason)) => { + if reason != "admin" { + self.checkpoint.record_failure(reason, None, now); + } + (BackfillState::Cancelled, reason) + } + Err(Stop::Unavailable) => { + self.checkpoint.record_failure("state_unavailable", None, now); + (BackfillState::Cancelled, "state_unavailable") + } + Err(Stop::Failed(err)) => { + self.checkpoint.record_failure(err.class_label(), None, now); + (BackfillState::Failed, "source_error") + } + Err(Stop::Lost) => { + // The document on disk is authoritative; do not touch it. + info!( + event = EVENT_ODM_BACKFILL_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + state = "lost", + result = "superseded", + bucket = %self.bucket, + job_id = %self.checkpoint.job_id, + "On-demand migration backfill job stopped: checkpoint owned elsewhere" + ); + return; + } + }; + self.checkpoint.state = state; + if let Err(err) = self.save(now).await { + warn!( + event = EVENT_ODM_BACKFILL_CHECKPOINT, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + bucket = %self.bucket, + job_id = %self.checkpoint.job_id, + state = state.as_str(), + error = %err, + "On-demand migration backfill final checkpoint save failed" + ); + } + info!( + event = EVENT_ODM_BACKFILL_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + state = state.as_str(), + result = result, + bucket = %self.bucket, + job_id = %self.checkpoint.job_id, + listed = self.checkpoint.listed, + enqueued = self.checkpoint.enqueued, + pulled = self.checkpoint.pulled, + skipped_existing = self.checkpoint.skipped_existing, + failed = self.checkpoint.failed, + bytes = self.checkpoint.bytes, + "On-demand migration backfill job finished" + ); + } + + fn cancel_reason(&self) -> Option<&'static str> { + if self.handle.cancel.is_cancelled() { + Some("admin") + } else if self.context.cancel_token().is_cancelled() { + Some("config_changed") + } else { + None + } + } + + async fn main_loop(&mut self) -> Result<(), Stop> { + loop { + self.check_cancel()?; + let page = self.list_page().await?; + for object in &page.objects { + self.check_cancel()?; + self.checkpoint.listed += 1; + self.checkpoint.last_key = Some(object.key.clone()); + self.keys_since_save += 1; + if !self.checkpoint.dry_run { + self.process_key(object).await?; + } + self.drain_ready(); + self.tick(false).await?; + } + // Only advance the cursor once every pull of this page reported + // back, so a takeover re-lists at most this page. + self.drain_all().await?; + self.checkpoint.continuation_token = page.next_continuation_token.clone(); + self.tick(true).await?; + if !page.is_truncated { + return Ok(()); + } + } + } + + fn check_cancel(&self) -> Result<(), Stop> { + match self.cancel_reason() { + Some(reason) => Err(Stop::Cancelled(reason)), + None => Ok(()), + } + } + + async fn list_page(&mut self) -> Result { + let mut attempt = 0; + loop { + while !self.context.source_available() { + self.sleep(BACKFILL_IDLE_POLL).await?; + self.tick(false).await?; + } + let prefix = self.checkpoint.prefix.clone(); + let token = self.checkpoint.continuation_token.clone(); + match self + .context + .list_page(prefix.as_deref(), token.as_deref(), BACKFILL_LIST_PAGE_SIZE) + .await + { + Ok(page) => return Ok(page), + Err(err) if err.is_retryable() && attempt < LIST_MAX_RETRIES => { + let delay = LIST_RETRY_BASE_DELAYS[attempt]; + attempt += 1; + debug!( + event = EVENT_ODM_BACKFILL_CHECKPOINT, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + bucket = %self.bucket, + job_id = %self.checkpoint.job_id, + error_class = err.class_label(), + attempt, + "On-demand migration backfill listing failed; retrying" + ); + self.sleep(delay).await?; + self.tick(false).await?; + } + Err(err) => return Err(Stop::Failed(err)), + } + } + } + + async fn process_key(&mut self, object: &super::source_client::SourceObject) -> Result<(), Stop> { + let key = object.key.as_str(); + match self.context.local_object(key).await { + Ok(Some(local)) => { + let skip = match self.checkpoint.skip_existing { + SkipExisting::Always => true, + SkipExisting::EtagOrSize => local.size == object.size && local.source_etag == object.etag, + }; + if skip { + self.checkpoint.skipped_existing += 1; + trace!( + event = EVENT_ODM_BACKFILL_KEY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + result = "skipped_existing", + bucket = %self.bucket, + key = %key, + "On-demand migration backfill skipped an existing object" + ); + return Ok(()); + } + } + Ok(None) => {} + Err(err) => { + let now = OffsetDateTime::now_utc(); + self.checkpoint.failed += 1; + self.checkpoint.record_failure("local_lookup", Some(key), now); + debug!( + event = EVENT_ODM_BACKFILL_KEY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + result = "local_lookup_failed", + bucket = %self.bucket, + key_hash = %key_hash(key), + error = %err, + "On-demand migration backfill could not read the local object" + ); + return Ok(()); + } + } + while self.outstanding.len() >= self.context.max_outstanding() { + self.wait_one().await?; + self.tick(false).await?; + } + loop { + match self.context.enqueue(key) { + (EnqueueOutcome::Enqueued, report) => { + self.checkpoint.enqueued += 1; + if let Some(rx) = report { + let key = key.to_string(); + self.outstanding.push(Box::pin(async move { (key, rx.await) })); + } + trace!( + event = EVENT_ODM_BACKFILL_KEY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + result = "enqueued", + bucket = %self.bucket, + key = %key, + "On-demand migration backfill queued a pull" + ); + return Ok(()); + } + (EnqueueOutcome::Coalesced, _) => { + // Someone else pulls it; its result is not ours to count. + self.checkpoint.enqueued += 1; + return Ok(()); + } + (EnqueueOutcome::QueueFull, _) => { + // Wait, never drop: one completion frees a slot. + if self.outstanding.is_empty() { + self.sleep(BACKFILL_IDLE_POLL).await?; + } else { + self.wait_one().await?; + } + self.tick(false).await?; + } + (EnqueueOutcome::Unavailable, _) => return Err(Stop::Unavailable), + } + } + } + + fn record_report(&mut self, key: &str, report: Result) { + let now = OffsetDateTime::now_utc(); + match report { + Ok(QueuedPullOutcome::Stored { size }) => { + self.checkpoint.pulled += 1; + self.checkpoint.bytes += size; + } + Ok(QueuedPullOutcome::AlreadyPresent) => { + self.checkpoint.skipped_existing += 1; + } + Ok(QueuedPullOutcome::Failed(err)) => { + self.checkpoint.failed += 1; + self.checkpoint.record_failure(err.reason.as_str(), Some(key), now); + } + Err(_) => { + self.checkpoint.failed += 1; + self.checkpoint.record_failure("canceled", Some(key), now); + } + } + } + + /// Consumes every report that already arrived without waiting. + fn drain_ready(&mut self) { + use futures::FutureExt; + while let Some(Some((key, report))) = self.outstanding.next().now_or_never() { + self.record_report(&key, report); + } + } + + /// Waits for one report, keeping the checkpoint fresh meanwhile. + async fn wait_one(&mut self) -> Result<(), Stop> { + let admin_cancel = self.handle.cancel.clone(); + let state_cancel = self.context.cancel_token(); + loop { + self.check_cancel()?; + let next = tokio::select! { + next = self.outstanding.next() => next, + _ = tokio::time::sleep(BACKFILL_SAVE_INTERVAL) => { + self.tick(false).await?; + continue; + } + _ = admin_cancel.cancelled() => return Err(Stop::Cancelled("admin")), + _ = state_cancel.cancelled() => return Err(Stop::Cancelled("config_changed")), + }; + match next { + Some((key, report)) => { + self.record_report(&key, report); + return Ok(()); + } + None => return Ok(()), + } + } + } + + async fn drain_all(&mut self) -> Result<(), Stop> { + while !self.outstanding.is_empty() { + self.wait_one().await?; + self.tick(false).await?; + } + Ok(()) + } + + async fn sleep(&mut self, duration: Duration) -> Result<(), Stop> { + let state_cancel = self.context.cancel_token(); + tokio::select! { + _ = tokio::time::sleep(duration) => Ok(()), + _ = self.handle.cancel.cancelled() => Err(Stop::Cancelled("admin")), + _ = state_cancel.cancelled() => Err(Stop::Cancelled("config_changed")), + } + } + + /// Saves when due (or `force`d); every save renews the lease. + async fn tick(&mut self, force: bool) -> Result<(), Stop> { + if !force && self.keys_since_save < BACKFILL_SAVE_EVERY_KEYS && self.last_save.elapsed() < BACKFILL_SAVE_INTERVAL { + return Ok(()); + } + let now = OffsetDateTime::now_utc(); + match self.save(now).await { + Ok(()) => Ok(()), + Err(BackfillError::Conflict(_)) => match self.reconcile_conflict().await { + Ok(()) => Ok(()), + Err(stop) => Err(stop), + }, + Err(err) => { + // Storage hiccup: keep going, the next tick retries. + warn!( + event = EVENT_ODM_BACKFILL_CHECKPOINT, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + bucket = %self.bucket, + job_id = %self.checkpoint.job_id, + error = %err, + "On-demand migration backfill checkpoint save failed" + ); + Ok(()) + } + } + } + + /// The `If-Match` failed: adopt the on-disk ETag when the document is + /// still ours and running, else stop. + async fn reconcile_conflict(&mut self) -> Result<(), Stop> { + let stored = read_checkpoint(&self.api, &self.bucket).await.map_err(|_| Stop::Lost)?; + let Some(stored) = stored else { + return Err(Stop::Lost); + }; + let on_disk = &stored.checkpoint; + if on_disk.job_id != self.checkpoint.job_id + || on_disk.state != BackfillState::Running + || on_disk.owner.as_ref().is_some_and(|owner| owner.node != self.node) + { + return Err(Stop::Lost); + } + self.etag = stored.etag; + Ok(()) + } + + async fn save(&mut self, now: OffsetDateTime) -> Result<(), BackfillError> { + self.checkpoint.updated_at = now; + if self.checkpoint.state.is_active() { + self.checkpoint.owner = Some(BackfillOwner { + node: self.node.clone(), + lease_until: now + BACKFILL_LEASE, + }); + } + let etag = write_checkpoint(&self.api, &self.bucket, &self.checkpoint, Some(&self.etag)).await?; + self.etag = etag; + self.keys_since_save = 0; + self.last_save = Instant::now(); + *self.handle.snapshot.lock() = self.checkpoint.clone(); + Ok(()) + } +} + +/// Spawns [`run_backfill_recovery_loop`] on the store's shutdown token; +/// `false` (nothing spawned) when the store has no background token. +pub fn spawn_backfill_recovery_loop(runner: Arc) -> bool { + let Some(cancel) = runner.api.ctx.background_cancel_token() else { + return false; + }; + tokio::spawn(run_backfill_recovery_loop(runner, cancel)); + true +} + +/// Background recovery: scans every [`BACKFILL_RECOVERY_INTERVAL`], sooner +/// while a takeover is deferred. Registered by the binary at startup. +pub async fn run_backfill_recovery_loop(runner: Arc, cancel: CancellationToken) { + let mut wait = Duration::ZERO; + loop { + tokio::select! { + biased; + _ = cancel.cancelled() => return, + _ = tokio::time::sleep(wait) => {} + } + let stats = runner.recover_once().await; + wait = if stats.deferred > 0 { + BACKFILL_RECOVERY_RETRY_INTERVAL + } else { + BACKFILL_RECOVERY_INTERVAL + }; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks; + use crate::bucket::on_demand_migration::source_client::SourceObject; + use crate::bucket::on_demand_migration::sys::PullError; + use std::collections::{BTreeSet, HashSet}; + use std::sync::atomic::AtomicBool; + + const GOLDEN: &str = r#"{"format_version":1,"job_id":"11111111-1111-4111-8111-111111111111","state":"running","config_updated_at":"2026-09-02T10:00:00Z","prefix":"photos/","skip_existing":"always","dry_run":false,"continuation_token":"cGhvdG9zLzEwMDA=","listed":2000,"enqueued":1500,"pulled":1400,"skipped_existing":500,"failed":3,"bytes":73400320,"last_key":"photos/2024/02.jpg","last_error":{"class":"source_timeout","key_hash":"9f2c3b0a1d4e5f60","at":"2026-09-02T10:05:00Z"},"failed_keys":["9f2c3b0a1d4e5f60"],"started_at":"2026-09-02T10:00:30Z","updated_at":"2026-09-02T10:05:10Z","owner":{"node":"node-a:9000","lease_until":"2026-09-02T10:06:10Z"}}"#; + + fn ts(unix: i64) -> OffsetDateTime { + OffsetDateTime::from_unix_timestamp(unix).expect("timestamp") + } + + #[test] + fn checkpoint_golden_round_trips_byte_for_byte() { + let checkpoint = BackfillCheckpoint::from_json(GOLDEN.as_bytes()).expect("golden decodes"); + assert_eq!(checkpoint.state, BackfillState::Running); + assert_eq!(checkpoint.skip_existing, SkipExisting::Always); + assert_eq!(checkpoint.listed, 2000); + assert_eq!(checkpoint.failed_keys, vec!["9f2c3b0a1d4e5f60".to_string()]); + assert_eq!(checkpoint.owner.as_ref().map(|o| o.node.as_str()), Some("node-a:9000")); + assert!(checkpoint.extra.is_empty()); + let encoded = String::from_utf8(checkpoint.to_json().expect("encodes")).expect("utf-8"); + assert_eq!(encoded, GOLDEN, "field order and formatting are the on-disk contract"); + } + + #[test] + fn checkpoint_keeps_unknown_fields_and_rejects_foreign_versions() { + let newer = GOLDEN.replacen("\"listed\":2000", "\"listed\":2000,\"throttle_hint\":{\"mode\":\"soft\"}", 1); + let checkpoint = BackfillCheckpoint::from_json(newer.as_bytes()).expect("unknown fields are tolerated"); + assert_eq!(checkpoint.listed, 2000); + assert_eq!(checkpoint.extra.get("throttle_hint").and_then(|v| v["mode"].as_str()), Some("soft")); + let re_encoded = String::from_utf8(checkpoint.to_json().expect("encodes")).expect("utf-8"); + assert!( + re_encoded.contains("\"throttle_hint\":{\"mode\":\"soft\"}"), + "unknown fields survive a save" + ); + + let foreign = GOLDEN.replacen("\"format_version\":1", "\"format_version\":2", 1); + match BackfillCheckpoint::from_json(foreign.as_bytes()) { + Err(BackfillError::UnsupportedFormatVersion { found: 2, supported: 1 }) => {} + other => panic!("expected a typed version error, got {other:?}"), + } + assert!(matches!( + BackfillCheckpoint::from_json(b"{\"format_version\":1,\"job_id\":42}"), + Err(BackfillError::Malformed(_)) + )); + assert!(matches!(BackfillCheckpoint::from_json(b"not json"), Err(BackfillError::Malformed(_)))); + } + + #[test] + fn failed_keys_ring_holds_hashes_only() { + let mut checkpoint = BackfillCheckpoint::from_json(GOLDEN.as_bytes()).expect("golden"); + checkpoint.failed_keys.clear(); + for i in 0..(BACKFILL_FAILED_KEYS_CAPACITY + 5) { + checkpoint.record_failure("local_write", Some(&format!("secret/key-{i}")), ts(1)); + } + assert_eq!(checkpoint.failed_keys.len(), BACKFILL_FAILED_KEYS_CAPACITY); + assert_eq!(checkpoint.failed_keys.last(), Some(&key_hash("secret/key-1004"))); + assert_eq!(checkpoint.failed_keys.first(), Some(&key_hash("secret/key-5"))); + let json = String::from_utf8(checkpoint.to_json().expect("encodes")).expect("utf-8"); + assert!(!json.contains("secret/key-"), "plaintext keys must never reach the checkpoint"); + assert_eq!(key_hash("a").len(), 16); + } + + #[test] + fn skip_existing_labels_round_trip() { + assert_eq!(SkipExisting::parse("always"), Some(SkipExisting::Always)); + assert_eq!(SkipExisting::parse("etag_or_size"), Some(SkipExisting::EtagOrSize)); + assert_eq!(SkipExisting::parse("never"), None); + assert_eq!(SkipExisting::EtagOrSize.as_str(), "etag_or_size"); + assert_eq!(BackfillState::CompletedWithFailures.as_str(), "completed_with_failures"); + assert!(BackfillState::Pending.is_active() && BackfillState::Running.is_active()); + assert!(!BackfillState::Cancelled.is_active()); + } + + #[tokio::test] + async fn online_waiters_take_permits_before_backfill() { + let permits = PriorityPullPermits::new(1); + let order = Arc::new(Mutex::new(Vec::new())); + let held = permits.acquire(PullPriority::Online).await.expect("first online permit"); + + let backfill = { + let permits = Arc::clone(&permits); + let order = Arc::clone(&order); + tokio::spawn(async move { + let permit = permits.acquire(PullPriority::Backfill).await.expect("backfill permit"); + order.lock().push("backfill"); + permit + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!backfill.is_finished(), "backfill must wait while the permit is held"); + + let online = { + let permits = Arc::clone(&permits); + let order = Arc::clone(&order); + tokio::spawn(async move { + let permit = permits.acquire(PullPriority::Online).await.expect("online permit"); + order.lock().push("online"); + permit + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(permits.online_waiters(), 1); + + drop(held); + let online_permit = online.await.expect("online task"); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + order.lock().as_slice(), + &["online"], + "the later online waiter wins over the earlier backfill waiter" + ); + assert!(!backfill.is_finished()); + + drop(online_permit); + let backfill_permit = backfill.await.expect("backfill task"); + assert_eq!(order.lock().as_slice(), &["online", "backfill"]); + assert_eq!(permits.available_permits(), 0); + drop(backfill_permit); + assert_eq!(permits.available_permits(), 1); + } + + /// Scripted source + local store + queue with a controllable report path. + struct MockContext { + objects: Vec, + page_size: usize, + local: Mutex>, + enqueued: Mutex>, + list_requests: Mutex>>, + queue_capacity: usize, + pending: Mutex)>>, + fail_keys: HashSet, + auto_complete: AtomicBool, + cancel: CancellationToken, + config_updated_at: Mutex>, + list_error: Mutex>, + } + + impl MockContext { + fn new(keys: usize, page_size: usize) -> Arc { + let objects = (0..keys) + .map(|i| SourceObject { + key: format!("k/{i:05}"), + etag: Some(format!("etag-{i}")), + size: 10 + i as u64, + last_modified: None, + storage_class: None, + is_multipart_etag: false, + }) + .collect(); + Arc::new(Self { + objects, + page_size, + local: Mutex::new(HashMap::new()), + enqueued: Mutex::new(Vec::new()), + list_requests: Mutex::new(Vec::new()), + queue_capacity: usize::MAX, + pending: Mutex::new(Vec::new()), + fail_keys: HashSet::new(), + auto_complete: AtomicBool::new(true), + cancel: CancellationToken::new(), + config_updated_at: Mutex::new(Some(ts(1_700_000_000))), + list_error: Mutex::new(None), + }) + } + + fn complete_pending(&self) { + for (key, tx) in self.pending.lock().drain(..) { + let outcome = if self.fail_keys.contains(&key) { + QueuedPullOutcome::Failed(PullError::new(super::super::stats::PullFailureReason::LocalWrite, "disk full")) + } else { + QueuedPullOutcome::Stored { size: 10 } + }; + let _ = tx.send(outcome); + } + } + + fn enqueued_keys(&self) -> Vec { + self.enqueued.lock().clone() + } + } + + #[async_trait] + impl BackfillContext for MockContext { + async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result { + if let Some(err) = self.list_error.lock().take() { + return Err(err); + } + self.list_requests.lock().push(token.map(str::to_string)); + let start: usize = token.map(|t| t.parse().expect("mock token")).unwrap_or(0); + let page = self.page_size.min(max_keys as usize); + let objects: Vec = self + .objects + .iter() + .filter(|o| prefix.is_none_or(|p| o.key.starts_with(p))) + .skip(start) + .take(page) + .cloned() + .collect(); + let total = self + .objects + .iter() + .filter(|o| prefix.is_none_or(|p| o.key.starts_with(p))) + .count(); + let end = start + objects.len(); + let is_truncated = end < total; + Ok(SourcePage { + objects, + is_truncated, + next_continuation_token: is_truncated.then(|| end.to_string()), + }) + } + + fn source_available(&self) -> bool { + true + } + + async fn local_object(&self, key: &str) -> Result, StorageError> { + Ok(self.local.lock().get(key).cloned()) + } + + fn enqueue(&self, key: &str) -> (EnqueueOutcome, PullReport) { + if self.pending.lock().len() >= self.queue_capacity { + return (EnqueueOutcome::QueueFull, None); + } + self.enqueued.lock().push(key.to_string()); + let (tx, rx) = oneshot::channel(); + if self.auto_complete.load(Ordering::Relaxed) { + let outcome = if self.fail_keys.contains(key) { + QueuedPullOutcome::Failed(PullError::new(super::super::stats::PullFailureReason::LocalWrite, "disk full")) + } else { + QueuedPullOutcome::Stored { size: 10 } + }; + let _ = tx.send(outcome); + } else { + self.pending.lock().push((key.to_string(), tx)); + } + (EnqueueOutcome::Enqueued, Some(rx)) + } + + fn cancel_token(&self) -> CancellationToken { + self.cancel.clone() + } + + async fn config_updated_at(&self) -> Result, StorageError> { + Ok(*self.config_updated_at.lock()) + } + } + + struct MockContexts(Mutex>>); + + impl BackfillContextFactory for MockContexts { + fn context(&self, bucket: &str) -> Option> { + self.0 + .lock() + .get(bucket) + .map(|ctx| Arc::clone(ctx) as Arc) + } + + fn buckets(&self) -> Vec { + self.0.lock().keys().cloned().collect() + } + } + + async fn runner_with( + node: &str, + bucket: &str, + context: Arc, + ) -> (Vec, Arc, Arc) { + let (dirs, store) = isolated_store_over_temp_disks().await; + // The isolated store has no bucket metadata system; the checkpoint + // only needs the bucket's directory under the metadata volume. + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(RUSTFS_META_BUCKET).join(BUCKET_META_PREFIX).join(bucket)) + .expect("test bucket metadata directory"); + } + let runner = runner_on(node, bucket, context, Arc::clone(&store)); + (dirs, store, runner) + } + + fn runner_on(node: &str, bucket: &str, context: Arc, store: Arc) -> Arc { + let contexts = MockContexts(Mutex::new(HashMap::from([(bucket.to_string(), context)]))); + BackfillRunner::new(store, node, Arc::new(contexts)) + } + + #[tokio::test] + async fn full_backfill_lists_pages_and_counts_every_key() { + let bucket = "backfill-full"; + let context = MockContext::new(2500, 1000); + let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await; + + let started = runner.start(bucket, BackfillRequest::default()).await.expect("start"); + assert_eq!(started.state, BackfillState::Running); + runner.wait_until_idle(bucket).await; + + let stored = read_checkpoint(&store, bucket).await.expect("read").expect("checkpoint"); + let cp = stored.checkpoint; + assert_eq!(cp.state, BackfillState::Completed); + assert_eq!((cp.listed, cp.enqueued, cp.pulled, cp.failed), (2500, 2500, 2500, 0)); + assert_eq!(cp.bytes, 25_000); + assert_eq!(cp.continuation_token, None); + assert_eq!(cp.last_key.as_deref(), Some("k/02499")); + assert_eq!(context.list_requests.lock().len(), 3, "2500 keys at 1000 per page is three source lists"); + assert_eq!(context.enqueued_keys().len(), 2500); + assert!(!runner.is_running_locally(bucket)); + let status = runner.status(bucket).await.expect("status").expect("present"); + assert_eq!(status.state, BackfillState::Completed); + } + + #[tokio::test] + async fn dry_run_lists_without_enqueueing_and_prefix_scopes_the_listing() { + let bucket = "backfill-dry"; + let context = MockContext::new(300, 1000); + let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await; + runner + .start( + bucket, + BackfillRequest { + prefix: Some("k/001".to_string()), + skip_existing: SkipExisting::EtagOrSize, + dry_run: true, + }, + ) + .await + .expect("start"); + runner.wait_until_idle(bucket).await; + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::Completed); + assert!(cp.dry_run); + assert_eq!(cp.listed, 100, "k/00100..k/00199"); + assert_eq!((cp.enqueued, cp.pulled), (0, 0)); + assert!(context.enqueued_keys().is_empty(), "a dry run never writes back"); + } + + #[tokio::test] + async fn skip_existing_policies_decide_what_is_re_pulled() { + let bucket = "backfill-skip"; + let context = MockContext::new(4, 1000); + context.local.lock().insert( + "k/00000".to_string(), + LocalBackfillObject { + size: 10, + source_etag: Some("etag-0".to_string()), + }, + ); + context.local.lock().insert( + "k/00001".to_string(), + LocalBackfillObject { + size: 11, + source_etag: Some("stale".to_string()), + }, + ); + context.local.lock().insert( + "k/00002".to_string(), + LocalBackfillObject { + size: 5, + source_etag: Some("etag-2".to_string()), + }, + ); + let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await; + + runner.start(bucket, BackfillRequest::default()).await.expect("start"); + runner.wait_until_idle(bucket).await; + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.skipped_existing, 3, "always: every existing key is skipped"); + assert_eq!(context.enqueued_keys(), vec!["k/00003".to_string()]); + + context.enqueued.lock().clear(); + runner + .start( + bucket, + BackfillRequest { + skip_existing: SkipExisting::EtagOrSize, + ..Default::default() + }, + ) + .await + .expect("second start after completion"); + runner.wait_until_idle(bucket).await; + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.skipped_existing, 1, "only the matching etag+size key is skipped"); + assert_eq!( + context.enqueued_keys(), + vec!["k/00001".to_string(), "k/00002".to_string(), "k/00003".to_string()], + "etag mismatch and size mismatch are re-pulled" + ); + } + + #[tokio::test] + async fn failed_pulls_are_counted_hashed_and_finish_with_failures() { + let bucket = "backfill-failed"; + let mut context = MockContext::new(5, 1000); + Arc::get_mut(&mut context) + .expect("unshared") + .fail_keys + .insert("k/00002".to_string()); + let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await; + runner.start(bucket, BackfillRequest::default()).await.expect("start"); + runner.wait_until_idle(bucket).await; + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::CompletedWithFailures); + assert_eq!((cp.pulled, cp.failed), (4, 1)); + assert_eq!(cp.failed_keys, vec![key_hash("k/00002")]); + let last = cp.last_error.expect("last error"); + assert_eq!(last.class, "local_write"); + assert_eq!(last.key_hash.as_deref(), Some(key_hash("k/00002").as_str())); + } + + #[tokio::test] + async fn listing_failure_marks_the_job_failed_with_the_error_class() { + let bucket = "backfill-list-error"; + let context = MockContext::new(5, 1000); + *context.list_error.lock() = Some(SourceError::AccessDenied); + let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await; + runner.start(bucket, BackfillRequest::default()).await.expect("start"); + runner.wait_until_idle(bucket).await; + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::Failed); + assert_eq!(cp.last_error.map(|e| e.class), Some("access_denied".to_string())); + } + + #[tokio::test] + async fn queue_full_waits_instead_of_dropping() { + let bucket = "backfill-queue-full"; + let mut context = MockContext::new(6, 1000); + { + let ctx = Arc::get_mut(&mut context).expect("unshared"); + ctx.queue_capacity = 2; + ctx.auto_complete = AtomicBool::new(false); + } + let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await; + runner.start(bucket, BackfillRequest::default()).await.expect("start"); + + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(context.enqueued_keys().len(), 2, "the job blocks on a full queue"); + assert!(runner.is_running_locally(bucket)); + for _ in 0..3 { + context.complete_pending(); + tokio::time::sleep(Duration::from_millis(300)).await; + } + runner.wait_until_idle(bucket).await; + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::Completed); + assert_eq!((cp.enqueued, cp.pulled), (6, 6), "every key was queued exactly once"); + } + + #[tokio::test] + async fn second_start_is_rejected_while_running_and_cancel_stops_enqueueing() { + let bucket = "backfill-cancel"; + let mut context = MockContext::new(50, 1000); + { + let ctx = Arc::get_mut(&mut context).expect("unshared"); + ctx.queue_capacity = 3; + ctx.auto_complete = AtomicBool::new(false); + } + let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await; + let first = runner.start(bucket, BackfillRequest::default()).await.expect("start"); + tokio::time::sleep(Duration::from_millis(200)).await; + + match runner.start(bucket, BackfillRequest::default()).await { + Err(BackfillError::AlreadyRunning { job_id, owner, .. }) => { + assert_eq!(job_id, first.job_id); + assert_eq!(owner, "node-a"); + } + other => panic!("expected AlreadyRunning, got {other:?}"), + } + + let cancelled = runner.cancel(bucket).await.expect("cancel"); + assert_eq!(cancelled.state, BackfillState::Cancelled); + assert_eq!(cancelled.job_id, first.job_id); + let enqueued_at_cancel = context.enqueued_keys().len(); + assert!(enqueued_at_cancel <= 3); + context.complete_pending(); + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(context.enqueued_keys().len(), enqueued_at_cancel, "nothing is queued after cancel"); + assert!(!runner.is_running_locally(bucket)); + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::Cancelled); + assert_eq!(runner.cancel(bucket).await.expect("idempotent cancel").state, BackfillState::Cancelled); + + assert!(matches!( + runner.cancel("never-started").await, + Err(BackfillError::NotFound(_)) | Err(BackfillError::Storage(_)) + )); + } + + #[tokio::test] + async fn config_change_cancels_a_running_job() { + let bucket = "backfill-config-change"; + let mut context = MockContext::new(50, 1000); + { + let ctx = Arc::get_mut(&mut context).expect("unshared"); + ctx.queue_capacity = 2; + ctx.auto_complete = AtomicBool::new(false); + } + let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await; + runner.start(bucket, BackfillRequest::default()).await.expect("start"); + tokio::time::sleep(Duration::from_millis(200)).await; + context.cancel.cancel(); + runner.wait_until_idle(bucket).await; + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::Cancelled); + assert_eq!(cp.last_error.map(|e| e.class), Some("config_changed".to_string())); + } + + #[tokio::test] + async fn concurrent_starts_admit_exactly_one_job() { + let bucket = "backfill-race"; + let mut context = MockContext::new(20, 1000); + { + let ctx = Arc::get_mut(&mut context).expect("unshared"); + ctx.queue_capacity = 1; + ctx.auto_complete = AtomicBool::new(false); + } + let (_dirs, store, runner_a) = runner_with("node-a", bucket, Arc::clone(&context)).await; + let runner_b = runner_on("node-b", bucket, Arc::clone(&context), Arc::clone(&store)); + + let (a, b) = tokio::join!( + runner_a.start(bucket, BackfillRequest::default()), + runner_b.start(bucket, BackfillRequest::default()) + ); + let rejected = match (a, b) { + (Ok(_), Err(rejected)) | (Err(rejected), Ok(_)) => rejected, + (a, b) => panic!("exactly one node must win: {a:?} / {b:?}"), + }; + assert!( + matches!(rejected, BackfillError::AlreadyRunning { .. }), + "the loser sees the winner's lease: {rejected:?}" + ); + context.cancel.cancel(); + runner_a.wait_until_idle(bucket).await; + runner_b.wait_until_idle(bucket).await; + } + + #[tokio::test] + async fn recovery_takes_over_an_expired_lease_and_resumes_from_the_cursor() { + let bucket = "backfill-takeover"; + let context = MockContext::new(2500, 1000); + let (_dirs, store, runner) = runner_with("node-b", bucket, Arc::clone(&context)).await; + + // A crashed node-a left a running checkpoint: one page done, lease expired. + let now = OffsetDateTime::now_utc(); + let mut crashed = + BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", now - Duration::from_secs(300)); + crashed.continuation_token = Some("1000".to_string()); + crashed.listed = 1000; + crashed.enqueued = 1000; + crashed.pulled = 1000; + crashed.owner = Some(BackfillOwner { + node: "node-a".to_string(), + lease_until: now - Duration::from_secs(120), + }); + let etag = write_checkpoint(&store, bucket, &crashed, None) + .await + .expect("seed checkpoint"); + + // A live lease is left alone. + let mut live = crashed.clone(); + live.owner = Some(BackfillOwner { + node: "node-a".to_string(), + lease_until: now + Duration::from_secs(60), + }); + live.updated_at = now; + let etag = write_checkpoint(&store, bucket, &live, Some(&etag)) + .await + .expect("live lease"); + assert_eq!(runner.recover_once().await.taken_over, 0, "unexpired lease must not be taken over"); + assert!(!runner.is_running_locally(bucket)); + match runner.start(bucket, BackfillRequest::default()).await { + Err(BackfillError::AlreadyRunning { owner, .. }) => assert_eq!(owner, "node-a"), + other => panic!("live lease must reject start, got {other:?}"), + } + + // Expire it again and recover. + let mut expired = live.clone(); + expired.owner = Some(BackfillOwner { + node: "node-a".to_string(), + lease_until: now - Duration::from_secs(1), + }); + expired.updated_at = now + Duration::from_millis(1); + write_checkpoint(&store, bucket, &expired, Some(&etag)) + .await + .expect("expire lease"); + let stats = runner.recover_once().await; + assert_eq!(stats.taken_over, 1); + runner.wait_until_idle(bucket).await; + + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::Completed); + assert_eq!(cp.job_id, crashed.job_id, "the same job continues"); + assert_eq!(cp.owner.as_ref().map(|o| o.node.as_str()), Some("node-b")); + assert_eq!(cp.listed, 2500, "1000 from before the crash plus 1500 resumed"); + assert_eq!( + context.list_requests.lock().as_slice(), + &[Some("1000".to_string()), Some("2000".to_string())], + "the resumed job lists from the persisted cursor, never from the start" + ); + assert_eq!(runner.recover_once().await.taken_over, 0, "a finished job is not recovered"); + } + + #[tokio::test] + async fn recovery_cancels_a_job_whose_config_changed_and_reclaims_own_node_jobs() { + let bucket = "backfill-recovery-config"; + let context = MockContext::new(10, 1000); + let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await; + let now = OffsetDateTime::now_utc(); + + // Same node name, unexpired lease: only a restart can produce this. + let own = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", now); + let etag = write_checkpoint(&store, bucket, &own, None).await.expect("seed"); + assert_eq!(runner.recover_once().await.taken_over, 1, "own-node running job is reclaimed at once"); + runner.wait_until_idle(bucket).await; + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::Completed); + let _ = etag; + + // Config changed since the (expired) job started: cancelled, not resumed. + let stored = read_checkpoint(&store, bucket).await.expect("read").expect("checkpoint"); + let mut stale = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_600_000_000), "node-z", now); + stale.owner = Some(BackfillOwner { + node: "node-z".to_string(), + lease_until: now - Duration::from_secs(1), + }); + write_checkpoint(&store, bucket, &stale, Some(&stored.etag)) + .await + .expect("seed stale"); + let stats = runner.recover_once().await; + assert_eq!((stats.taken_over, stats.cancelled), (0, 1)); + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::Cancelled); + assert_eq!(cp.last_error.map(|e| e.class), Some("config_changed".to_string())); + assert!(!runner.is_running_locally(bucket)); + } + + #[tokio::test] + async fn remote_cancel_stops_the_owner_at_its_next_save() { + let bucket = "backfill-remote-cancel"; + let mut context = MockContext::new(3000, 1000); + { + let ctx = Arc::get_mut(&mut context).expect("unshared"); + ctx.queue_capacity = 4; + ctx.auto_complete = AtomicBool::new(false); + } + let (_dirs, store, owner) = runner_with("node-a", bucket, Arc::clone(&context)).await; + let other = runner_on("node-b", bucket, Arc::clone(&context), Arc::clone(&store)); + owner.start(bucket, BackfillRequest::default()).await.expect("start"); + tokio::time::sleep(Duration::from_millis(200)).await; + + let cancelled = other.cancel(bucket).await.expect("remote cancel"); + assert_eq!(cancelled.state, BackfillState::Cancelled); + // Let the owner's queue drain so it reaches a save and observes the conflict. + for _ in 0..1000 { + context.complete_pending(); + if !owner.is_running_locally(bucket) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + owner.wait_until_idle(bucket).await; + let cp = read_checkpoint(&store, bucket) + .await + .expect("read") + .expect("checkpoint") + .checkpoint; + assert_eq!(cp.state, BackfillState::Cancelled, "the owner must not overwrite the remote cancel"); + let keys: BTreeSet = context.enqueued_keys().into_iter().collect(); + assert!(keys.len() < 3000, "the owner stopped before the listing ended"); + } +} diff --git a/crates/ecstore/src/bucket/on_demand_migration/mod.rs b/crates/ecstore/src/bucket/on_demand_migration/mod.rs index 78d92c038..e6c53bebb 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/mod.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/mod.rs @@ -20,6 +20,7 @@ //! clients guarded by a breaker, a negative cache, singleflight and a pull //! concurrency limit (rustfs/backlog#2147). +pub mod backfill; pub mod breaker; pub mod config; pub mod negative_cache; @@ -40,8 +41,8 @@ pub use config::{ pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache}; pub use pull::{ EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS, PullCompletion, - PullQueue, PullReason, PullSource, SourceBody, WriteBackBody, WriteBackError, WriteBackOutcome, WriteBackPart, - WriteBackRequest, commit_inline, commit_inline_with, + PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, WriteBackBody, WriteBackError, WriteBackOutcome, + WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with, }; pub use stats::{ GaugeGuard, LastSourceError, LatencyBucketSnapshot, OdmOp, OdmOutcome, OdmStats, OdmStatsSnapshot, PullFailureReason, diff --git a/crates/ecstore/src/bucket/on_demand_migration/pull.rs b/crates/ecstore/src/bucket/on_demand_migration/pull.rs index 93caa572b..05c8ab542 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/pull.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/pull.rs @@ -40,6 +40,7 @@ //! oversized body therefore fails the local write before it can commit, //! independently of the digest check the write-back performs. +use super::backfill::PullPriority; use super::source_client::{SourceClient, SourceError, SourceHead}; use super::stats::{PullFailureReason, PullPath}; use super::sys::{BucketOdmState, OnDemandMigrationSys, PullError, PullOutcome, PullSlot}; @@ -57,7 +58,7 @@ use std::task::{Context, Poll}; use std::time::{Duration, Instant}; use time::OffsetDateTime; use tokio::sync::mpsc::{self, error::TrySendError}; -use tokio::sync::watch; +use tokio::sync::{oneshot, watch}; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::{debug, trace}; @@ -108,6 +109,27 @@ impl PullReason { PullReason::Backfill => PullPath::Backfill, } } + + /// Backfill pulls yield pull permits to online requests. + pub fn priority(self) -> PullPriority { + match self { + PullReason::RangeGet | PullReason::LargeObject => PullPriority::Online, + PullReason::Backfill => PullPriority::Backfill, + } + } +} + +/// How one queued pull ended, delivered to the requester that asked for a +/// report (the backfill job counts these). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum QueuedPullOutcome { + /// A new local object of `size` bytes was written. + Stored { + size: u64, + }, + /// A current local version already existed; nothing was pulled. + AlreadyPresent, + Failed(PullError), } /// Result of [`PullQueue::enqueue`]. @@ -731,6 +753,8 @@ pub async fn commit_inline_with( struct PullJob { key: String, reason: PullReason, + /// Dropped without a send when the job is cancelled before it runs. + report: Option>, } /// Bounded per-bucket queue of background pulls. Keys are unique while @@ -809,26 +833,39 @@ impl PullQueue { } pub fn enqueue(&self, key: &str, reason: PullReason) -> EnqueueOutcome { + self.enqueue_with_report(key, reason).0 + } + + /// [`Self::enqueue`] that also hands back the job's report channel when + /// a new job was queued (`Coalesced` pulls report to their first + /// requester only). + pub fn enqueue_with_report( + &self, + key: &str, + reason: PullReason, + ) -> (EnqueueOutcome, Option>) { if self.cancel.is_cancelled() { - return EnqueueOutcome::Unavailable; + return (EnqueueOutcome::Unavailable, None); } let mut pending = self.pending.lock(); if pending.contains(key) { - return EnqueueOutcome::Coalesced; + return (EnqueueOutcome::Coalesced, None); } + let (report_tx, report_rx) = oneshot::channel(); match self.tx.try_send(PullJob { key: key.to_string(), reason, + report: Some(report_tx), }) { Ok(()) => { pending.insert(key.to_string()); - EnqueueOutcome::Enqueued + (EnqueueOutcome::Enqueued, Some(report_rx)) } Err(TrySendError::Full(_)) => { self.stats.record_pull_failure(PullFailureReason::QueueFull); - EnqueueOutcome::QueueFull + (EnqueueOutcome::QueueFull, None) } - Err(TrySendError::Closed(_)) => EnqueueOutcome::Unavailable, + Err(TrySendError::Closed(_)) => (EnqueueOutcome::Unavailable, None), } } } @@ -860,7 +897,7 @@ async fn dispatch( queue: Arc::clone(&queue), key: job.key.clone(), }; - let slot = match state.acquire_pull_slot(&job.key).await { + let slot = match state.acquire_pull_slot_with_priority(&job.key, job.reason.priority()).await { Ok(slot) => slot, Err(err) => { let result: Result = Err(err); @@ -905,11 +942,14 @@ async fn run_job( ) { let _pending = pending; let cancel = state.cancel_token(); - match slot { + let report = match slot { PullSlot::Follower(follower) => { // Someone else (inline GET or an earlier job) is pulling the key; // its result makes this job redundant. - let _ = follower.wait().await; + match follower.wait().await { + Ok(outcome) => QueuedPullOutcome::Stored { size: outcome.size }, + Err(err) => QueuedPullOutcome::Failed(err), + } } PullSlot::Leader(leader) => { let ctx = PullContext { @@ -921,8 +961,17 @@ async fn run_job( }; let result = pull_object(&ctx).await; record_completion(&state, &job.key, job.reason.path(), &result); + let report = match &result { + Ok(PullCompletion::Stored(outcome)) => QueuedPullOutcome::Stored { size: outcome.size }, + Ok(PullCompletion::AlreadyPresent(_)) => QueuedPullOutcome::AlreadyPresent, + Err(err) => QueuedPullOutcome::Failed(err.clone()), + }; leader.complete(result.map(|completion| completion.outcome())); + report } + }; + if let Some(tx) = job.report { + let _ = tx.send(report); } } @@ -948,9 +997,18 @@ impl BucketOdmState { /// Queues a background pull of `key`; see [`EnqueueOutcome`]. pub fn enqueue_pull(self: &Arc, key: &str, reason: PullReason) -> EnqueueOutcome { + self.enqueue_pull_with_report(key, reason).0 + } + + /// [`Self::enqueue_pull`] with the job's report channel. + pub fn enqueue_pull_with_report( + self: &Arc, + key: &str, + reason: PullReason, + ) -> (EnqueueOutcome, Option>) { match self.pull_queue() { - Some(queue) => queue.enqueue(key, reason), - None => EnqueueOutcome::Unavailable, + Some(queue) => queue.enqueue_with_report(key, reason), + None => (EnqueueOutcome::Unavailable, None), } } } diff --git a/crates/ecstore/src/bucket/on_demand_migration/sys.rs b/crates/ecstore/src/bucket/on_demand_migration/sys.rs index 3e5ae9cc5..cf68f2025 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/sys.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/sys.rs @@ -39,6 +39,7 @@ //! (`pull.rs`) stores objects with; each bucket state captures it at build //! time together with its lazily started [`PullQueue`]. +use super::backfill::{PriorityPullPermits, PullPermit, PullPriority}; use super::breaker::{Breaker, BreakerState, BreakerTransition, BreakerVerdict}; use super::config::{ ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig, PathStyle as ConfigPathStyle, Provider, SourceConfig, @@ -57,7 +58,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::Duration; use time::OffsetDateTime; -use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch}; +use tokio::sync::watch; use tokio::time::Instant; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; @@ -182,7 +183,7 @@ pub struct PullLeader { state: Arc, key: String, tx: watch::Sender>, - _permit: OwnedSemaphorePermit, + _permit: PullPermit, _inflight: GaugeGuard, completed: bool, } @@ -269,7 +270,8 @@ pub struct BucketOdmState { breaker: Breaker, negative_cache: NegativeCache, inflight: Mutex>>>, - pull_semaphore: Arc, + /// Online misses first, backfill pulls when nobody waits (ODM-12). + pull_permits: Arc, stats: Arc, cancel: CancellationToken, last_source_error_logged_at: Mutex>, @@ -318,7 +320,7 @@ impl BucketOdmState { breaker: Breaker::new(), negative_cache: NegativeCache::new(Duration::from_secs(policy.negative_cache_ttl_secs)), inflight: Mutex::new(HashMap::new()), - pull_semaphore: Arc::new(Semaphore::new(policy.max_concurrent_pulls.max(1) as usize)), + pull_permits: PriorityPullPermits::new(policy.max_concurrent_pulls.max(1) as usize), stats, cancel: CancellationToken::new(), last_source_error_logged_at: Mutex::new(None), @@ -477,6 +479,16 @@ impl BucketOdmState { /// later callers for the same key become followers and never touch the /// semaphore. Fails with `Canceled` when the state is torn down. pub async fn acquire_pull_slot(self: &Arc, key: &str) -> Result { + self.acquire_pull_slot_with_priority(key, PullPriority::Online).await + } + + /// [`Self::acquire_pull_slot`] at the given permit priority; the pull + /// queue passes `Backfill` for backfill jobs. + pub async fn acquire_pull_slot_with_priority( + self: &Arc, + key: &str, + priority: PullPriority, + ) -> Result { if self.cancel.is_cancelled() { return Err(PullError::canceled("bucket on-demand migration state was removed")); } @@ -499,7 +511,7 @@ impl BucketOdmState { let permit = { let _queued = self.stats.queue_guard(); tokio::select! { - permit = Arc::clone(&self.pull_semaphore).acquire_owned() => permit, + permit = self.pull_permits.acquire(priority) => permit, _ = self.cancel.cancelled() => { return Err(PullError::canceled("bucket on-demand migration state was removed")); } @@ -524,6 +536,10 @@ impl BucketOdmState { self.inflight.lock().len() } + pub fn pull_permits(&self) -> &Arc { + &self.pull_permits + } + pub fn snapshot(&self) -> OdmBucketSnapshot { OdmBucketSnapshot { bucket: self.bucket.clone(), diff --git a/crates/madmin/fixtures/on_demand_migration/backfill_job.json b/crates/madmin/fixtures/on_demand_migration/backfill_job.json new file mode 100644 index 000000000..b087ce776 --- /dev/null +++ b/crates/madmin/fixtures/on_demand_migration/backfill_job.json @@ -0,0 +1 @@ +{"bucket":"photos","job":{"format_version":1,"job_id":"11111111-1111-4111-8111-111111111111","state":"running","config_updated_at":"2026-09-02T10:00:00Z","prefix":"photos/","skip_existing":"always","dry_run":false,"continuation_token":"cGhvdG9zLzEwMDA=","listed":2000,"enqueued":1500,"pulled":1400,"skipped_existing":500,"failed":3,"bytes":73400320,"last_key":"photos/2024/02.jpg","last_error":{"class":"source_timeout","key_hash":"9f2c3b0a1d4e5f60","at":"2026-09-02T10:05:00Z"},"failed_keys":["9f2c3b0a1d4e5f60"],"started_at":"2026-09-02T10:00:30Z","updated_at":"2026-09-02T10:05:10Z","owner":{"node":"node-a:9000","lease_until":"2026-09-02T10:06:10Z"}}} diff --git a/crates/madmin/fixtures/on_demand_migration/status_with_backfill.json b/crates/madmin/fixtures/on_demand_migration/status_with_backfill.json new file mode 100644 index 000000000..ec14979bd --- /dev/null +++ b/crates/madmin/fixtures/on_demand_migration/status_with_backfill.json @@ -0,0 +1 @@ +{"configured":true,"enabled":true,"module_enabled":true,"provider":"minio","endpoint_host":"source.example.com","breaker":{"state":"half_open","opened_at":null},"counters":{"requests_total":{"get":{"breaker_open":0,"filtered":0,"negative_cached":0,"source_error":0,"source_hit":2,"source_miss":0,"unsupported":0},"head":{"breaker_open":0,"filtered":0,"negative_cached":1,"source_error":0,"source_hit":0,"source_miss":0,"unsupported":0}},"pulled_bytes_total":4096,"pulled_objects_total":{"backfill":0,"background":0,"inline":1},"pull_failures_total":{"canceled":0,"etag_mismatch":0,"local_write":0,"queue_full":0,"source_access_denied":0,"source_connect":0,"source_not_found":0,"source_other":0,"source_server_error":0,"source_throttled":0,"source_timeout":1,"source_unsupported":0},"source_latency":{"buckets":[{"le_ms":5,"count":1},{"le_ms":10,"count":1},{"le_ms":20,"count":1},{"le_ms":50,"count":1},{"le_ms":100,"count":1},{"le_ms":200,"count":1},{"le_ms":500,"count":1},{"le_ms":1000,"count":2},{"le_ms":2000,"count":2},{"le_ms":5000,"count":2},{"le_ms":10000,"count":2},{"le_ms":20000,"count":2},{"le_ms":30000,"count":2},{"le_ms":60000,"count":2}],"count":3,"sum_ms":90753}},"last_source_error":{"class":"server_error","at":"2026-09-02T10:00:00Z"},"inflight_pulls":1,"queue_depth":1,"served_by_source_ratio":null,"updated_at":"2026-09-02T10:00:00Z","backfill":{"job_id":"11111111-1111-4111-8111-111111111111","state":"running","listed":2000,"enqueued":1500,"pulled":1400,"skipped_existing":500,"failed":3,"bytes":73400320,"updated_at":"2026-09-02T10:05:10Z"}} diff --git a/crates/madmin/src/client.rs b/crates/madmin/src/client.rs index 6662946b9..83f6c33d7 100644 --- a/crates/madmin/src/client.rs +++ b/crates/madmin/src/client.rs @@ -400,7 +400,7 @@ impl AdminClient { } /// Signed POST returning a decoded JSON body. - async fn post_json Deserialize<'de>>( + pub(crate) async fn post_json Deserialize<'de>>( &self, path: &str, query: &[(&str, String)], diff --git a/crates/madmin/src/on_demand_migration.rs b/crates/madmin/src/on_demand_migration.rs index 3d213098a..52f5380a4 100644 --- a/crates/madmin/src/on_demand_migration.rs +++ b/crates/madmin/src/on_demand_migration.rs @@ -14,8 +14,9 @@ //! On-Demand Migration admin API contract (ODM-07, rustfs/backlog#2154). //! -//! Wire types for `PUT`/`GET`/`DELETE /v3/on-demand-migration/{bucket}` and -//! `GET .../status`, mirroring the server's config model +//! Wire types for `PUT`/`GET`/`DELETE /v3/on-demand-migration/{bucket}`, +//! `GET .../status`, `POST .../backfill?op=start|cancel` and +//! `GET .../backfill` (ODM-12), mirroring the server's config model //! (`crates/ecstore/src/bucket/on_demand_migration/config.rs`) and handler //! responses (`rustfs/src/admin/handlers/on_demand_migration.rs`). The SDK //! owns its own copies, madmin-go style; the fixtures under @@ -34,6 +35,8 @@ pub const ON_DEMAND_MIGRATION_CONFIG_VERSION: u32 = 1; /// Query flag that validates and probes a config without saving it. const DRY_RUN_QUERY: &str = "dry-run"; +/// `POST .../backfill?op=` selector. +const BACKFILL_OP_QUERY: &str = "op"; /// Bucket-level on-demand migration configuration (request body of the /// `PUT`, redacted copy in every response). @@ -300,7 +303,7 @@ pub struct OnDemandMigrationGetResponse { /// snapshot of the bucket. The runtime fields are `null` while the bucket has /// no live state on the answering node (module off, config absent or /// disabled); `provider` and `endpoint_host` then still describe the saved -/// config, if any. +/// config, if any. `backfill` is present once the bucket had a backfill job. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OnDemandMigrationStatus { pub configured: bool, @@ -328,6 +331,10 @@ pub struct OnDemandMigrationStatus { /// RFC 3339 save time of the config; `None` when not configured. #[serde(default)] pub updated_at: Option, + /// Counters of the bucket's latest backfill job; absent until the bucket + /// has had one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backfill: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -385,6 +392,124 @@ pub struct OnDemandMigrationSourceError { pub at: String, } +/// Lifecycle of a backfill job. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OnDemandMigrationBackfillState { + Pending, + Running, + Paused, + Cancelled, + Completed, + CompletedWithFailures, + Failed, +} + +impl OnDemandMigrationBackfillState { + /// Whether the job still runs (or is about to). + pub fn is_active(self) -> bool { + matches!(self, Self::Pending | Self::Running) + } +} + +/// What to do with a listed key that already exists locally. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OnDemandMigrationSkipExisting { + #[default] + Always, + EtagOrSize, +} + +/// Body of `POST .../backfill?op=start`; every field is optional. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct OnDemandMigrationBackfillRequest { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prefix: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub skip_existing: Option, + #[serde(default)] + pub dry_run: bool, +} + +/// Last failure recorded by a backfill job; `key_hash` is a hash, never the key. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OnDemandMigrationBackfillError { + pub class: String, + #[serde(default)] + pub key_hash: Option, + pub at: String, +} + +/// Node running a backfill job and the lease it holds (RFC 3339). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OnDemandMigrationBackfillOwner { + pub node: String, + pub lease_until: String, +} + +/// The backfill checkpoint as the server stores it. Field order is the +/// on-disk and on-wire contract; unknown fields from newer servers are +/// tolerated. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OnDemandMigrationBackfillJob { + pub format_version: u32, + pub job_id: String, + pub state: OnDemandMigrationBackfillState, + pub config_updated_at: String, + #[serde(default)] + pub prefix: Option, + #[serde(default)] + pub skip_existing: OnDemandMigrationSkipExisting, + #[serde(default)] + pub dry_run: bool, + #[serde(default)] + pub continuation_token: Option, + #[serde(default)] + pub listed: u64, + #[serde(default)] + pub enqueued: u64, + #[serde(default)] + pub pulled: u64, + #[serde(default)] + pub skipped_existing: u64, + #[serde(default)] + pub failed: u64, + #[serde(default)] + pub bytes: u64, + #[serde(default)] + pub last_key: Option, + #[serde(default)] + pub last_error: Option, + #[serde(default)] + pub failed_keys: Vec, + pub started_at: String, + pub updated_at: String, + #[serde(default)] + pub owner: Option, +} + +/// `POST`/`GET .../backfill` response. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OnDemandMigrationBackfillResponse { + pub bucket: String, + pub job: OnDemandMigrationBackfillJob, +} + +/// Counters of the latest backfill job, embedded in the status response. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OnDemandMigrationBackfillSummary { + pub job_id: String, + pub state: OnDemandMigrationBackfillState, + pub listed: u64, + pub enqueued: u64, + pub pulled: u64, + pub skipped_existing: u64, + pub failed: u64, + pub bytes: u64, + pub updated_at: String, +} + fn config_path(bucket: &str) -> String { format!("/v3/on-demand-migration/{}", percent_encode_path_segment(bucket)) } @@ -428,6 +553,43 @@ impl AdminClient { pub async fn on_demand_migration_status(&self, bucket: &str) -> Result { self.get_json(&format!("{}/status", config_path(bucket))).await } + + /// Start the backfill job of `bucket`. A job that still holds its lease + /// answers HTTP 409 `OnDemandMigrationBackfillRunning`. + pub async fn start_on_demand_migration_backfill( + &self, + bucket: &str, + request: &OnDemandMigrationBackfillRequest, + ) -> Result { + let body = serde_json::to_vec(request).map_err(|err| AdminClientError::Decode { + message: err.to_string(), + })?; + self.post_json(&backfill_path(bucket), &[(BACKFILL_OP_QUERY, "start".to_string())], body) + .await + } + + /// Cancel the backfill job of `bucket`; idempotent on a finished job. A + /// bucket that never had a job answers HTTP 404 `NoSuchBackfillJob`. + pub async fn cancel_on_demand_migration_backfill( + &self, + bucket: &str, + ) -> Result { + self.post_json(&backfill_path(bucket), &[(BACKFILL_OP_QUERY, "cancel".to_string())], Vec::new()) + .await + } + + /// Read the backfill checkpoint of `bucket` (404 `NoSuchBackfillJob` + /// when none was ever started). + pub async fn on_demand_migration_backfill( + &self, + bucket: &str, + ) -> Result { + self.get_json(&backfill_path(bucket)).await + } +} + +fn backfill_path(bucket: &str) -> String { + format!("{}/backfill", config_path(bucket)) } #[cfg(test)] @@ -439,6 +601,8 @@ mod tests { const SET_RESPONSE_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/set_response.json"); const GET_RESPONSE_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/get_response.json"); const STATUS_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/status.json"); + const STATUS_WITH_BACKFILL_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/status_with_backfill.json"); + const BACKFILL_JOB_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/backfill_job.json"); fn round_trip Deserialize<'de>>(fixture: &str) -> T { let value: T = serde_json::from_str(fixture.trim()).expect("fixture decodes"); @@ -505,6 +669,7 @@ mod tests { assert_eq!(status.queue_depth, 1); assert_eq!(status.served_by_source_ratio, None, "the ratio is null, never a fabricated 0"); assert_eq!(status.updated_at.as_deref(), Some("2026-09-02T10:00:00Z")); + assert!(status.backfill.is_none(), "a bucket without a job carries no summary"); } #[test] @@ -521,6 +686,108 @@ mod tests { assert_eq!(status.updated_at, None); } + #[test] + fn backfill_fixtures_round_trip_byte_for_byte() { + let response: OnDemandMigrationBackfillResponse = round_trip(BACKFILL_JOB_FIXTURE); + assert_eq!(response.bucket, "photos"); + let job = &response.job; + assert_eq!(job.format_version, 1); + assert_eq!(job.state, OnDemandMigrationBackfillState::Running); + assert!(job.state.is_active()); + assert_eq!(job.skip_existing, OnDemandMigrationSkipExisting::Always); + assert_eq!(job.continuation_token.as_deref(), Some("cGhvdG9zLzEwMDA=")); + assert_eq!((job.listed, job.enqueued, job.pulled, job.failed), (2000, 1500, 1400, 3)); + assert_eq!(job.failed_keys, vec!["9f2c3b0a1d4e5f60".to_string()]); + assert_eq!(job.last_error.as_ref().map(|e| e.class.as_str()), Some("source_timeout")); + assert_eq!(job.owner.as_ref().map(|o| o.node.as_str()), Some("node-a:9000")); + + let status: OnDemandMigrationStatus = round_trip(STATUS_WITH_BACKFILL_FIXTURE); + let summary = status.backfill.expect("summary present"); + assert_eq!(summary.job_id, job.job_id); + assert_eq!(summary.state, OnDemandMigrationBackfillState::Running); + assert_eq!(summary.bytes, 73_400_320); + + // A newer server may add checkpoint fields; the client keeps decoding. + let newer = + BACKFILL_JOB_FIXTURE + .trim() + .replacen("\"listed\":2000", "\"listed\":2000,\"throttle_hint\":{\"mode\":\"soft\"}", 1); + let decoded: OnDemandMigrationBackfillResponse = serde_json::from_str(&newer).expect("unknown fields tolerated"); + assert_eq!(decoded.job.listed, 2000); + } + + #[test] + fn backfill_request_serializes_only_what_was_set() { + let minimal = serde_json::to_string(&OnDemandMigrationBackfillRequest::default()).expect("serialize"); + assert_eq!(minimal, r#"{"dry_run":false}"#); + let full = serde_json::to_string(&OnDemandMigrationBackfillRequest { + prefix: Some("photos/".to_string()), + skip_existing: Some(OnDemandMigrationSkipExisting::EtagOrSize), + dry_run: true, + }) + .expect("serialize"); + assert_eq!(full, r#"{"prefix":"photos/","skip_existing":"etag_or_size","dry_run":true}"#); + } + + #[tokio::test] + async fn backfill_start_cancel_and_get_use_the_registered_route() { + let server = TestServer::spawn(BACKFILL_JOB_FIXTURE, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + let response = client + .start_on_demand_migration_backfill( + "photos", + &OnDemandMigrationBackfillRequest { + prefix: Some("photos/".to_string()), + ..Default::default() + }, + ) + .await + .expect("start decodes"); + assert_eq!(response.job.state, OnDemandMigrationBackfillState::Running); + let request = server.recorded(); + assert_eq!(request.method, "POST"); + assert_eq!(request.path, "/rustfs/admin/v3/on-demand-migration/photos/backfill"); + assert_eq!(request.query, "op=start"); + assert_eq!(request.header("content-type").as_deref(), Some("application/json")); + assert_eq!(request.body, r#"{"prefix":"photos/","dry_run":false}"#); + + let server = TestServer::spawn(BACKFILL_JOB_FIXTURE, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + client + .cancel_on_demand_migration_backfill("photos") + .await + .expect("cancel decodes"); + let request = server.recorded(); + assert_eq!(request.method, "POST"); + assert_eq!(request.query, "op=cancel"); + assert_eq!(request.body, "", "cancel sends no body"); + + let server = TestServer::spawn(BACKFILL_JOB_FIXTURE, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + client.on_demand_migration_backfill("photos").await.expect("get decodes"); + let request = server.recorded(); + assert_eq!(request.method, "GET"); + assert_eq!(request.path, "/rustfs/admin/v3/on-demand-migration/photos/backfill"); + + let server = TestServer::spawn( + r#"{"code":"OnDemandMigrationBackfillRunning","message":"a backfill job is already running"}"#, + 409, + ) + .await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + match client + .start_on_demand_migration_backfill("photos", &OnDemandMigrationBackfillRequest::default()) + .await + .unwrap_err() + { + AdminClientError::HttpStatus { status, body } => { + assert_eq!(status, 409); + assert!(body.contains("OnDemandMigrationBackfillRunning")); + } + other => panic!("expected HttpStatus, got {other:?}"), + } + } + #[test] fn minimal_config_expands_to_the_server_defaults() { let config = OnDemandMigrationConfig::new(OnDemandMigrationSource { diff --git a/crates/obs/src/metrics/collectors/mod.rs b/crates/obs/src/metrics/collectors/mod.rs index bdb1d8210..e641540de 100644 --- a/crates/obs/src/metrics/collectors/mod.rs +++ b/crates/obs/src/metrics/collectors/mod.rs @@ -70,7 +70,8 @@ pub use notification::{NotificationStats, collect_notification_metrics}; pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics}; pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics}; pub use on_demand_migration::{ - OnDemandMigrationBreakerState, OnDemandMigrationBucketStats, collect_on_demand_migration_metrics, source_latency_le_label, + OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBreakerState, OnDemandMigrationBucketStats, + collect_on_demand_migration_backfill_metrics, collect_on_demand_migration_metrics, source_latency_le_label, }; pub use replication::{ReplicationMetricsSnapshot, collect_replication_metrics}; pub(crate) use replication::{ReplicationRuntimeStats, collect_replication_runtime_metrics}; diff --git a/crates/obs/src/metrics/collectors/on_demand_migration.rs b/crates/obs/src/metrics/collectors/on_demand_migration.rs index b854a0b3c..894a53877 100644 --- a/crates/obs/src/metrics/collectors/on_demand_migration.rs +++ b/crates/obs/src/metrics/collectors/on_demand_migration.rs @@ -21,10 +21,12 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::on_demand_migration::{ - BREAKER_STATE_CLOSED, BREAKER_STATE_HALF_OPEN, BREAKER_STATE_OPEN, BUCKET_L, LE_L, ODM_BREAKER_STATE_MD, - ODM_INFLIGHT_PULLS_MD, ODM_PULL_FAILURES_TOTAL_MD, ODM_PULLED_BYTES_TOTAL_MD, ODM_PULLED_OBJECTS_TOTAL_MD, - ODM_QUEUE_DEPTH_MD, ODM_REQUESTS_TOTAL_MD, ODM_SOURCE_LATENCY_SECONDS_COUNT_MD, ODM_SOURCE_LATENCY_SECONDS_DISTRIBUTION_MD, - ODM_SOURCE_LATENCY_SECONDS_SUM_MD, OP_L, OUTCOME_L, PATH_L, REASON_L, + BREAKER_STATE_CLOSED, BREAKER_STATE_HALF_OPEN, BREAKER_STATE_OPEN, BUCKET_L, LE_L, ODM_BACKFILL_BYTES_MD, + ODM_BACKFILL_ENQUEUED_MD, ODM_BACKFILL_FAILED_MD, ODM_BACKFILL_JOBS_MD, ODM_BACKFILL_LISTED_MD, ODM_BACKFILL_PULLED_MD, + ODM_BACKFILL_SKIPPED_EXISTING_MD, ODM_BREAKER_STATE_MD, ODM_INFLIGHT_PULLS_MD, ODM_PULL_FAILURES_TOTAL_MD, + ODM_PULLED_BYTES_TOTAL_MD, ODM_PULLED_OBJECTS_TOTAL_MD, ODM_QUEUE_DEPTH_MD, ODM_REQUESTS_TOTAL_MD, + ODM_SOURCE_LATENCY_SECONDS_COUNT_MD, ODM_SOURCE_LATENCY_SECONDS_DISTRIBUTION_MD, ODM_SOURCE_LATENCY_SECONDS_SUM_MD, OP_L, + OUTCOME_L, PATH_L, REASON_L, SERVER_L, STATE_L, }; use std::borrow::Cow; use std::collections::BTreeMap; @@ -170,6 +172,48 @@ pub fn collect_on_demand_migration_metrics(stats: &[OnDemandMigrationBucketStats metrics } +/// Counters of one bucket's latest backfill job (ODM-12, +/// rustfs/backlog#2159). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct OdmBackfillBucketStats { + pub bucket: String, + /// Checkpoint state label (`running`, `completed`, ...). + pub state: String, + pub listed: u64, + pub enqueued: u64, + pub pulled: u64, + pub skipped_existing: u64, + pub failed: u64, + pub bytes: u64, +} + +/// Backfill stats of every bucket with a checkpoint, labelled by node. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct OdmBackfillRuntimeStats { + pub server: String, + pub buckets: Vec, +} + +/// Seven series per bucket: the state gauge and six counters. +pub fn collect_on_demand_migration_backfill_metrics(stats: &OdmBackfillRuntimeStats) -> Vec { + let mut metrics = Vec::with_capacity(stats.buckets.len() * 7); + for bucket in &stats.buckets { + let labelled = |descriptor: &'static std::sync::LazyLock, value: u64| { + PrometheusMetric::from_descriptor(descriptor, value as f64) + .with_label_owned(SERVER_L, stats.server.clone()) + .with_label_owned(BUCKET_L, bucket.bucket.clone()) + }; + metrics.push(labelled(&ODM_BACKFILL_JOBS_MD, 1).with_label_owned(STATE_L, bucket.state.clone())); + metrics.push(labelled(&ODM_BACKFILL_LISTED_MD, bucket.listed)); + metrics.push(labelled(&ODM_BACKFILL_ENQUEUED_MD, bucket.enqueued)); + metrics.push(labelled(&ODM_BACKFILL_PULLED_MD, bucket.pulled)); + metrics.push(labelled(&ODM_BACKFILL_SKIPPED_EXISTING_MD, bucket.skipped_existing)); + metrics.push(labelled(&ODM_BACKFILL_FAILED_MD, bucket.failed)); + metrics.push(labelled(&ODM_BACKFILL_BYTES_MD, bucket.bytes)); + } + metrics +} + #[cfg(test)] pub(crate) mod tests { use super::*; @@ -354,4 +398,81 @@ pub(crate) mod tests { fn empty_snapshot_emits_nothing() { assert!(collect_on_demand_migration_metrics(&[]).is_empty()); } + + /// One running and one completed job, as the scheduler would see them + /// from the local backfill runner. + pub(crate) fn backfill_golden_stats(server: &str) -> OdmBackfillRuntimeStats { + OdmBackfillRuntimeStats { + server: server.to_string(), + buckets: vec![ + OdmBackfillBucketStats { + bucket: "photos".to_string(), + state: "running".to_string(), + listed: 2000, + enqueued: 1500, + pulled: 1400, + skipped_existing: 500, + failed: 3, + bytes: 73_400_320, + }, + OdmBackfillBucketStats { + bucket: "docs".to_string(), + state: "completed".to_string(), + ..Default::default() + }, + ], + } + } + + fn backfill_series<'a>(metrics: &'a [PrometheusMetric], name: &str, bucket: &str) -> Option<&'a PrometheusMetric> { + metrics.iter().find(|metric| { + metric.name == name + && metric + .labels + .iter() + .any(|(label, value)| *label == BUCKET_L && value.as_ref() == bucket) + }) + } + + #[test] + fn backfill_collects_seven_series_per_bucket_with_the_odm_subsystem_prefix() { + let stats = backfill_golden_stats("node1:9000"); + let metrics = collect_on_demand_migration_backfill_metrics(&stats); + assert_eq!(metrics.len(), 14); + assert!( + metrics + .iter() + .all(|metric| metric.name.starts_with("rustfs_on_demand_migration_backfill_")) + ); + assert!(metrics.iter().all(|metric| { + metric + .labels + .iter() + .any(|(label, value)| *label == SERVER_L && value.as_ref() == "node1:9000") + })); + + let jobs = backfill_series(&metrics, &ODM_BACKFILL_JOBS_MD.get_full_metric_name(), "photos").expect("jobs gauge"); + assert_eq!(jobs.value, 1.0); + assert!( + jobs.labels + .iter() + .any(|(label, value)| *label == STATE_L && value.as_ref() == "running") + ); + let listed = backfill_series(&metrics, &ODM_BACKFILL_LISTED_MD.get_full_metric_name(), "photos").expect("listed"); + assert_eq!(listed.value, 2000.0); + let bytes = backfill_series(&metrics, &ODM_BACKFILL_BYTES_MD.get_full_metric_name(), "photos").expect("bytes"); + assert_eq!(bytes.value, 73_400_320.0); + let docs_failed = backfill_series(&metrics, &ODM_BACKFILL_FAILED_MD.get_full_metric_name(), "docs").expect("docs failed"); + assert_eq!(docs_failed.value, 0.0); + assert_eq!( + ODM_BACKFILL_LISTED_MD.get_full_metric_name(), + "rustfs_on_demand_migration_backfill_listed_total" + ); + } + + #[test] + fn backfill_no_buckets_means_no_series() { + let metrics = collect_on_demand_migration_backfill_metrics(&OdmBackfillRuntimeStats::default()); + assert!(metrics.is_empty()); + } } diff --git a/crates/obs/src/metrics/mod.rs b/crates/obs/src/metrics/mod.rs index 529a66b15..aea11381d 100644 --- a/crates/obs/src/metrics/mod.rs +++ b/crates/obs/src/metrics/mod.rs @@ -34,6 +34,7 @@ pub(crate) use storage_api::metrics::{ BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore, StorageAdminApi, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled, - obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_on_demand_migration_snapshot, - obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, obs_transition_state_handle, + obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_on_demand_migration_backfill_snapshot, + obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, + obs_transition_state_handle, }; diff --git a/crates/obs/src/metrics/scheduler.rs b/crates/obs/src/metrics/scheduler.rs index 39a1a37fd..9ac05e79e 100644 --- a/crates/obs/src/metrics/scheduler.rs +++ b/crates/obs/src/metrics/scheduler.rs @@ -33,6 +33,7 @@ use crate::metrics::collectors::{ NotificationStats, NotificationTargetRuntimeStats, NotificationTargetStats, + OdmBackfillRuntimeStats, OnDemandMigrationBucketStats, // System monitoring collectors (migrated from rustfs-obs::system) ProcessAttributeError, @@ -65,6 +66,7 @@ use crate::metrics::collectors::{ collect_node_metrics, collect_notification_runtime_metrics, collect_notification_target_runtime_metrics, + collect_on_demand_migration_backfill_metrics, collect_on_demand_migration_metrics, collect_process_attributes, collect_process_cpu_metrics, @@ -121,12 +123,15 @@ use crate::metrics::schema::notification_target::{ TARGET_ID as NOTIFICATION_TARGET_ID_LABEL, TARGET_TYPE as NOTIFICATION_TARGET_TYPE_LABEL, }; use crate::metrics::schema::on_demand_migration::{ - BUCKET_L as ODM_BUCKET_L, LE_L as ODM_LE_L, ODM_BREAKER_STATE_MD, ODM_INFLIGHT_PULLS_MD, ODM_PULL_FAILURES_TOTAL_MD, + BACKFILL_STATES as ODM_BACKFILL_STATES, BUCKET_L as ODM_BUCKET_L, LE_L as ODM_LE_L, ODM_BACKFILL_BYTES_MD, + ODM_BACKFILL_ENQUEUED_MD, ODM_BACKFILL_FAILED_MD, ODM_BACKFILL_JOBS_MD, ODM_BACKFILL_LISTED_MD, ODM_BACKFILL_PULLED_MD, + ODM_BACKFILL_SKIPPED_EXISTING_MD, ODM_BREAKER_STATE_MD, ODM_INFLIGHT_PULLS_MD, ODM_PULL_FAILURES_TOTAL_MD, ODM_PULLED_BYTES_TOTAL_MD, ODM_PULLED_OBJECTS_TOTAL_MD, ODM_QUEUE_DEPTH_MD, ODM_REQUESTS_TOTAL_MD, ODM_SOURCE_LATENCY_SECONDS_COUNT_MD, ODM_SOURCE_LATENCY_SECONDS_DISTRIBUTION_MD, ODM_SOURCE_LATENCY_SECONDS_SUM_MD, OP_L as ODM_OP_L, OUTCOME_L as ODM_OUTCOME_L, PATH_L as ODM_PATH_L, PULL_FAILURE_REASONS as ODM_PULL_FAILURE_REASONS, PULL_PATHS as ODM_PULL_PATHS, REASON_L as ODM_REASON_L, REQUEST_OPS as ODM_REQUEST_OPS, - REQUEST_OUTCOMES as ODM_REQUEST_OUTCOMES, SOURCE_LATENCY_LE as ODM_SOURCE_LATENCY_LE, + REQUEST_OUTCOMES as ODM_REQUEST_OUTCOMES, SERVER_L as ODM_SERVER_L, SOURCE_LATENCY_LE as ODM_SOURCE_LATENCY_LE, + STATE_L as ODM_STATE_L, }; use crate::metrics::schema::scanner::{ BUCKET_LABEL as SCANNER_BUCKET_LABEL, CYCLE_SCOPE_LABEL as SCANNER_CYCLE_SCOPE_LABEL, DRIVE_LABEL as SCANNER_DRIVE_LABEL, @@ -144,9 +149,9 @@ use crate::metrics::stats_collector::{ collect_bucket_replication_stats_bundle, collect_bucket_stats, collect_cluster_and_health_stats, collect_cluster_config_stats, collect_cluster_usage_metric_stats, collect_compression_cluster_stats, collect_disk_and_system_drive_runtime_stats, collect_erasure_set_stats, collect_host_network_stats, collect_iam_stats, - collect_ilm_runtime_metric_stats, collect_internode_network_stats, collect_on_demand_migration_stats, - collect_process_metric_bundle_with, collect_replication_stats, collect_scanner_runtime_metric_stats, - collect_system_cpu_and_memory_stats_with, + collect_ilm_runtime_metric_stats, collect_internode_network_stats, collect_on_demand_migration_backfill_stats, + collect_on_demand_migration_stats, collect_process_metric_bundle_with, collect_replication_stats, + collect_scanner_runtime_metric_stats, collect_system_cpu_and_memory_stats_with, }; use crate::node_identity::{SERVER_LABEL, current_local_node_identity}; use crate::telemetry::retire_metric_series; @@ -914,6 +919,10 @@ fn on_demand_migration_bucket_live_keys(stats: &[OnDemandMigrationBucketStats]) stats.iter().map(|stat| stat.bucket.clone()).collect() } +fn on_demand_migration_backfill_bucket_live_keys(stats: &OdmBackfillRuntimeStats) -> HashSet { + stats.buckets.iter().map(|stat| stat.bucket.clone()).collect() +} + fn update_series_zero_tombstones( has_seen_valid_snapshot: &mut bool, prev_live_keys: &mut HashSet, @@ -1646,6 +1655,42 @@ fn retire_on_demand_migration_metric_series(bucket: &str) -> usize { .sum() } +/// Every on-demand migration backfill series one node's job can own. The +/// `state` gauge is enumerated over the checkpoint's fixed lifecycle values +/// because only one of them is emitted per cycle. +fn on_demand_migration_backfill_metric_series(server: &str, bucket: &str) -> Vec { + let job_labels = || { + vec![ + (ODM_SERVER_L, Cow::Owned(server.to_string())), + (ODM_BUCKET_L, Cow::Owned(bucket.to_string())), + ] + }; + let mut series = Vec::new(); + for state in ODM_BACKFILL_STATES { + let mut labels = job_labels(); + labels.push((ODM_STATE_L, Cow::Borrowed(state))); + series.push((ODM_BACKFILL_JOBS_MD.get_full_metric_name(), labels)); + } + for descriptor in [ + &*ODM_BACKFILL_LISTED_MD, + &*ODM_BACKFILL_ENQUEUED_MD, + &*ODM_BACKFILL_PULLED_MD, + &*ODM_BACKFILL_SKIPPED_EXISTING_MD, + &*ODM_BACKFILL_FAILED_MD, + &*ODM_BACKFILL_BYTES_MD, + ] { + series.push((descriptor.get_full_metric_name(), job_labels())); + } + series +} + +fn retire_on_demand_migration_backfill_metric_series(server: &str, bucket: &str) -> usize { + on_demand_migration_backfill_metric_series(server, bucket) + .iter() + .map(|(name, labels)| retire_metric_series(name, labels)) + .sum() +} + fn retire_repl_backlog_target_metric_series(bucket: &str, target_arn: &str) -> usize { let labels = [ (BUCKET_L, Cow::Owned(bucket.to_string())), @@ -2037,6 +2082,8 @@ pub fn init_metrics_runtime(token: CancellationToken) { let mut has_seen_proxy_bucket_snapshot = false; let mut prev_on_demand_migration_live_keys: HashSet = HashSet::new(); let mut has_seen_on_demand_migration_snapshot = false; + let mut prev_on_demand_migration_backfill_live_keys: HashSet = HashSet::new(); + let mut has_seen_on_demand_migration_backfill_snapshot = false; loop { tokio::select! { _ = interval.tick() => { @@ -2125,6 +2172,24 @@ pub fn init_metrics_runtime(token: CancellationToken) { prev_on_demand_migration_live_keys = current_on_demand_migration_live_keys; has_seen_on_demand_migration_snapshot = true; metrics.extend(collect_on_demand_migration_metrics(&on_demand_migration)); + // Backfill jobs come and go independently of the bucket's config, + // so their series are retired on their own key set. + let on_demand_migration_backfill = collect_on_demand_migration_backfill_stats(); + let current_on_demand_migration_backfill_live_keys = + on_demand_migration_backfill_bucket_live_keys(&on_demand_migration_backfill); + let retire_on_demand_migration_backfill_buckets = if has_seen_on_demand_migration_backfill_snapshot + { + prev_on_demand_migration_backfill_live_keys + .difference(¤t_on_demand_migration_backfill_live_keys) + .cloned() + .collect::>() + } else { + Vec::new() + }; + prev_on_demand_migration_backfill_live_keys = current_on_demand_migration_backfill_live_keys; + has_seen_on_demand_migration_backfill_snapshot = true; + let on_demand_migration_backfill_server = on_demand_migration_backfill.server.clone(); + metrics.extend(collect_on_demand_migration_backfill_metrics(&on_demand_migration_backfill)); let replication = collect_replication_stats().await; metrics.extend(collect_replication_runtime_metrics(&ReplicationRuntimeStats { server: current_local_node_identity(), @@ -2154,6 +2219,12 @@ pub fn init_metrics_runtime(token: CancellationToken) { for bucket in retire_on_demand_migration_buckets { let _ = retire_on_demand_migration_metric_series(&bucket); } + for bucket in retire_on_demand_migration_backfill_buckets { + let _ = retire_on_demand_migration_backfill_metric_series( + &on_demand_migration_backfill_server, + &bucket, + ); + } }, ).await; } @@ -2980,6 +3051,66 @@ mod tests { assert_eq!(on_demand_migration_metric_series("photos").len(), emitted.len()); } + #[test] + fn on_demand_migration_backfill_bucket_keys_detect_finished_jobs() { + let stats = crate::metrics::collectors::on_demand_migration::tests::backfill_golden_stats("node1:9000"); + let previous = on_demand_migration_backfill_bucket_live_keys(&stats); + let current = on_demand_migration_backfill_bucket_live_keys(&OdmBackfillRuntimeStats { + server: stats.server.clone(), + buckets: stats.buckets[..1].to_vec(), + }); + let retired = previous.difference(¤t).cloned().collect::>(); + + assert_eq!(retired, bucket_keys(&["docs"])); + assert_eq!(current, bucket_keys(&["photos"])); + assert!(on_demand_migration_backfill_bucket_live_keys(&OdmBackfillRuntimeStats::default()).is_empty()); + } + + /// Every emitted backfill series must be named by the retirement walk. + /// The walk is wider than one cycle's emission on purpose: only the + /// job's current `state` gauge is emitted, so retirement enumerates all + /// lifecycle values to clear whichever one is live. + #[test] + fn on_demand_migration_backfill_retirement_covers_every_emitted_series() { + let stats = crate::metrics::collectors::on_demand_migration::tests::backfill_golden_stats("node1:9000"); + let emitted = collect_on_demand_migration_backfill_metrics(&stats) + .into_iter() + .map(|metric| { + ( + metric.name.to_string(), + metric + .labels + .into_iter() + .map(|(key, value)| (key, value.to_string())) + .collect::>(), + ) + }) + .collect::>(); + let retired = stats + .buckets + .iter() + .flat_map(|bucket| on_demand_migration_backfill_metric_series(&stats.server, &bucket.bucket)) + .map(|(name, labels)| { + ( + name, + labels + .into_iter() + .map(|(key, value)| (key, value.to_string())) + .collect::>(), + ) + }) + .collect::>(); + + assert!(emitted.is_subset(&retired), "emitted: {emitted:?}, retired: {retired:?}"); + assert_eq!( + on_demand_migration_backfill_metric_series("node1:9000", "photos").len(), + ODM_BACKFILL_STATES.len() + 6 + ); + // Without a process-global recorder there is nothing to retire; the + // walk itself must still cover every series. + assert_eq!(retire_on_demand_migration_backfill_metric_series("node1:9000", "photos"), 0); + } + #[test] fn metrics_runtime_status_reports_disabled_state() { let snapshot = build_metrics_runtime_status_snapshot(false, false, fixed_metrics_runtime_config(), false); diff --git a/crates/obs/src/metrics/schema/on_demand_migration.rs b/crates/obs/src/metrics/schema/on_demand_migration.rs index 62266683c..c8971bc5a 100644 --- a/crates/obs/src/metrics/schema/on_demand_migration.rs +++ b/crates/obs/src/metrics/schema/on_demand_migration.rs @@ -35,6 +35,10 @@ pub const PATH_L: &str = "path"; pub const REASON_L: &str = "reason"; /// Upper bound (seconds) of a source latency bucket. pub const LE_L: &str = "le"; +/// Node the backfill job runs on. +pub const SERVER_L: &str = "server"; +/// Lifecycle state of a backfill job. +pub const STATE_L: &str = "state"; /// Fixed `op` label values. pub const REQUEST_OPS: [&str; 2] = ["get", "head"]; @@ -71,6 +75,17 @@ pub const PULL_FAILURE_REASONS: [&str; 12] = [ pub const SOURCE_LATENCY_LE: [&str; 15] = [ "0.005", "0.01", "0.02", "0.05", "0.1", "0.2", "0.5", "1", "2", "5", "10", "20", "30", "60", "+Inf", ]; +/// Fixed `state` label values of `backfill_jobs`; mirrors the checkpoint's +/// `BackfillState` variants in ecstore. +pub const BACKFILL_STATES: [&str; 7] = [ + "pending", + "running", + "paused", + "cancelled", + "completed", + "completed_with_failures", + "failed", +]; /// `breaker_state` gauge value: the breaker admits every request. pub const BREAKER_STATE_CLOSED: f64 = 0.0; @@ -183,8 +198,80 @@ pub static ODM_BREAKER_STATE_MD: LazyLock = LazyLock::new(|| { ) }); -// backfill_* descriptors (`listed_total`, `pulled_total`, `skipped_total`, -// `failed_total`, `state`) are added here by ODM-12 (rustfs/backlog#2159). +// backfill_* descriptors (ODM-12, rustfs/backlog#2159). Unlike the request +// path above, a backfill job runs on exactly one node at a time, so every +// backfill series carries the owning node in `server` on top of `bucket`. + +const BACKFILL_JOBS: &str = "backfill_jobs"; +const BACKFILL_LISTED_TOTAL: &str = "backfill_listed_total"; +const BACKFILL_ENQUEUED_TOTAL: &str = "backfill_enqueued_total"; +const BACKFILL_PULLED_TOTAL: &str = "backfill_pulled_total"; +const BACKFILL_SKIPPED_EXISTING_TOTAL: &str = "backfill_skipped_existing_total"; +const BACKFILL_FAILED_TOTAL: &str = "backfill_failed_total"; +const BACKFILL_BYTES_TOTAL: &str = "backfill_bytes_total"; + +pub static ODM_BACKFILL_JOBS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::from(BACKFILL_JOBS), + "On-demand migration backfill jobs by server, bucket and state (1 for the bucket's current state)", + &[SERVER_L, BUCKET_L, STATE_L], + subsystems::ON_DEMAND_MIGRATION, + ) +}); + +pub static ODM_BACKFILL_LISTED_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::from(BACKFILL_LISTED_TOTAL), + "Source keys listed by the on-demand migration backfill job, by server and bucket", + &[SERVER_L, BUCKET_L], + subsystems::ON_DEMAND_MIGRATION, + ) +}); + +pub static ODM_BACKFILL_ENQUEUED_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::from(BACKFILL_ENQUEUED_TOTAL), + "Keys queued for pulling by the on-demand migration backfill job, by server and bucket", + &[SERVER_L, BUCKET_L], + subsystems::ON_DEMAND_MIGRATION, + ) +}); + +pub static ODM_BACKFILL_PULLED_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::from(BACKFILL_PULLED_TOTAL), + "Objects stored locally by the on-demand migration backfill job, by server and bucket", + &[SERVER_L, BUCKET_L], + subsystems::ON_DEMAND_MIGRATION, + ) +}); + +pub static ODM_BACKFILL_SKIPPED_EXISTING_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::from(BACKFILL_SKIPPED_EXISTING_TOTAL), + "Keys the on-demand migration backfill job skipped because a local object already existed, by server and bucket", + &[SERVER_L, BUCKET_L], + subsystems::ON_DEMAND_MIGRATION, + ) +}); + +pub static ODM_BACKFILL_FAILED_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::from(BACKFILL_FAILED_TOTAL), + "Keys the on-demand migration backfill job could not pull, by server and bucket", + &[SERVER_L, BUCKET_L], + subsystems::ON_DEMAND_MIGRATION, + ) +}); + +pub static ODM_BACKFILL_BYTES_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::from(BACKFILL_BYTES_TOTAL), + "Bytes stored locally by the on-demand migration backfill job, by server and bucket", + &[SERVER_L, BUCKET_L], + subsystems::ON_DEMAND_MIGRATION, + ) +}); #[cfg(test)] mod tests { @@ -208,6 +295,13 @@ mod tests { (&*ODM_SOURCE_LATENCY_SECONDS_SUM_MD, "source_latency_seconds_sum"), (&*ODM_SOURCE_LATENCY_SECONDS_COUNT_MD, "source_latency_seconds_count"), (&*ODM_BREAKER_STATE_MD, "breaker_state"), + (&*ODM_BACKFILL_JOBS_MD, "backfill_jobs"), + (&*ODM_BACKFILL_LISTED_MD, "backfill_listed_total"), + (&*ODM_BACKFILL_ENQUEUED_MD, "backfill_enqueued_total"), + (&*ODM_BACKFILL_PULLED_MD, "backfill_pulled_total"), + (&*ODM_BACKFILL_SKIPPED_EXISTING_MD, "backfill_skipped_existing_total"), + (&*ODM_BACKFILL_FAILED_MD, "backfill_failed_total"), + (&*ODM_BACKFILL_BYTES_MD, "backfill_bytes_total"), ] { assert_eq!(descriptor.get_full_metric_name(), format!("rustfs_on_demand_migration_{suffix}")); assert_eq!(descriptor.subsystem, subsystems::ON_DEMAND_MIGRATION); @@ -259,6 +353,23 @@ mod tests { assert_eq!(SOURCE_LATENCY_LE[14], "+Inf"); } + #[test] + fn backfill_series_are_server_and_bucket_scoped() { + assert_eq!(ODM_BACKFILL_JOBS_MD.metric_type, MetricType::Gauge); + assert_eq!(labels(&ODM_BACKFILL_JOBS_MD), vec!["server", "bucket", "state"]); + for descriptor in [ + &*ODM_BACKFILL_LISTED_MD, + &*ODM_BACKFILL_ENQUEUED_MD, + &*ODM_BACKFILL_PULLED_MD, + &*ODM_BACKFILL_SKIPPED_EXISTING_MD, + &*ODM_BACKFILL_FAILED_MD, + &*ODM_BACKFILL_BYTES_MD, + ] { + assert_eq!(descriptor.metric_type, MetricType::Counter); + assert_eq!(labels(descriptor), vec!["server", "bucket"]); + } + } + #[test] fn fixed_label_values_are_unique() { for values in [ @@ -267,6 +378,7 @@ mod tests { PULL_PATHS.as_slice(), PULL_FAILURE_REASONS.as_slice(), SOURCE_LATENCY_LE.as_slice(), + BACKFILL_STATES.as_slice(), ] { let unique: std::collections::BTreeSet<_> = values.iter().collect(); assert_eq!(unique.len(), values.len(), "{values:?}"); diff --git a/crates/obs/src/metrics/stats_collector.rs b/crates/obs/src/metrics/stats_collector.rs index ecc7f7b23..b3ed51655 100644 --- a/crates/obs/src/metrics/stats_collector.rs +++ b/crates/obs/src/metrics/stats_collector.rs @@ -26,15 +26,16 @@ use crate::metrics::collectors::{ ClusterHealthStats, ClusterStats, ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmBackpressureStats, IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats, - OnDemandMigrationBucketStats, ProcessStats, ProcessStatusType, ReplicationMetricsSnapshot, ResourceStats, - ScannerRuntimeStats, ScannerStats, + OdmBackfillRuntimeStats, OnDemandMigrationBucketStats, ProcessStats, ProcessStatusType, ReplicationMetricsSnapshot, + ResourceStats, ScannerRuntimeStats, ScannerStats, }; use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot}; use crate::metrics::{ BucketOperations, BucketOptions, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore, StorageAdminApi, obs_bucket_replication_stats_snapshot, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, - obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, + obs_on_demand_migration_backfill_snapshot, obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot, + obs_resolve_object_store_handle, }; use crate::node_identity::current_local_node_identity; use jiff::Timestamp; @@ -667,6 +668,11 @@ pub fn collect_on_demand_migration_stats() -> Vec obs_on_demand_migration_snapshot() } +/// Collect this node's on-demand migration backfill job progress. +pub fn collect_on_demand_migration_backfill_stats() -> OdmBackfillRuntimeStats { + obs_on_demand_migration_backfill_snapshot(current_local_node_identity()) +} + /// Collect site-level replication stats from the global replication runtime. pub async fn collect_replication_stats() -> ReplicationMetricsSnapshot { obs_site_replication_stats().await diff --git a/crates/obs/src/metrics/storage_api.rs b/crates/obs/src/metrics/storage_api.rs index 30145f770..e69cc64d8 100644 --- a/crates/obs/src/metrics/storage_api.rs +++ b/crates/obs/src/metrics/storage_api.rs @@ -17,6 +17,9 @@ use std::time::Duration; pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor; pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config; +use rustfs_ecstore::api::bucket::on_demand_migration::backfill::{ + BackfillCheckpoint as SourceBackfillCheckpoint, global_backfill_runner as source_global_backfill_runner, +}; use rustfs_ecstore::api::bucket::on_demand_migration::{ BreakerState as SourceOdmBreakerState, OdmBucketSnapshot as SourceOdmBucketSnapshot, OnDemandMigrationSys as SourceOnDemandMigrationSys, @@ -41,7 +44,9 @@ pub(crate) use rustfs_ecstore::api::runtime::{ pub(crate) use rustfs_ecstore::api::storage::ECStore as ObsStore; use rustfs_storage_api as storage_contracts; -use crate::metrics::collectors::{OnDemandMigrationBreakerState, OnDemandMigrationBucketStats}; +use crate::metrics::collectors::{ + OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBreakerState, OnDemandMigrationBucketStats, +}; #[derive(Debug, Clone, PartialEq)] pub(crate) struct ObsBucketReplicationTargetStatsSnapshot { @@ -496,6 +501,38 @@ pub(crate) fn obs_on_demand_migration_snapshot() -> Vec OdmBackfillBucketStats { + OdmBackfillBucketStats { + bucket, + state: checkpoint.state.as_str().to_string(), + listed: checkpoint.listed, + enqueued: checkpoint.enqueued, + pulled: checkpoint.pulled, + skipped_existing: checkpoint.skipped_existing, + failed: checkpoint.failed, + bytes: checkpoint.bytes, + } +} + +/// Backfill jobs running on this node, sorted by bucket. Empty until the +/// runner is installed, and empty again once a job finishes: the series are +/// per-node job progress, not a cluster-wide history. +pub(crate) fn obs_on_demand_migration_backfill_snapshot(server: String) -> OdmBackfillRuntimeStats { + let buckets = source_global_backfill_runner() + .map(|runner| { + runner + .local_job_snapshots() + .into_iter() + .map(|(bucket, checkpoint)| on_demand_migration_backfill_stats_from_checkpoint(bucket, checkpoint)) + .collect() + }) + .unwrap_or_default(); + OdmBackfillRuntimeStats { server, buckets } +} + pub(crate) async fn obs_replication_site_stats_snapshot(current_data_transfer_rate: f64) -> ObsReplicationSiteStatsSnapshot { let Some(stats) = get_global_replication_stats() else { return ObsReplicationSiteStatsSnapshot::default(); @@ -808,7 +845,8 @@ pub(crate) mod metrics { ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled, - obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_on_demand_migration_snapshot, - obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, obs_transition_state_handle, + obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_on_demand_migration_backfill_snapshot, + obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, + obs_transition_state_handle, }; } diff --git a/docs/architecture/admin-route-action-snapshot.md b/docs/architecture/admin-route-action-snapshot.md index b6b18aa47..5fdf806e1 100644 --- a/docs/architecture/admin-route-action-snapshot.md +++ b/docs/architecture/admin-route-action-snapshot.md @@ -33,6 +33,8 @@ Some bucket-scoped routes add gates after the `AdminAction` check. The gates are | Route | Actions | Extra gates after authorization | |---|---|---| | `PUT`/`DELETE /rustfs/admin/v3/on-demand-migration/{bucket}` (`?dry-run=true` validates and probes without saving) | `SetBucketOnDemandMigrationAction` (`admin:SetBucketOnDemandMigration`) | bucket must exist (`NoSuchBucket`); `PUT` also requires the `RUSTFS_ON_DEMAND_MIGRATION_ENABLED` module switch (`OnDemandMigrationDisabled`, 400) and the server license (`license_check()`, same mapping as object zip downloads); the source must answer HEAD + a one-key list (`OnDemandMigrationSourceUnreachable`, 400). Handler: `rustfs/src/admin/handlers/on_demand_migration.rs` | -| `GET /rustfs/admin/v3/on-demand-migration/{bucket}` and `GET .../{bucket}/status` | `GetBucketOnDemandMigrationAction` (`admin:GetBucketOnDemandMigration`) | bucket must exist; reads work while the module switch is off so operators can inspect a disabled deployment; `GET` answers `NoSuchConfiguration` (404) when nothing is configured | +| `GET /rustfs/admin/v3/on-demand-migration/{bucket}` and `GET .../{bucket}/status` | `GetBucketOnDemandMigrationAction` (`admin:GetBucketOnDemandMigration`) | bucket must exist; reads work while the module switch is off so operators can inspect a disabled deployment; `GET` answers `NoSuchConfiguration` (404) when nothing is configured; `status` carries a `backfill` summary of the latest backfill job when one was recorded | +| `POST /rustfs/admin/v3/on-demand-migration/{bucket}/backfill?op=start\|cancel` (body `{prefix?, skip_existing?, dry_run?}` for `start`) | `SetBucketOnDemandMigrationAction` (`admin:SetBucketOnDemandMigration`) | bucket must exist; `op` must be `start` or `cancel` (`InvalidArgument`, 400); `start` also requires the module switch (`OnDemandMigrationDisabled`, 400), the server license, a saved config (`NoSuchConfiguration`, 404) and an enabled bucket state on this node (`OnDemandMigrationDisabled`, 400); a job holding its lease answers `OnDemandMigrationBackfillRunning` (409); `cancel` answers `NoSuchBackfillJob` (404) when the bucket never had a job. Handler: `rustfs/src/admin/handlers/on_demand_migration.rs` | +| `GET /rustfs/admin/v3/on-demand-migration/{bucket}/backfill` | `GetBucketOnDemandMigrationAction` (`admin:GetBucketOnDemandMigration`) | bucket must exist; returns the backfill checkpoint document (`failed_keys` are key hashes, no credentials); `NoSuchBackfillJob` (404) when the bucket never had a job | -Responses on these routes carry the redacted configuration (`secret_key` and `session_token` replaced by `REDACTED`); the wire shape is pinned by the fixtures under `crates/madmin/fixtures/on_demand_migration/`, shared by the server handler tests and the `rustfs-madmin` client tests. +Responses on the config routes carry the redacted configuration (`secret_key` and `session_token` replaced by `REDACTED`); the wire shapes of every route in this group, including the backfill checkpoint and the `status` summary, are pinned by the fixtures under `crates/madmin/fixtures/on_demand_migration/`, shared by the server handler tests and the `rustfs-madmin` client tests. diff --git a/rustfs/src/admin/handlers/on_demand_migration.rs b/rustfs/src/admin/handlers/on_demand_migration.rs index 16de974fc..01832a4ee 100644 --- a/rustfs/src/admin/handlers/on_demand_migration.rs +++ b/rustfs/src/admin/handlers/on_demand_migration.rs @@ -18,9 +18,12 @@ //! pulled on first access. This module is the management plane only: //! `PUT`/`GET`/`DELETE /v3/on-demand-migration/{bucket}` configure, read and //! clear the source, `?dry-run=true` validates and probes without saving, and -//! `GET .../status` reports the switch state plus this node's runtime -//! snapshot of the bucket (breaker, counters, last source error). The data -//! plane and backfill live in other ODM tasks. +//! `GET .../status` reports the switch state, this node's runtime snapshot +//! of the bucket (breaker, counters, last source error) and a summary of the +//! backfill job. `POST .../backfill?op=start|cancel` and `GET .../backfill` +//! drive the background backfill job (ODM-12, rustfs/backlog#2159) through +//! the process-wide `BackfillRunner`; the checkpoint document is the wire +//! shape. //! //! Credentials in the request body are never echoed: every response carries //! the `redacted()` config, probe failures name only the error class, and no @@ -35,6 +38,9 @@ use crate::admin::runtime_sources::{ }; use crate::admin::storage_api::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG; use crate::admin::storage_api::bucket::metadata_sys; +use crate::admin::storage_api::bucket::on_demand_migration::backfill::{ + BackfillCheckpoint, BackfillError, BackfillRequest, BackfillState, SkipExisting, global_backfill_runner, +}; use crate::admin::storage_api::bucket::on_demand_migration::source_client::{ SourceClient, SourceClientSpec, SourceError, SourceProbe, SourceProvider, SourceTimeouts, }; @@ -54,7 +60,7 @@ use matchit::Params; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::Credentials; use rustfs_policy::policy::action::{Action, AdminAction}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::num::NonZeroU64; use std::sync::Arc; @@ -69,7 +75,12 @@ const EVENT_ADMIN_ON_DEMAND_MIGRATION_CONFIG: &str = "admin_bucket_on_demand_mig const ROUTE_PATH: &str = "/v3/on-demand-migration/{bucket}"; const STATUS_ROUTE_PATH: &str = "/v3/on-demand-migration/{bucket}/status"; +const BACKFILL_ROUTE_PATH: &str = "/v3/on-demand-migration/{bucket}/backfill"; const DRY_RUN_QUERY: &str = "dry-run"; +/// `POST .../backfill?op=` selector. +const BACKFILL_OP_QUERY: &str = "op"; +const BACKFILL_OP_START: &str = "start"; +const BACKFILL_OP_CANCEL: &str = "cancel"; /// Error code returned when the module switch is off and a write is attempted. pub(crate) const ERR_CODE_MODULE_DISABLED: &str = "OnDemandMigrationDisabled"; @@ -77,6 +88,10 @@ pub(crate) const ERR_CODE_MODULE_DISABLED: &str = "OnDemandMigrationDisabled"; pub(crate) const ERR_CODE_SOURCE_UNREACHABLE: &str = "OnDemandMigrationSourceUnreachable"; /// Error code returned by `GET` when the bucket has no configuration. pub(crate) const ERR_CODE_NO_SUCH_CONFIGURATION: &str = "NoSuchConfiguration"; +/// Error code (409) returned by `start` while a backfill job holds the lease. +pub(crate) const ERR_CODE_BACKFILL_RUNNING: &str = "OnDemandMigrationBackfillRunning"; +/// Error code (404) returned when the bucket never had a backfill job. +pub(crate) const ERR_CODE_NO_SUCH_BACKFILL_JOB: &str = "NoSuchBackfillJob"; /// The published switch is `RUSTFS_ON_DEMAND_MIGRATION_ENABLED`, owned by /// ODM-05 in `module_switches.rs`. This is the only read of it in the admin @@ -148,6 +163,9 @@ pub(crate) struct BucketOnDemandMigrationStatus { pub served_by_source_ratio: Option, /// RFC 3339 save time of the config; `null` when not configured. pub updated_at: Option, + /// Latest backfill job of the bucket, absent when none was ever started. + #[serde(skip_serializing_if = "Option::is_none")] + pub backfill: Option, } #[derive(Debug, Serialize)] @@ -215,6 +233,7 @@ fn bucket_status( queue_depth: 0, served_by_source_ratio: None, updated_at, + backfill: None, }; let Some(runtime) = runtime else { return Ok(status); @@ -259,10 +278,62 @@ fn bucket_status( Ok(status) } +/// Counters of the bucket's backfill job for the status endpoint. +#[derive(Debug, Serialize)] +pub(crate) struct BackfillSummary { + pub job_id: String, + pub state: BackfillState, + pub listed: u64, + pub enqueued: u64, + pub pulled: u64, + pub skipped_existing: u64, + pub failed: u64, + pub bytes: u64, + pub updated_at: String, +} + +impl BackfillSummary { + fn from_checkpoint(checkpoint: &BackfillCheckpoint) -> S3Result { + Ok(Self { + job_id: checkpoint.job_id.to_string(), + state: checkpoint.state, + listed: checkpoint.listed, + enqueued: checkpoint.enqueued, + pulled: checkpoint.pulled, + skipped_existing: checkpoint.skipped_existing, + failed: checkpoint.failed, + bytes: checkpoint.bytes, + updated_at: format_updated_at(checkpoint.updated_at)?, + }) + } +} + +/// Body of `POST .../backfill?op=start`; every field is optional. +#[derive(Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct BackfillStartRequest { + #[serde(default)] + pub prefix: Option, + /// `always` (default) or `etag_or_size`. + #[serde(default)] + pub skip_existing: Option, + #[serde(default)] + pub dry_run: bool, +} + +/// `POST`/`GET .../backfill` response: the checkpoint document as stored. +#[derive(Debug, Serialize)] +pub(crate) struct BackfillJobResponse { + pub bucket: String, + pub job: BackfillCheckpoint, +} + pub struct SetBucketOnDemandMigrationHandler; pub struct GetBucketOnDemandMigrationHandler; pub struct DeleteBucketOnDemandMigrationHandler; pub struct GetBucketOnDemandMigrationStatusHandler; +pub struct ControlBucketOnDemandMigrationBackfillHandler; +pub struct GetBucketOnDemandMigrationBackfillHandler; pub fn register_on_demand_migration_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( @@ -285,6 +356,16 @@ pub fn register_on_demand_migration_route(r: &mut S3Router) -> s format!("{ADMIN_PREFIX}{STATUS_ROUTE_PATH}").as_str(), AdminOperation(&GetBucketOnDemandMigrationStatusHandler {}), )?; + r.insert( + Method::POST, + format!("{ADMIN_PREFIX}{BACKFILL_ROUTE_PATH}").as_str(), + AdminOperation(&ControlBucketOnDemandMigrationBackfillHandler {}), + )?; + r.insert( + Method::GET, + format!("{ADMIN_PREFIX}{BACKFILL_ROUTE_PATH}").as_str(), + AdminOperation(&GetBucketOnDemandMigrationBackfillHandler {}), + )?; Ok(()) } @@ -357,6 +438,87 @@ fn is_dry_run(req: &S3Request) -> bool { .is_some_and(|value| value.eq_ignore_ascii_case("true")) } +/// What `POST .../backfill` was asked to do. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum BackfillOp { + Start, + Cancel, +} + +fn backfill_op(req: &S3Request) -> S3Result { + match extract_query_params(&req.uri).get(BACKFILL_OP_QUERY).map(String::as_str) { + Some(BACKFILL_OP_START) => Ok(BackfillOp::Start), + Some(BACKFILL_OP_CANCEL) => Ok(BackfillOp::Cancel), + _ => Err(admin_s3_error( + S3ErrorCode::InvalidArgument, + format!("query parameter {BACKFILL_OP_QUERY} must be {BACKFILL_OP_START} or {BACKFILL_OP_CANCEL}"), + )), + } +} + +/// An empty body means "defaults"; unknown fields and unknown +/// `skip_existing` labels are input errors. +fn parse_backfill_request(body: &[u8]) -> S3Result { + let request: BackfillStartRequest = if body.iter().all(u8::is_ascii_whitespace) { + BackfillStartRequest::default() + } else { + serde_json::from_slice(body) + .map_err(|err| admin_s3_error(S3ErrorCode::InvalidArgument, format!("invalid backfill request: {err}")))? + }; + let skip_existing = match request.skip_existing.as_deref() { + None => SkipExisting::default(), + Some(label) => SkipExisting::parse(label).ok_or_else(|| { + admin_s3_error( + S3ErrorCode::InvalidArgument, + format!("skip_existing must be always or etag_or_size, got {label}"), + ) + })?, + }; + let prefix = request.prefix.filter(|prefix| !prefix.is_empty()); + Ok(BackfillRequest { + prefix, + skip_existing, + dry_run: request.dry_run, + }) +} + +/// Runner errors onto HTTP: a held lease is a 409, a missing job a 404, a +/// missing or disabled config the same codes the config routes use. +fn backfill_error(bucket: &str, err: BackfillError) -> S3Error { + match err { + BackfillError::AlreadyRunning { job_id, owner, .. } => custom_error( + ERR_CODE_BACKFILL_RUNNING, + StatusCode::CONFLICT, + format!("a backfill job is already running for bucket {bucket} (job {job_id}, owner {owner})"), + ), + BackfillError::LeaseBusy(_) | BackfillError::Conflict(_) => custom_error( + ERR_CODE_BACKFILL_RUNNING, + StatusCode::CONFLICT, + format!("the backfill job of bucket {bucket} is being started or updated elsewhere; retry"), + ), + BackfillError::NotFound(_) => custom_error( + ERR_CODE_NO_SUCH_BACKFILL_JOB, + StatusCode::NOT_FOUND, + format!("no backfill job recorded for bucket {bucket}"), + ), + BackfillError::NotConfigured(_) => custom_error( + ERR_CODE_NO_SUCH_CONFIGURATION, + StatusCode::NOT_FOUND, + format!("on-demand migration is not configured for bucket {bucket}"), + ), + BackfillError::Unavailable(_) => custom_error( + ERR_CODE_MODULE_DISABLED, + StatusCode::BAD_REQUEST, + format!("bucket {bucket} has no enabled on-demand migration source on this node"), + ), + BackfillError::RunnerNotInstalled => admin_s3_error(S3ErrorCode::InternalError, "backfill runner is not installed"), + BackfillError::Malformed(_) | BackfillError::UnsupportedFormatVersion { .. } => { + admin_s3_error(S3ErrorCode::InternalError, format!("backfill checkpoint unreadable: {err}")) + } + BackfillError::Storage(_) => admin_s3_error(S3ErrorCode::InternalError, format!("backfill checkpoint failed: {err}")), + } +} + /// Every endpoint of this deployment, as `scheme://host:port`, so a source /// naming one of them with the same bucket is rejected as a self-reference. /// Single-node local-disk layouts carry no host and contribute nothing; the @@ -671,15 +833,100 @@ impl Operation for GetBucketOnDemandMigrationStatusHandler { })?; let runtime = OnDemandMigrationSys::get().bucket_snapshot(&bucket); - let status = bucket_status( + let backfill = match global_backfill_runner() { + Some(runner) => runner + .status(&bucket) + .await + .map_err(|err| backfill_error(&bucket, err))? + .map(|checkpoint| BackfillSummary::from_checkpoint(&checkpoint)) + .transpose()?, + None => None, + }; + + let mut status = bucket_status( config.as_ref().map(|(config, updated_at)| (config, *updated_at)), runtime, module_enabled(), )?; + status.backfill = backfill; admin_json_response(req.uri.path(), &cred.secret_key, StatusCode::OK, &status) } } +#[async_trait::async_trait] +impl Operation for ControlBucketOnDemandMigrationBackfillHandler { + #[tracing::instrument(skip_all)] + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let bucket = bucket_from_params(¶ms)?; + let cred = authorize_for_bucket(&req, AdminAction::SetBucketOnDemandMigrationAction, &bucket).await?; + let op = backfill_op(&req)?; + let path = req.uri.path().to_string(); + let runner = global_backfill_runner().ok_or_else(|| backfill_error(&bucket, BackfillError::RunnerNotInstalled))?; + + let job = match op { + BackfillOp::Start => { + if !module_enabled() { + return Err(custom_error( + ERR_CODE_MODULE_DISABLED, + StatusCode::BAD_REQUEST, + format!("on-demand migration is disabled: set {ENV_ON_DEMAND_MIGRATION_ENABLED}=true"), + )); + } + license_gate()?; + let body = read_compatible_admin_body(req.input, MAX_ADMIN_REQUEST_BODY_SIZE, &path, &cred.secret_key).await?; + let request = parse_backfill_request(&body)?; + let job = runner + .start(&bucket, request) + .await + .map_err(|err| backfill_error(&bucket, err))?; + info!( + event = EVENT_ADMIN_ON_DEMAND_MIGRATION_CONFIG, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + bucket = %bucket, + job_id = %job.job_id, + dry_run = job.dry_run, + skip_existing = job.skip_existing.as_str(), + "on-demand migration backfill started" + ); + job + } + BackfillOp::Cancel => { + let job = runner.cancel(&bucket).await.map_err(|err| backfill_error(&bucket, err))?; + info!( + event = EVENT_ADMIN_ON_DEMAND_MIGRATION_CONFIG, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION, + bucket = %bucket, + job_id = %job.job_id, + state = job.state.as_str(), + "on-demand migration backfill cancel requested" + ); + job + } + }; + let response = BackfillJobResponse { bucket, job }; + admin_json_response(&path, &cred.secret_key, StatusCode::OK, &response) + } +} + +#[async_trait::async_trait] +impl Operation for GetBucketOnDemandMigrationBackfillHandler { + #[tracing::instrument(skip_all)] + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let bucket = bucket_from_params(¶ms)?; + let cred = authorize_for_bucket(&req, AdminAction::GetBucketOnDemandMigrationAction, &bucket).await?; + let runner = global_backfill_runner().ok_or_else(|| backfill_error(&bucket, BackfillError::RunnerNotInstalled))?; + let job = runner + .status(&bucket) + .await + .map_err(|err| backfill_error(&bucket, err))? + .ok_or_else(|| backfill_error(&bucket, BackfillError::NotFound(bucket.clone())))?; + let response = BackfillJobResponse { bucket, job }; + admin_json_response(req.uri.path(), &cred.secret_key, StatusCode::OK, &response) + } +} + #[cfg(test)] mod tests { use super::*; @@ -699,6 +946,9 @@ mod tests { const SET_RESPONSE_FIXTURE: &str = include_str!("../../../../crates/madmin/fixtures/on_demand_migration/set_response.json"); const GET_RESPONSE_FIXTURE: &str = include_str!("../../../../crates/madmin/fixtures/on_demand_migration/get_response.json"); const STATUS_FIXTURE: &str = include_str!("../../../../crates/madmin/fixtures/on_demand_migration/status.json"); + const STATUS_WITH_BACKFILL_FIXTURE: &str = + include_str!("../../../../crates/madmin/fixtures/on_demand_migration/status_with_backfill.json"); + const BACKFILL_JOB_FIXTURE: &str = include_str!("../../../../crates/madmin/fixtures/on_demand_migration/backfill_job.json"); const FIXTURE_UPDATED_AT: &str = "2026-09-02T10:00:00Z"; fn fixture_config() -> OnDemandMigrationConfig { @@ -800,6 +1050,7 @@ mod tests { let json = serde_json::to_string(&status).expect("serialize"); assert_eq!(json, STATUS_FIXTURE.trim()); assert!(json.contains(r#""served_by_source_ratio":null"#), "the ratio field is present as null"); + assert!(status.backfill.is_none(), "a bucket without a job carries no summary"); } #[test] @@ -853,6 +1104,119 @@ mod tests { assert_eq!(config_endpoint_host(&config).as_deref(), Some("bucket.s3.example")); } + fn fixture_job() -> BackfillCheckpoint { + let value: serde_json::Value = serde_json::from_str(BACKFILL_JOB_FIXTURE.trim()).expect("fixture parses"); + BackfillCheckpoint::from_json(value["job"].to_string().as_bytes()).expect("fixture job decodes") + } + + #[test] + fn backfill_job_response_matches_madmin_golden_fixture() { + let response = BackfillJobResponse { + bucket: "photos".to_string(), + job: fixture_job(), + }; + assert_eq!(serde_json::to_string(&response).expect("serialize"), BACKFILL_JOB_FIXTURE.trim()); + } + + #[test] + fn status_with_backfill_summary_matches_madmin_golden_fixture() { + let config = fixture_config(); + let updated_at = OffsetDateTime::from_unix_timestamp(1_788_343_200).expect("timestamp"); + let mut status = bucket_status(Some((&config, updated_at)), Some(fixture_runtime_snapshot()), true).expect("status"); + status.backfill = Some(BackfillSummary::from_checkpoint(&fixture_job()).expect("summary")); + assert_eq!(serde_json::to_string(&status).expect("serialize"), STATUS_WITH_BACKFILL_FIXTURE.trim()); + } + + #[test] + fn backfill_request_defaults_validates_skip_existing_and_rejects_unknown_fields() { + assert_eq!(parse_backfill_request(b"").expect("empty body"), BackfillRequest::default()); + assert_eq!(parse_backfill_request(b" \n").expect("blank body"), BackfillRequest::default()); + assert_eq!(parse_backfill_request(b"{}").expect("empty object"), BackfillRequest::default()); + let full = parse_backfill_request(br#"{"prefix":"photos/","skip_existing":"etag_or_size","dry_run":true}"#) + .expect("full request"); + assert_eq!( + full, + BackfillRequest { + prefix: Some("photos/".to_string()), + skip_existing: SkipExisting::EtagOrSize, + dry_run: true, + } + ); + assert_eq!( + parse_backfill_request(br#"{"prefix":""}"#).expect("empty prefix").prefix, + None, + "an empty prefix means no prefix" + ); + let err = parse_backfill_request(br#"{"skip_existing":"never"}"#).unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + assert!(err.message().unwrap_or_default().contains("etag_or_size")); + let err = parse_backfill_request(br#"{"bogus":1}"#).unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + assert!(err.message().unwrap_or_default().contains("bogus")); + } + + #[test] + fn backfill_op_requires_start_or_cancel() { + let request = |uri: &'static str| S3Request { + input: Body::empty(), + method: Method::POST, + uri: Uri::from_static(uri), + headers: http::HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + assert_eq!( + backfill_op(&request("/rustfs/admin/v3/on-demand-migration/b/backfill?op=start")).expect("start"), + BackfillOp::Start + ); + assert_eq!( + backfill_op(&request("/rustfs/admin/v3/on-demand-migration/b/backfill?op=cancel")).expect("cancel"), + BackfillOp::Cancel + ); + for uri in [ + "/rustfs/admin/v3/on-demand-migration/b/backfill", + "/rustfs/admin/v3/on-demand-migration/b/backfill?op=pause", + "/rustfs/admin/v3/on-demand-migration/b/backfill?op=START", + ] { + assert_eq!(backfill_op(&request(uri)).unwrap_err().code(), &S3ErrorCode::InvalidArgument, "{uri}"); + } + } + + #[test] + fn backfill_errors_map_to_conflict_not_found_and_config_codes() { + let running = backfill_error( + "b", + BackfillError::AlreadyRunning { + bucket: "b".to_string(), + job_id: fixture_job().job_id, + owner: "node-a:9000".to_string(), + }, + ); + assert_eq!(running.status_code(), Some(StatusCode::CONFLICT)); + assert_eq!(running.code(), &S3ErrorCode::Custom(ERR_CODE_BACKFILL_RUNNING.into())); + assert!(running.message().unwrap_or_default().contains("node-a:9000")); + + let busy = backfill_error("b", BackfillError::LeaseBusy("b".to_string())); + assert_eq!(busy.status_code(), Some(StatusCode::CONFLICT)); + + let missing = backfill_error("b", BackfillError::NotFound("b".to_string())); + assert_eq!(missing.status_code(), Some(StatusCode::NOT_FOUND)); + assert_eq!(missing.code(), &S3ErrorCode::Custom(ERR_CODE_NO_SUCH_BACKFILL_JOB.into())); + + let unconfigured = backfill_error("b", BackfillError::NotConfigured("b".to_string())); + assert_eq!(unconfigured.code(), &S3ErrorCode::Custom(ERR_CODE_NO_SUCH_CONFIGURATION.into())); + + let unavailable = backfill_error("b", BackfillError::Unavailable("b".to_string())); + assert_eq!(unavailable.status_code(), Some(StatusCode::BAD_REQUEST)); + assert_eq!(unavailable.code(), &S3ErrorCode::Custom(ERR_CODE_MODULE_DISABLED.into())); + + let broken = backfill_error("b", BackfillError::UnsupportedFormatVersion { found: 9, supported: 1 }); + assert_eq!(broken.code(), &S3ErrorCode::InternalError); + } + #[test] fn updated_at_uses_rfc3339_utc() { let ts = OffsetDateTime::from_unix_timestamp(1_788_343_200).expect("timestamp"); diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 11adead27..b18bd4012 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -416,6 +416,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ GET_BUCKET_ON_DEMAND_MIGRATION, RouteRiskLevel::Sensitive, ), + admin( + HttpMethod::Post, + "/rustfs/admin/v3/on-demand-migration/{bucket}/backfill", + SET_BUCKET_ON_DEMAND_MIGRATION, + RouteRiskLevel::High, + ), + admin( + HttpMethod::Get, + "/rustfs/admin/v3/on-demand-migration/{bucket}/backfill", + GET_BUCKET_ON_DEMAND_MIGRATION, + RouteRiskLevel::Sensitive, + ), admin( HttpMethod::Get, "/rustfs/admin/export-bucket-metadata", @@ -2211,6 +2223,22 @@ mod tests { "/rustfs/admin/v3/on-demand-migration/{bucket}/status", GET_BUCKET_ON_DEMAND_MIGRATION, ); + // Backfill control (start/cancel) is a write; reading the checkpoint is not. + assert_action( + HttpMethod::Post, + "/rustfs/admin/v3/on-demand-migration/{bucket}/backfill", + SET_BUCKET_ON_DEMAND_MIGRATION, + ); + assert_action( + HttpMethod::Get, + "/rustfs/admin/v3/on-demand-migration/{bucket}/backfill", + GET_BUCKET_ON_DEMAND_MIGRATION, + ); + assert_not_action( + HttpMethod::Get, + "/rustfs/admin/v3/on-demand-migration/{bucket}/backfill", + SET_BUCKET_ON_DEMAND_MIGRATION, + ); // Reads never require the write action, and the routes are not bucket-target routes. assert_not_action( HttpMethod::Get, diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 85f7c029a..5a8e65694 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -248,6 +248,16 @@ fn expected_admin_route_matrix() -> Vec { "/v3/on-demand-migration/{bucket}/status", "/v3/on-demand-migration/test-bucket/status", ), + admin_route_sample( + Method::POST, + "/v3/on-demand-migration/{bucket}/backfill", + "/v3/on-demand-migration/test-bucket/backfill", + ), + admin_route_sample( + Method::GET, + "/v3/on-demand-migration/{bucket}/backfill", + "/v3/on-demand-migration/test-bucket/backfill", + ), admin_route(Method::GET, "/export-bucket-metadata"), admin_route(Method::GET, "/v3/export-bucket-metadata"), admin_route(Method::PUT, "/import-bucket-metadata"), @@ -1286,6 +1296,8 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::GET, &admin_path("/v3/on-demand-migration/test-bucket")); assert_route(&router, Method::DELETE, &admin_path("/v3/on-demand-migration/test-bucket")); assert_route(&router, Method::GET, &admin_path("/v3/on-demand-migration/test-bucket/status")); + assert_route(&router, Method::POST, &admin_path("/v3/on-demand-migration/test-bucket/backfill")); + assert_route(&router, Method::GET, &admin_path("/v3/on-demand-migration/test-bucket/backfill")); assert_route(&router, Method::GET, &admin_path("/export-bucket-metadata")); assert_route(&router, Method::GET, &admin_path("/v3/export-bucket-metadata")); @@ -1419,6 +1431,8 @@ fn test_admin_alias_paths_match_existing_admin_routes() { (Method::GET, compat_admin_alias_path("/v3/on-demand-migration/b")), (Method::DELETE, compat_admin_alias_path("/v3/on-demand-migration/b")), (Method::GET, compat_admin_alias_path("/v3/on-demand-migration/b/status")), + (Method::POST, compat_admin_alias_path("/v3/on-demand-migration/b/backfill")), + (Method::GET, compat_admin_alias_path("/v3/on-demand-migration/b/backfill")), ] { assert!( router.contains_compatible_route(method.clone(), &path), diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 143dd2714..95285291d 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -292,6 +292,15 @@ pub(crate) mod on_demand_migration { pub(crate) type Provider = super::ecstore_bucket::on_demand_migration::Provider; pub(crate) type ValidationContext<'a> = super::ecstore_bucket::on_demand_migration::ValidationContext<'a>; + pub(crate) mod backfill { + pub(crate) type BackfillCheckpoint = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillCheckpoint; + pub(crate) type BackfillError = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillError; + pub(crate) type BackfillRequest = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillRequest; + pub(crate) type BackfillState = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillState; + pub(crate) type SkipExisting = super::super::ecstore_bucket::on_demand_migration::backfill::SkipExisting; + pub(crate) use super::super::ecstore_bucket::on_demand_migration::backfill::global_backfill_runner; + } + pub(crate) mod source_client { pub(crate) type SourceClient = super::super::ecstore_bucket::on_demand_migration::source_client::SourceClient; pub(crate) type SourceClientSpec = super::super::ecstore_bucket::on_demand_migration::source_client::SourceClientSpec; diff --git a/rustfs/src/startup_background.rs b/rustfs/src/startup_background.rs index fd5366740..4d2077b0c 100644 --- a/rustfs/src/startup_background.rs +++ b/rustfs/src/startup_background.rs @@ -14,9 +14,13 @@ use crate::bitrot_selftest::run_startup_bitrot_self_test; use crate::module_switches::{ - bitrot_selftest_enabled_from_env, bitrot_selftest_strict_from_env, heal_enabled_from_env, scanner_enabled_from_env, + bitrot_selftest_enabled_from_env, bitrot_selftest_strict_from_env, heal_enabled_from_env, + is_on_demand_migration_module_enabled, scanner_enabled_from_env, +}; +use crate::storage_api::startup::background::{ + BackfillRunner, ECStore, OnDemandMigrationSys, SysBackfillContexts, install_global_backfill_runner, + set_workload_admission_snapshot_provider, spawn_backfill_recovery_loop, }; -use crate::storage_api::startup::background::{ECStore, set_workload_admission_snapshot_provider}; use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; use rustfs_heal::{ @@ -28,6 +32,7 @@ use tracing::{debug, info}; const LOG_COMPONENT_MAIN: &str = "main"; const LOG_SUBSYSTEM_STARTUP: &str = "startup"; const EVENT_BACKGROUND_SERVICES_CONFIGURED: &str = "background_services_configured"; +const EVENT_ODM_BACKFILL_RECOVERY_CONFIGURED: &str = "odm_backfill_recovery_configured"; pub(crate) async fn init_background_service_runtime(store: Arc) -> Result { // Pin the bitrot algorithms before anything can write or verify a shard: @@ -56,7 +61,7 @@ pub(crate) async fn init_background_service_runtime(store: Arc) -> Resu let _ = set_workload_admission_snapshot_provider(workload_provider.clone()); if enable_heal || enable_scanner { - let heal_storage = Arc::new(ECStoreHealStorage::new(store)); + let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone())); init_heal_manager_with_workload_provider(heal_storage, None, Some(workload_provider)).await?; } @@ -74,5 +79,45 @@ pub(crate) async fn init_background_service_runtime(store: Arc) -> Resu ); } + init_on_demand_migration_backfill_runtime(store).await; + Ok(enable_scanner) } + +/// Installs the backfill runner (admin start/cancel/status need it even +/// while the module switch is off, to read checkpoints) and, with the switch +/// on, the recovery loop that takes over expired leases (rustfs/backlog#2159). +async fn init_on_demand_migration_backfill_runtime(store: Arc) { + let contexts = Arc::new(SysBackfillContexts::new(store.clone(), OnDemandMigrationSys::get())); + let runner = BackfillRunner::for_local_node(store.clone(), contexts).await; + if !install_global_backfill_runner(runner.clone()) { + debug!( + target: "rustfs::main::run", + event = EVENT_ODM_BACKFILL_RECOVERY_CONFIGURED, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "already_installed", + "On-demand migration backfill runner already installed" + ); + return; + } + let module_enabled = is_on_demand_migration_module_enabled(); + let node = runner.node().to_string(); + let state = if !module_enabled { + "skipped_module_disabled" + } else if spawn_backfill_recovery_loop(runner) { + "started" + } else { + "skipped_no_cancel_token" + }; + info!( + target: "rustfs::main::run", + event = EVENT_ODM_BACKFILL_RECOVERY_CONFIGURED, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = state, + module_enabled = module_enabled, + node = %node, + "On-demand migration backfill recovery configured" + ); +} diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index 2f7a442a9..e74e88d02 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -278,6 +278,10 @@ pub(crate) mod startup { } pub(crate) mod background { + pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::OnDemandMigrationSys; + pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::backfill::{ + BackfillRunner, SysBackfillContexts, install_global_backfill_runner, spawn_backfill_recovery_loop, + }; pub(crate) use crate::storage::storage_api::{ BitrotSelfTestError, ECStore, bitrot_self_test, set_workload_admission_snapshot_provider, };