From 7dc759f9cf597ecd553d660403eb0580c8f7ee9f Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 21 Nov 2026 22:10:37 +0800 Subject: [PATCH] fix(scanner): preserve progress age and bound backup work Retain accepted measurement times across cold source handoffs and checkpoint resaves. Bound preparation, reconciliation and reclamation with the same adaptive best-effort backup deadline. Keep the Admin SDK a leaf by projecting server progress into SDK-owned DTOs without changing the JSON contract. Cover backup deadlines, retained compacted receipts, legacy codecs and future diagnostic labels with regressions. --- Cargo.lock | 1 - crates/madmin/Cargo.toml | 1 - crates/madmin/src/user.rs | 46 +++++++++- crates/scanner/src/data_usage_define.rs | 26 ++++++ .../src/data_usage_define/display_progress.rs | 89 ++++++++++++++++-- .../src/data_usage_define/persistence.rs | 6 +- .../src/data_usage_define/source_tree.rs | 86 ++++++++++++++++- crates/scanner/src/data_usage_define/tests.rs | 10 +- crates/scanner/src/scanner/tests.rs | 4 + .../tests/paged_checkpoint/save_review.rs | 92 +++++++++++++++++++ crates/scanner/src/scanner_folder.rs | 3 + .../src/scanner_folder/directory_cursor.rs | 6 ++ .../directory_cursor/checkpoint.rs | 55 ++++++++++- .../checkpoint/storage_fixture.rs | 70 +++++++++++++- .../tests/checkpoint_fixture.rs | 1 + .../checkpoint_fixture/deep_compacted.rs | 21 +++-- docs/operations/scanner-usage-statistics.md | 4 +- rustfs/src/admin/handlers/account_info.rs | 78 +++++++++++++++- 18 files changed, 564 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e8865c2b77..928aa14f09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10821,7 +10821,6 @@ dependencies = [ "jiff 1.0.1", "reqwest", "rmp-serde", - "rustfs-data-usage", "rustfs-signer", "serde", "serde_json", diff --git a/crates/madmin/Cargo.toml b/crates/madmin/Cargo.toml index c80e4968ec..665524ac46 100644 --- a/crates/madmin/Cargo.toml +++ b/crates/madmin/Cargo.toml @@ -42,7 +42,6 @@ http.workspace = true hyper = { workspace = true, features = ["http2", "http1", "server"] } reqwest = { workspace = true, features = ["json"] } rustfs-signer.workspace = true -rustfs-data-usage.workspace = true jiff = { workspace = true, features = ["serde"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } diff --git a/crates/madmin/src/user.rs b/crates/madmin/src/user.rs index 6271ee391a..8af358564b 100644 --- a/crates/madmin/src/user.rs +++ b/crates/madmin/src/user.rs @@ -413,9 +413,10 @@ pub struct BucketAccessInfo { pub name: String, /// Display-only accepted scan progress, never a replacement for total usage. #[serde(default, skip_serializing_if = "Option::is_none")] - pub scan_progress: Option, + pub scan_progress: Option, + /// Value provenance: certified, carried, observed or unknown. Preserve future labels. #[serde(default, skip_serializing_if = "Option::is_none")] - pub usage_value_state: Option, + pub usage_value_state: Option, /// Unix milliseconds of the original complete statistic, when known. #[serde(default, skip_serializing_if = "Option::is_none")] pub usage_updated_at: Option, @@ -437,6 +438,20 @@ pub struct BucketAccessInfo { pub access: AccountAccess, } +/// Accepted scan coverage for display, independent of total usage and quotas. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct BucketScanProgress { + /// Observation stage. Clients must preserve unfamiliar diagnostic labels. + pub stage: String, + /// Oldest contributing measurement time in Unix milliseconds. + pub updated_at: u64, + /// Enumeration alone leaves both counters absent. + pub objects: Option, + pub size: Option, + pub sources_observed: u64, + pub sources_total: u64, +} + #[derive(Debug, Serialize, Deserialize, Default)] pub struct BucketDetails { pub versioning: bool, @@ -762,6 +777,33 @@ mod tests { assert!(!obj.contains_key("objects")); assert!(!obj.contains_key("object_sizes_histogram")); assert!(!obj.contains_key("object_versions_histogram")); + assert!(!obj.contains_key("scan_progress")); + assert!(!obj.contains_key("usage_value_state")); + assert!(!obj.contains_key("usage_updated_at")); + } + + #[test] + fn bucket_access_info_scan_progress_does_not_fill_total_usage() { + let mut wire = serde_json::to_value(BucketAccessInfo::default()).expect("empty account bucket response"); + wire["scan_progress"] = serde_json::json!({ + "stage": "scanning", "updated_at": 1000, "objects": 7, "size": 71, + "sources_observed": 1, "sources_total": 2, + }); + let bucket: BucketAccessInfo = serde_json::from_value(wire.clone()).expect("decode scan progress response"); + let value = serde_json::to_value(bucket).expect("encode scan progress response"); + assert_eq!(value, wire, "scan coverage must round-trip without synthesizing total usage"); + } + + #[test] + fn bucket_access_info_preserves_future_progress_and_provenance_values() { + let mut wire = serde_json::to_value(BucketAccessInfo::default()).expect("empty account bucket response"); + wire["scan_progress"] = serde_json::json!({ + "stage": "future_stage", "updated_at": 1000, "objects": null, "size": null, + "sources_observed": 1, "sources_total": 2, + }); + wire["usage_value_state"] = serde_json::json!("future_provenance"); + let bucket: BucketAccessInfo = serde_json::from_value(wire.clone()).expect("SDK accepts future diagnostic labels"); + assert_eq!(serde_json::to_value(bucket).expect("preserve diagnostic labels"), wire); } /// Wire pin (rustfs/backlog#1306): populated usage stats keep their diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 639f30b729..5c6b6f0d0f 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -524,6 +524,10 @@ pub struct DataUsageScanCoverageReceipt { pub digest: [u8; 32], } +pub(crate) fn scan_measurement_time_millis(time: SystemTime) -> Option { + u64::try_from(time.duration_since(SystemTime::UNIX_EPOCH).ok()?.as_millis()).ok() +} + impl Serialize for DataUsageScanCoverageReceipt { fn serialize(&self, serializer: S) -> Result { let mut map = serializer.serialize_map(Some(2))?; @@ -1063,6 +1067,9 @@ pub struct DataUsageCacheInfo { pub scan_bucket_observations: HashMap, #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub scan_bucket_value_origins: HashMap, + /// Original accepted-frontier measurement time; absent on legacy checkpoints. + #[serde(default)] + pub scan_coverage_measured_at: Option, } impl Serialize for DataUsageCacheInfo { @@ -1083,6 +1090,7 @@ impl Serialize for DataUsageCacheInfo { + usize::from(self.scan_progress.is_some()) + usize::from(self.scan_usage_watch.is_some()) + usize::from(self.scan_coverage_receipt.is_some()) + + usize::from(self.scan_coverage_measured_at.is_some()) + usize::from(self.scan_coverage_digest.is_some()) + usize::from(self.tier_registry_generation.is_some()) + usize::from(!self.size_reconciliation.is_empty()) @@ -1138,6 +1146,9 @@ impl Serialize for DataUsageCacheInfo { if let Some(receipt) = &self.scan_coverage_receipt { state.serialize_entry("scan_coverage_receipt", receipt)?; } + if let Some(measured_at) = self.scan_coverage_measured_at { + state.serialize_entry("scan_coverage_measured_at", &measured_at)?; + } state.serialize_entry("pending_heals", &self.pending_heals)?; state.serialize_entry("object_lock", &crate::object_lock_cache::ObjectLockCache(&self.object_lock))?; state.serialize_entry("source", &self.source)?; @@ -1569,6 +1580,7 @@ impl DataUsageCache { self.info.scan_directory_source = None; self.info.scan_source_trees.clear(); self.info.scan_coverage_receipt = None; + self.info.scan_coverage_measured_at = None; } // Old readers do not understand coverage sweeps. An absent plan makes // their existing prepare path rebuild instead of promoting mixed data. @@ -1748,9 +1760,22 @@ impl DataUsageCache { pub(crate) fn seal_scan_frontier(&mut self, frontier: Option<&str>) -> Result<(), serde_json::Error> { if self.info.scan_progress.is_none() { self.info.scan_coverage_receipt = None; + self.info.scan_coverage_measured_at = None; return Ok(()); } let frontier = frontier.filter(|path| path.len() <= 16 * 1024 && self.find(path).is_some()); + let measured_at = frontier.and_then(|through| { + if self + .info + .scan_coverage_receipt + .as_ref() + .is_some_and(|previous| previous.through == through) + { + self.info.scan_coverage_measured_at + } else { + scan_measurement_time_millis(SystemTime::now()) + } + }); self.info.scan_coverage_receipt = match frontier { Some(through) => Some(DataUsageScanCoverageReceipt { through: clone_cache_value(through), @@ -1758,6 +1783,7 @@ impl DataUsageCache { }), None => None, }; + self.info.scan_coverage_measured_at = measured_at; self.info.scan_resume_after = frontier.map(clone_cache_value); let reason = self .info diff --git a/crates/scanner/src/data_usage_define/display_progress.rs b/crates/scanner/src/data_usage_define/display_progress.rs index 880e3b3639..ca384abf8a 100644 --- a/crates/scanner/src/data_usage_define/display_progress.rs +++ b/crates/scanner/src/data_usage_define/display_progress.rs @@ -132,13 +132,16 @@ impl DataUsageCache { .filter(|observation| Some(observation.completed_at) == self.info.last_update && !self.has_scan_position()) .map(|observation| observation.entry.clone()) }; - let mut accepted = completed.as_ref().map(|entry| (entry.objects, entry.size)); + let mut accepted = completed.as_ref().map(|entry| (entry.objects, entry.size, updated_at)); if completed.is_none() { let frontier = self.validated_scan_frontier(); let mut replicas = BTreeMap::<&str, Vec<_>>::new(); for checkpoint in &self.info.scan_directory_work { - if let Some((coverage, entry)) = checkpoint.display_usage(&self.info) { - replicas.entry(checkpoint.endpoint()).or_default().push((coverage, entry)); + if let Some((coverage, entry, measured_at)) = checkpoint.display_usage(&self.info) { + replicas + .entry(checkpoint.endpoint()) + .or_default() + .push((coverage, entry, measured_at)); } } if let Some(active) = self.info.scan_directory_source.as_deref() { @@ -150,7 +153,9 @@ impl DataUsageCache { receipts.sort_by(|a, b| a.0.parent.split('/').cmp(b.0.parent.split('/'))); let mut selected = Vec::new(); let mut subtotal: Option<(usize, usize)> = None; - for (coverage, entry) in &receipts { + let mut measured_at: Option = None; + let mut known_time = true; + for (coverage, entry, time) in &receipts { if selected.iter().any(|parent: &&crate::scanner_folder::DirectoryProgress| { parent.parent == coverage.parent || parent.covers(&coverage.parent) }) { @@ -163,6 +168,11 @@ impl DataUsageCache { } let (objects, size) = subtotal.unwrap_or_default(); subtotal = Some((objects.checked_add(entry.objects)?, size.checked_add(entry.size)?)); + if let Some(time) = time { + measured_at = Some(measured_at.map_or(*time, |previous| previous.min(*time))); + } else { + known_time = false; + } selected.push(coverage); } if Some(endpoint) == self.info.scan_directory_source.as_deref() @@ -171,16 +181,30 @@ impl DataUsageCache { // A validated receipt signs this prefix. Sum each scalar // once, excluding the ranges represented by compacted work. self.checked_flatten_complete_scope(&self.info.name)?; + let mut contributed = false; for (path, entry) in &self.cache { if scan_frontier_covers_path(path, through) && !selected.iter().any(|proof| proof.covers(path)) { let (objects, size) = subtotal.unwrap_or_default(); subtotal = Some((objects.checked_add(entry.objects)?, size.checked_add(entry.size)?)); + contributed = true; + } + } + if contributed { + if let Some(time) = self.info.scan_coverage_measured_at { + measured_at = Some(measured_at.map_or(time, |previous| previous.min(time))); + } else { + known_time = false; } } } // Replicas overlap. Select one sample; never sum or append them. - if subtotal.is_some_and(|sample| accepted.is_none_or(|previous| sample > previous)) { - accepted = subtotal; + // Legacy coverage without a measurement time cannot borrow a + // newer checkpoint time. Keep the prior companion instead. + if known_time + && let (Some((objects, size)), Some(time)) = (subtotal, measured_at) + && accepted.is_none_or(|previous| (objects, size, time) > previous) + { + accepted = Some((objects, size, time)); } } } @@ -195,7 +219,7 @@ impl DataUsageCache { movement_generation: Some(movement_generation), scanner_epoch: self.info.leader_epoch, scanner_cycle: self.info.next_cycle, - updated_at, + updated_at: accepted.map_or(updated_at, |(_, _, time)| time), stage: if completed.is_some() { BucketScanStage::Completed } else if accepted.is_some() { @@ -203,8 +227,8 @@ impl DataUsageCache { } else { BucketScanStage::Enumerating }, - objects: accepted.map(|(objects, _)| u64::try_from(objects)).transpose().ok()?, - size: accepted.map(|(_, size)| u64::try_from(size)).transpose().ok()?, + objects: accepted.map(|(objects, _, _)| u64::try_from(objects)).transpose().ok()?, + size: accepted.map(|(_, size, _)| u64::try_from(size)).transpose().ok()?, }) } @@ -480,6 +504,53 @@ mod tests { cache } + #[test] + fn display_progress_frontier_resealing_and_cold_source_handoff_preserve_time() { + let mut cache = checkpoint(); + cache.replace("bucket", "", DataUsageEntry::default()); + cache.replace( + "bucket/a", + "bucket", + DataUsageEntry { + objects: 1, + size: 11, + ..Default::default() + }, + ); + cache.seal_scan_frontier(Some("bucket/a")).expect("seal measured frontier"); + cache.info.scan_coverage_measured_at = Some(10_000); + cache.info.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(70)); + cache.seal_scan_frontier(Some("bucket/a")).expect("reseal unchanged frontier"); + assert_eq!(cache.display_scan_progress(0).expect("unchanged observation").updated_at, 10_000); + let endpoints = std::collections::HashSet::from(["source".to_owned(), "other".to_owned()]); + cache.activate_directory_source("other".into(), &endpoints); + cache.replace("bucket", "", DataUsageEntry::default()); + let bytes = cache.marshal_msg().expect("save inactive source with its measurement time"); + let mut cold = DataUsageCache::unmarshal(&bytes).expect("cold source handoff"); + cold.activate_directory_source("source".into(), &endpoints); + let sample = cold.display_scan_progress(0).expect("restored measured frontier"); + assert_eq!((sample.objects, sample.size, sample.updated_at), (Some(1), Some(11), 10_000)); + } + + #[test] + fn display_progress_legacy_frontier_cannot_borrow_a_new_checkpoint_time() { + let mut cache = checkpoint(); + cache.replace("bucket", "", DataUsageEntry::default()); + cache.replace( + "bucket/a", + "bucket", + DataUsageEntry { + objects: 1, + size: 11, + ..Default::default() + }, + ); + cache.seal_scan_frontier(Some("bucket/a")).expect("legacy coverage"); + cache.info.scan_coverage_measured_at = None; + let sample = cache.display_scan_progress(0).expect("enumeration remains observable"); + assert_eq!((sample.objects, sample.size), (None, None)); + } + #[test] fn display_progress_excludes_old_tail_and_uncommitted_ancestors() { let mut cache = checkpoint(); diff --git a/crates/scanner/src/data_usage_define/persistence.rs b/crates/scanner/src/data_usage_define/persistence.rs index 79343c851f..4e85640f6e 100644 --- a/crates/scanner/src/data_usage_define/persistence.rs +++ b/crates/scanner/src/data_usage_define/persistence.rs @@ -813,7 +813,11 @@ impl DataUsageCache { } Ok::<_, StorageError>(()) }; - if let Err(e) = save_backup.await { + let backup_result = match timeout(backup_timeout_duration, save_backup).await { + Ok(result) => result, + Err(_) => Err(StorageError::other("scanner cache backup save timed out")), + }; + if let Err(e) = backup_result { warn!( target: "rustfs::scanner::data_usage", event = EVENT_SCANNER_CACHE_SAVE_STATE, diff --git a/crates/scanner/src/data_usage_define/source_tree.rs b/crates/scanner/src/data_usage_define/source_tree.rs index ded3980430..ec46c15be0 100644 --- a/crates/scanner/src/data_usage_define/source_tree.rs +++ b/crates/scanner/src/data_usage_define/source_tree.rs @@ -5,12 +5,11 @@ use super::*; /// A linked tree paused on a different physical replica. It contains only /// traversal fields, so dormant sources cannot recursively contain each other. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Deserialize)] pub struct SourceTree { bucket: String, identity: DataUsageScanIdentity, started_plan: DataUsageScanPlanDigest, - #[serde(serialize_with = "ordered_entries")] entries: HashMap, resume_after: Option, checkpoint: Option, @@ -18,6 +17,36 @@ pub struct SourceTree { cursor: Option, index: Option, ancestors: Vec, + #[serde(default)] + coverage_measured_at: Option, +} + +impl Serialize for SourceTree { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(Some(10 + usize::from(self.coverage_measured_at.is_some())))?; + map.serialize_entry("bucket", &self.bucket)?; + map.serialize_entry("identity", &self.identity)?; + map.serialize_entry("started_plan", &self.started_plan)?; + map.serialize_entry("entries", &OrderedEntries(&self.entries))?; + map.serialize_entry("resume_after", &self.resume_after)?; + map.serialize_entry("checkpoint", &self.checkpoint)?; + map.serialize_entry("coverage", &self.coverage)?; + map.serialize_entry("cursor", &self.cursor)?; + map.serialize_entry("index", &self.index)?; + map.serialize_entry("ancestors", &self.ancestors)?; + if let Some(measured_at) = self.coverage_measured_at { + map.serialize_entry("coverage_measured_at", &measured_at)?; + } + map.end() + } +} + +struct OrderedEntries<'a>(&'a HashMap); + +impl Serialize for OrderedEntries<'_> { + fn serialize(&self, serializer: S) -> Result { + ordered_entries(self.0, serializer) + } } impl SourceTree { @@ -89,6 +118,7 @@ impl DataUsageCache { resume_after: self.info.scan_resume_after.take(), checkpoint: self.info.scan_checkpoint.take(), coverage: self.info.scan_coverage_receipt.take(), + coverage_measured_at: self.info.scan_coverage_measured_at.take(), cursor: self.info.scan_raw_enumeration_cursor.take(), index: self.info.scan_raw_enumeration_page_index.take(), ancestors: std::mem::take(&mut self.info.scan_raw_enumeration_path_page_indexes), @@ -104,6 +134,7 @@ impl DataUsageCache { self.cache = tree.entries; self.info.scan_resume_after = tree.resume_after; self.info.scan_coverage_receipt = tree.coverage; + self.info.scan_coverage_measured_at = tree.coverage_measured_at; self.info.scan_checkpoint = tree.checkpoint; self.info.scan_raw_enumeration_cursor = tree.cursor; self.info.scan_raw_enumeration_page_index = tree.index; @@ -122,6 +153,7 @@ impl DataUsageCache { self.info.scan_resume_after = None; self.info.scan_checkpoint = None; self.info.scan_coverage_receipt = None; + self.info.scan_coverage_measured_at = None; self.info.scan_raw_enumeration_cursor = None; self.info.scan_raw_enumeration_page_index = None; self.info.scan_raw_enumeration_path_page_indexes.clear(); @@ -131,3 +163,53 @@ impl DataUsageCache { fn directory_source_location(source: &str) -> &str { source.split_once('|').map_or(source, |(_, endpoint)| endpoint) } + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Serialize, Deserialize)] + struct LegacySourceTree { + bucket: String, + identity: DataUsageScanIdentity, + started_plan: DataUsageScanPlanDigest, + entries: HashMap, + resume_after: Option, + checkpoint: Option, + coverage: Option, + cursor: Option, + index: Option, + ancestors: Vec, + } + + #[test] + fn source_tree_measurement_metadata_preserves_legacy_codecs() { + let tree = SourceTree { + bucket: "bucket".into(), + identity: DataUsageScanIdentity { + version: DATA_USAGE_SCAN_CHECKPOINT_VERSION, + bucket_incarnation: uuid::Uuid::from_u128(1), + set_layout: DataUsageScanPlanDigest([2; 32]), + publication_epoch: 0, + tier_registry_generation: 0, + scan_mode: HealScanMode::Normal, + }, + started_plan: DataUsageScanPlanDigest([3; 32]), + entries: HashMap::from([("bucket".into(), DataUsageEntry::default())]), + resume_after: None, + checkpoint: None, + coverage: None, + cursor: None, + index: None, + ancestors: Vec::new(), + coverage_measured_at: Some(10_000), + }; + let bytes = rmp_serde::to_vec(&tree).expect("encode additive named source tree"); + let old: LegacySourceTree = rmp_serde::from_slice(&bytes).expect("old reader ignores diagnostic time"); + let bytes = rmp_serde::to_vec(&old).expect("encode old positional source tree"); + let recovered: SourceTree = rmp_serde::from_slice(&bytes).expect("new reader accepts old positional source tree"); + assert_eq!(recovered.bucket, tree.bucket); + assert_eq!(recovered.entries.len(), 1); + assert_eq!(recovered.coverage_measured_at, None); + } +} diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index f3546d5384..360215353a 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -2924,7 +2924,15 @@ async fn test_checkpoint_budget_covers_retries_and_reconciliation() { .expect("a successful or reconciled primary must survive a best-effort backup failure"); assert_eq!(store.put_attempts[0].load(Ordering::SeqCst), 3); assert_eq!(store.put_attempts[1].load(Ordering::SeqCst), 1, "backup writes must not retry"); - assert_eq!(*store.backup_reads.lock().await, 1, "backup must finish its bounded reconciliation"); + assert_eq!( + *store.backup_reads.lock().await, + 0, + "backup reconciliation must not outlive the total backup budget" + ); + assert!( + store.puts.lock().await.iter().all(|put| !put.object.ends_with(".bkp")), + "a timed-out backup must not replace the previous fallback" + ); assert!(started.elapsed() < DataUsageCache::persistence_timeout()); } }) diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 97ae5b5fc6..47bb862c0f 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -992,6 +992,7 @@ impl Drop for ScannerDefaultCycleGuard { #[derive(Debug, Default)] struct MemoryConfigStore { + display_progress_generation: Option, #[cfg(feature = "diagnostic-read-version-caller")] entry_sources: Mutex>>, #[cfg(feature = "diagnostic-read-version-caller")] @@ -5028,6 +5029,9 @@ async fn scanner_backup_sync_distinguishes_movement_from_missing_or_corrupt_prim #[async_trait::async_trait] impl crate::ScannerConfigObjectDelete for MemoryConfigStore { + async fn scanner_data_usage_movement_generation(&self) -> Option { + self.display_progress_generation + } async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult { let key = memory_config_key(bucket, object); let mut objects = self.objects.lock().await; diff --git a/crates/scanner/src/scanner/tests/paged_checkpoint/save_review.rs b/crates/scanner/src/scanner/tests/paged_checkpoint/save_review.rs index a37a81dbd8..4ebe4a5988 100644 --- a/crates/scanner/src/scanner/tests/paged_checkpoint/save_review.rs +++ b/crates/scanner/src/scanner/tests/paged_checkpoint/save_review.rs @@ -3,6 +3,98 @@ use super::*; +#[tokio::test(start_paused = true)] +async fn paged_backup_preparation_stays_inside_the_backup_budget() { + let store = Arc::new(MemoryConfigStore::default()); + let mut cache = seeded(); + let revisions = DataUsageCache::read_revisions(store.clone(), NAME) + .await + .expect("initial heads"); + cache + .save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new()) + .await + .expect("seed primary and backup"); + let revisions = DataUsageCache::read_revisions(store.clone(), NAME) + .await + .expect("committed heads"); + cache.cache.get_mut("bucket/prefix-04095").expect("last leaf").size += 17; + let backup = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}.bkp")); + store.delayed_gets.lock().await.insert(backup, Duration::from_secs(60)); + let started = tokio::time::Instant::now(); + cache + .save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new()) + .await + .expect("primary commit survives a slow best-effort backup"); + assert!( + started.elapsed() <= Duration::from_secs(5), + "backup preparation exceeded its entire budget" + ); + let mut loaded = DataUsageCache::default(); + loaded + .load(store, NAME) + .await + .expect("primary remains readable after backup cancellation"); + assert_eq!(loaded.checked_flatten("bucket").expect("primary graph").size, 4096 * 4097 / 2 + 17); +} + +#[tokio::test] +async fn display_progress_retained_compacted_handoff_preserves_measurement_time() { + let store = Arc::new(MemoryConfigStore { + display_progress_generation: Some(0), + ..Default::default() + }); + let mut cache = empty_bucket_checkpoint(); + let plan = DataUsageScanPlanDigest([1; 32]); + cache.info.scan_progress = Some(crate::DataUsageScanProgress { + started_plan: plan, + requested_plan: plan, + }); + let first_source = format!("{}|disk-a", uuid::Uuid::from_u128(1)); + cache.info.scan_directory_source = Some(first_source); + cache.info.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)); + let [receipt, _] = crate::scanner_folder::DirectoryCheckpoint::storage_fixture(&cache.info, 4, [2, 3]).await; + cache.info.scan_directory_work = vec![receipt]; + let revisions = DataUsageCache::read_revisions(store.clone(), NAME) + .await + .expect("initial heads"); + cache + .save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new()) + .await + .expect("save compacted measurement"); + let progress_path = memory_config_key(RUSTFS_META_BUCKET, &rustfs_data_usage::bucket_scan_progress_path("bucket")); + let original: rustfs_data_usage::BucketScanProgressSource = + serde_json::from_slice(&store.objects.lock().await[&progress_path]).expect("original compacted sample"); + assert_eq!(original.objects, Some(2)); + for turn in 1..=2 { + let mut cold = DataUsageCache::default(); + let revisions = cold + .load_with_revisions(store.clone(), NAME) + .await + .expect("cold shared handoff"); + cold.activate_directory_source( + format!("{}|disk-b", uuid::Uuid::from_u128(2)), + &HashSet::from(["disk-a".into(), "disk-b".into()]), + ); + cold.replace("bucket", "", DataUsageEntry::default()); + cold.info.leader_epoch += 1; + cold.info.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10 + turn * 60)); + assert!( + !cold.info.scan_directory_work.is_empty(), + "handoff must retain the previous compacted receipt" + ); + cold.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new()) + .await + .expect("save enumerating executor"); + let sample: rustfs_data_usage::BucketScanProgressSource = + serde_json::from_slice(&store.objects.lock().await[&progress_path]).expect("retained compacted sample"); + assert_eq!((sample.objects, sample.size), (original.objects, original.size)); + assert_eq!( + sample.updated_at, original.updated_at, + "checkpoint saves cannot refresh retained measurements" + ); + } +} + #[tokio::test] async fn indefinite_partial_checkpoints_keep_unreachable_bytes_bounded() { let store = Arc::new(MemoryConfigStore::default()); diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index ed66148dc1..5ca0056910 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -3937,6 +3937,7 @@ pub(crate) async fn scan_data_folder_scoped( new_cache.info.scan_directory_source = None; new_cache.info.scan_source_trees.clear(); new_cache.info.scan_coverage_receipt = None; + new_cache.info.scan_coverage_measured_at = None; if had_scan_checkpoint { global_metrics().record_scanner_checkpoint_cleared(); } @@ -3997,6 +3998,7 @@ pub(crate) async fn scan_data_folder_scoped( new_cache.info.scan_checkpoint = None; new_cache.info.scan_resume_after = None; new_cache.info.scan_coverage_receipt = None; + new_cache.info.scan_coverage_measured_at = None; } if raw_enumeration_page_index.is_some() { new_cache.info.scan_raw_enumeration_page_index = raw_enumeration_page_index; @@ -4004,6 +4006,7 @@ pub(crate) async fn scan_data_folder_scoped( new_cache.info.scan_checkpoint = None; new_cache.info.scan_resume_after = None; new_cache.info.scan_coverage_receipt = None; + new_cache.info.scan_coverage_measured_at = None; } if partial_cache_is_useful(&root, pending_heals_changed) || !new_cache.info.scan_directory_work.is_empty() diff --git a/crates/scanner/src/scanner_folder/directory_cursor.rs b/crates/scanner/src/scanner_folder/directory_cursor.rs index 7c9434aa45..71aaa342a0 100644 --- a/crates/scanner/src/scanner_folder/directory_cursor.rs +++ b/crates/scanner/src/scanner_folder/directory_cursor.rs @@ -195,6 +195,7 @@ pub(super) struct DirectoryWork { // The final compaction threshold caps how many paths this list can hold. pub(super) linked: Vec, pub(super) compacted_total: super::DataUsageEntry, + pub(super) measured_at: Option, pub(super) gap: bool, compacted: bool, recorder: Option, @@ -225,6 +226,9 @@ impl DirectoryWork { pub(super) fn acknowledge(&mut self, name: &str, linked: bool, compacted: bool) { if self.names.acknowledge() { self.compacted = compacted; + if compacted { + self.measured_at = crate::data_usage_define::scan_measurement_time_millis(std::time::SystemTime::now()); + } if linked && !compacted { self.linked.push(name.to_owned()); } @@ -236,6 +240,7 @@ impl DirectoryWork { self.names.rewind(); self.linked.clear(); self.compacted_total = super::DataUsageEntry::default(); + self.measured_at = None; self.gap = false; } } @@ -467,6 +472,7 @@ impl ScannerDirectory { if let Some(aggregate) = &saved.aggregate { work.compacted = true; work.compacted_total = aggregate.value.clone(); + work.measured_at = saved.measured_at; } let physical_path: Arc = Arc::from(Path::new(path)); if let Some(part) = &saved.part { diff --git a/crates/scanner/src/scanner_folder/directory_cursor/checkpoint.rs b/crates/scanner/src/scanner_folder/directory_cursor/checkpoint.rs index 7e9d5ac991..5457957bb5 100644 --- a/crates/scanner/src/scanner_folder/directory_cursor/checkpoint.rs +++ b/crates/scanner/src/scanner_folder/directory_cursor/checkpoint.rs @@ -9,6 +9,7 @@ use super::{DirectoryScope, DirectoryStamp, DirectoryWork, FolderScanMode, nativ use crate::data_usage_define::{ DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageScanIdentity, DataUsageScanProgress, }; +use serde::ser::{SerializeMap, SerializeStruct}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -109,7 +110,7 @@ pub(super) struct Aggregate { /// Immutable directory membership and the last fully counted compacted child. /// The active child's provisional contribution is never part of this receipt. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Deserialize)] pub struct DirectoryCheckpoint { version: u8, pub(super) parent: String, @@ -124,6 +125,53 @@ pub struct DirectoryCheckpoint { pub(super) aggregate: Option, endpoint: String, digest: [u8; 32], + /// Diagnostic time of the last acknowledged aggregate, never a save time. + #[serde(default)] + pub(super) measured_at: Option, +} + +impl Serialize for DirectoryCheckpoint { + fn serialize(&self, serializer: S) -> Result { + if self.measured_at.is_none() { + // Enumeration-only receipts retain the compact version-one bytes + // so optional diagnostics do not enlarge each name-budget quantum. + let mut state = serializer.serialize_struct("DirectoryCheckpoint", 13)?; + state.serialize_field("version", &self.version)?; + state.serialize_field("parent", &self.parent)?; + state.serialize_field("scope", &self.scope)?; + state.serialize_field("physical", &self.physical)?; + state.serialize_field("pages", &self.pages)?; + state.serialize_field("anchor", &self.anchor)?; + state.serialize_field("entries_read", &self.entries_read)?; + state.serialize_field("sealed", &self.sealed)?; + state.serialize_field("part", &self.part)?; + state.serialize_field("metadata", &self.metadata)?; + state.serialize_field("aggregate", &self.aggregate)?; + state.serialize_field("endpoint", &self.endpoint)?; + state.serialize_field("digest", &self.digest)?; + return state.end(); + } + // Version-one readers accept named fields and ignore this additive + // diagnostic field. Keep the existing coverage checksum unchanged. + let mut map = serializer.serialize_map(Some(13 + usize::from(self.measured_at.is_some())))?; + map.serialize_entry("version", &self.version)?; + map.serialize_entry("parent", &self.parent)?; + map.serialize_entry("scope", &self.scope)?; + map.serialize_entry("physical", &self.physical)?; + map.serialize_entry("pages", &self.pages)?; + map.serialize_entry("anchor", &self.anchor)?; + map.serialize_entry("entries_read", &self.entries_read)?; + map.serialize_entry("sealed", &self.sealed)?; + map.serialize_entry("part", &self.part)?; + map.serialize_entry("metadata", &self.metadata)?; + map.serialize_entry("aggregate", &self.aggregate)?; + map.serialize_entry("endpoint", &self.endpoint)?; + map.serialize_entry("digest", &self.digest)?; + if let Some(measured_at) = self.measured_at { + map.serialize_entry("measured_at", &measured_at)?; + } + map.end() + } } /// Physical progress orders only callbacks from the same directory generation. @@ -252,8 +300,8 @@ impl DirectoryCheckpoint { }) } /// A display contribution contains only acknowledged compacted children. - pub(crate) fn display_usage(&self, info: &DataUsageCacheInfo) -> Option<(DirectoryProgress, &DataUsageEntry)> { - Some((self.progress(info)?, &self.aggregate.as_ref()?.value)) + pub(crate) fn display_usage(&self, info: &DataUsageCacheInfo) -> Option<(DirectoryProgress, &DataUsageEntry, Option)> { + Some((self.progress(info)?, &self.aggregate.as_ref()?.value, self.measured_at)) } pub(crate) fn endpoint(&self) -> &str { &self.endpoint @@ -352,6 +400,7 @@ impl Recorder { aggregate, endpoint: self.endpoint.clone(), digest: [0; 32], + measured_at: work.measured_at, }; checkpoint.checksum().map(|digest| { checkpoint.digest = digest; diff --git a/crates/scanner/src/scanner_folder/directory_cursor/checkpoint/storage_fixture.rs b/crates/scanner/src/scanner_folder/directory_cursor/checkpoint/storage_fixture.rs index 2f5ab1c07a..f804097ef6 100644 --- a/crates/scanner/src/scanner_folder/directory_cursor/checkpoint/storage_fixture.rs +++ b/crates/scanner/src/scanner_folder/directory_cursor/checkpoint/storage_fixture.rs @@ -5,6 +5,10 @@ use super::*; impl DirectoryCheckpoint { pub(crate) async fn wide_storage_fixture(info: &DataUsageCacheInfo) -> [Self; 2] { + Self::storage_fixture(info, 300_000, [121_034, 121_035]).await + } + + pub(crate) async fn storage_fixture(info: &DataUsageCacheInfo, width: usize, completed: [usize; 2]) -> [Self; 2] { let scope = Scope { bucket: info.name.clone(), source: info.source.expect("fixture source"), @@ -24,20 +28,23 @@ impl DirectoryCheckpoint { modified: SystemTime::UNIX_EPOCH, }; let mut recorder = Recorder::new(slot.clone(), &info.name, scope, endpoint, Some(physical), None); - recorder.entries_read = 300_000; + recorder.entries_read = u64::try_from(width).expect("fixture width"); let mut work = DirectoryWork { compacted: true, ..Default::default() }; - for i in 0..300_000 { + for i in 0..width { work.names.add(format!("obj-{i:06}")); work.names.flush_full().await.expect("record real immutable name page"); } work.names.finish(); - [121_034, 121_035].map(|objects| { + completed.map(|objects| { work.names.seek_after(Some(&format!("obj-{:06}", objects - 1))); work.compacted_total.objects = objects; work.compacted_total.size = objects * 1024; + work.measured_at = info + .last_update + .and_then(crate::data_usage_define::scan_measurement_time_millis); recorder.save(&work); slot.lock() .expect("fixture slot") @@ -46,3 +53,60 @@ impl DirectoryCheckpoint { }) } } + +#[derive(Serialize, Deserialize)] +struct LegacyCheckpoint { + version: u8, + parent: String, + scope: Scope, + physical: Option, + pages: Vec, + anchor: Option, + entries_read: u64, + sealed: bool, + part: Option, + metadata: bool, + aggregate: Option, + endpoint: String, + digest: [u8; 32], +} + +#[tokio::test] +async fn display_progress_measurement_metadata_keeps_version_one_checkpoint_compatibility() { + let plan = crate::DataUsageScanPlanDigest([1; 32]); + let info = DataUsageCacheInfo { + name: "bucket".into(), + source: Some(DataUsageCacheSource::new(0, 0)), + scan_identity: Some(DataUsageScanIdentity { + version: crate::data_usage_define::DATA_USAGE_SCAN_CHECKPOINT_VERSION, + bucket_incarnation: uuid::Uuid::from_u128(1), + set_layout: crate::DataUsageScanPlanDigest([7; 32]), + publication_epoch: 0, + tier_registry_generation: 1, + scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Normal, + }), + scan_progress: Some(DataUsageScanProgress { + started_plan: plan, + requested_plan: plan, + }), + scan_directory_source: Some("disk-a".into()), + last_update: Some(SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(10)), + ..Default::default() + }; + let [checkpoint, _] = DirectoryCheckpoint::storage_fixture(&info, 4, [2, 3]).await; + assert_eq!(checkpoint.measured_at, Some(10_000)); + let bytes = rmp_serde::to_vec(&checkpoint).expect("encode additive named checkpoint"); + let legacy: LegacyCheckpoint = rmp_serde::from_slice(&bytes).expect("version-one reader ignores measurement metadata"); + let old_bytes = rmp_serde::to_vec(&legacy).expect("encode legacy positional checkpoint"); + let recovered: DirectoryCheckpoint = rmp_serde::from_slice(&old_bytes).expect("new reader accepts legacy receipt"); + assert_eq!(recovered.measured_at, None); + assert_eq!( + rmp_serde::to_vec(&recovered).expect("encode receipt without measurement metadata"), + old_bytes, + "receipts without diagnostic metadata keep the compact legacy encoding" + ); + assert!( + recovered.is_bound_to(&info), + "diagnostic metadata must not change the version-one coverage checksum" + ); +} diff --git a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs index 24ec99276f..42b469be82 100644 --- a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs +++ b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs @@ -358,6 +358,7 @@ async fn display_progress_keeps_reference_during_executor_enumeration_then_repla let (mut cache, _) = bound_checkpoint(); let original_time = SystemTime::UNIX_EPOCH + Duration::from_secs(10); cache.info.last_update = Some(original_time); + cache.info.scan_coverage_measured_at = Some(10_000); cache.info.scan_directory_source = Some("first-executor".into()); let mut revisions = DataUsageCache::default() .load_with_revisions(writer.clone(), CACHE_NAME) diff --git a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/deep_compacted.rs b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/deep_compacted.rs index 8538c4fe59..5f0e90f008 100644 --- a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/deep_compacted.rs +++ b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/deep_compacted.rs @@ -36,7 +36,16 @@ async fn scan( (outcome, budget) } +async fn read_display_progress(store: &FixtureStore) -> Option { + match tokio::fs::read(store.root.join("progress")).await { + Ok(bytes) => Some(serde_json::from_slice(&bytes).expect("decode independent display progress")), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => panic!("read durable display progress: {error}"), + } +} + async fn save_reload(store: &Arc, cache: &DataUsageCache) -> DataUsageCache { + let previous = read_display_progress(store).await; let revisions = DataUsageCache::default() .load_with_revisions(store.clone(), CACHE_NAME) .await @@ -47,12 +56,12 @@ async fn save_reload(store: &Arc, cache: &DataUsageCache) -> DataU .expect("save compacted checkpoint through real codec and revision checks"); let loaded = store.strict_load().await; let restarted_store = FixtureStore::reopen(store.root.clone()); - let progress_bytes = tokio::fs::read(restarted_store.root.join("progress")) - .await - .expect("durable display progress survives a cold store reopen"); - let progress: rustfs_data_usage::BucketScanProgressSource = - serde_json::from_slice(&progress_bytes).expect("decode independent display progress"); - assert_eq!(Some(&progress), loaded.display_scan_progress(0).as_ref()); + let progress = read_display_progress(&restarted_store).await; + if let Some(expected) = loaded.display_scan_progress(0) { + assert_eq!(progress, Some(expected), "accepted progress survives a cold store reopen"); + } else { + assert_eq!(progress, previous, "unverified legacy coverage must not publish a new sample"); + } assert_eq!(loaded.info.snapshot_complete, cache.info.snapshot_complete); assert_eq!(loaded.info.scan_checkpoint, cache.info.scan_checkpoint); assert_eq!(loaded.info.scan_identity, cache.info.scan_identity); diff --git a/docs/operations/scanner-usage-statistics.md b/docs/operations/scanner-usage-statistics.md index aa29701259..70a6de8e5d 100644 --- a/docs/operations/scanner-usage-statistics.md +++ b/docs/operations/scanner-usage-statistics.md @@ -41,10 +41,12 @@ The companion [auto-testing scanner usage suite](https://github.com/rustfs/auto- ## Console bucket scan progress -`GET /rustfs/admin/v3/accountinfo` may attach `scan_progress` to each authorized bucket. Its `objects` and `size` describe accepted scanned coverage, not the bucket total, a lower bound on current contents, or a percentage. Concurrent changes can invalidate any such comparison. `stage` distinguishes `enumerating`, `scanning`, `awaiting_sources` and `completed`; completion means all logical sources supplied a completed traversal sample, without granting a usage certificate. Enumeration alone leaves both counters absent. `sources_observed` and `sources_total` make missing sets explicit. `updated_at` is the oldest contributing checkpoint time in Unix milliseconds. +`GET /rustfs/admin/v3/accountinfo` may attach `scan_progress` to each authorized bucket. Its `objects` and `size` describe accepted scanned coverage, not the bucket total, a lower bound on current contents, or a percentage. Concurrent changes can invalidate any such comparison. `stage` distinguishes `enumerating`, `scanning`, `awaiting_sources` and `completed`; completion means all logical sources supplied a completed traversal sample, without granting a usage certificate. Enumeration alone leaves both counters absent. `sources_observed` and `sources_total` make missing sets explicit. `updated_at` is the oldest contributing measurement time in Unix milliseconds. Saving an unchanged frontier or retained compacted receipt, including after a cold executor handoff, does not refresh that time. Legacy receipts without measurement metadata remain reusable for scanning but cannot supply a new numeric display sample with a fabricated age. The scanner saves compact progress companions on their contributing erasure sets after committing bucket checkpoints. Retries replace a sample instead of adding deltas, and physical replicas are alternative views of one logical source. A new executor that has not yet measured another range retains the prior accepted sample and its original timestamp. Reader-local admission epochs and wakeup counters never fence a shared diagnostic sample; its movement scope comes from persisted pool/rebalance metadata. Missing, unsupported or corrupt companions do not affect checkpoint recovery or complete usage. Admin reads bind companions to the current bucket incarnation, set layout and durable movement generation, use bounded concurrent reads, and never list S3 objects or load the full checkpoint graph. The response keeps existing `objects` and `size` fields independent; Console retains their prior values and shows scan progress separately. Optional `usage_updated_at` preserves the original bucket certificate time, independently of the progress timestamp. Older servers and clients can omit or ignore these additive fields. The two-second Admin read budget reserves its final quarter for movement-scope validation. Each validated source contributes immediately, so a stalled source or another bucket's identity lookup cannot erase completed reads. A bucket with missing sources reports `awaiting_sources`; unavailable final scope validation still omits the diagnostic view. Progress extraction shares the checkpoint's serialized buffers, including paged payloads, with at most one dedicated CPU worker per process. It releases movement admission after capturing the durable generation and leaves native I/O workers available for primary saves. Waiting for the worker and preparing its input are included in the two-second diagnostic preparation budget; cancellation is checked during decoding. A nonpreemptible validation may finish after its waiter expires, retaining one decoded checkpoint until it exits. Its resource permit remains held, preventing repeated saves from accumulating workers or decoded copies, and its expired result is not published. Primary and companion publication retain their existing epoch and revision checks; scanner cycle budgets are unchanged. + +The adaptive best-effort backup budget covers preparation, manifest publication, reconciliation and page reclamation together. Exhausting it stops waiting for the backup without undoing a successful primary checkpoint. An uncertain backup outcome can be recovered from storage on a later turn; it does not extend the current turn's budget. diff --git a/rustfs/src/admin/handlers/account_info.rs b/rustfs/src/admin/handlers/account_info.rs index 9f4ed4e6fa..3bc34a64fc 100644 --- a/rustfs/src/admin/handlers/account_info.rs +++ b/rustfs/src/admin/handlers/account_info.rs @@ -30,7 +30,7 @@ use http::header::CONTENT_TYPE; use http::{HeaderMap, HeaderValue}; use hyper::{Method, StatusCode}; use matchit::Params; -use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo}; +use rustfs_data_usage::{BucketScanStage, BucketUsageInfo, BucketUsageValueState, DataUsageInfo}; use rustfs_gateway_types::dto::Status; use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration; use rustfs_gateway_types::persistence::PersistedReplicationConfiguration; @@ -91,7 +91,15 @@ fn apply_usage_to_bucket_access_info(bucket_info: &mut rustfs_madmin::BucketAcce bucket_info.size = Some(usage.size); bucket_info.objects = Some(usage.objects_count); - bucket_info.usage_value_state = usage.usage_value_state; + bucket_info.usage_value_state = usage.usage_value_state.map(|state| { + match state { + BucketUsageValueState::Certified => "certified", + BucketUsageValueState::Carried => "carried", + BucketUsageValueState::Observed => "observed", + BucketUsageValueState::Unknown => "unknown", + } + .to_owned() + }); bucket_info.usage_updated_at = usage.usage_certification.and_then(|certificate| { u64::try_from( certificate @@ -106,6 +114,23 @@ fn apply_usage_to_bucket_access_info(bucket_info: &mut rustfs_madmin::BucketAcce bucket_info.object_versions_histogram = Some(usage.object_versions_histogram.clone()); } +fn bucket_scan_progress_response(progress: rustfs_data_usage::BucketScanProgress) -> rustfs_madmin::BucketScanProgress { + let stage = match progress.stage { + BucketScanStage::Enumerating => "enumerating", + BucketScanStage::Scanning => "scanning", + BucketScanStage::AwaitingSources => "awaiting_sources", + BucketScanStage::Completed => "completed", + }; + rustfs_madmin::BucketScanProgress { + stage: stage.to_owned(), + updated_at: progress.updated_at, + objects: progress.objects, + size: progress.size, + sources_observed: progress.sources_observed, + sources_total: progress.sources_total, + } +} + fn object_lock_config_enabled(config: &ObjectLockConfiguration) -> bool { config .object_lock_enabled @@ -323,7 +348,7 @@ impl Operation for AccountInfoHandler { .collect::>(); let mut progress = load_bucket_scan_progress_for_admin(store, &visible_buckets, &data_usage_info).await; for bucket in &mut account_info.buckets { - bucket.scan_progress = progress.remove(&bucket.name); + bucket.scan_progress = progress.remove(&bucket.name).map(bucket_scan_progress_response); } let data = serde_json::to_vec(&AccountInfoResponse { @@ -442,14 +467,14 @@ mod tests { fn accountinfo_partial_progress_never_fills_total_usage_fields() { let mut bucket_info = rustfs_madmin::BucketAccessInfo { name: "scanning".into(), - scan_progress: Some(rustfs_data_usage::BucketScanProgress { + scan_progress: Some(bucket_scan_progress_response(rustfs_data_usage::BucketScanProgress { stage: rustfs_data_usage::BucketScanStage::Scanning, updated_at: 1000, objects: Some(7), size: Some(71), sources_observed: 1, sources_total: 2, - }), + })), ..Default::default() }; apply_usage_to_bucket_access_info(&mut bucket_info, None); @@ -474,6 +499,49 @@ mod tests { assert_eq!(bucket_info.scan_progress.expect("separate partial view").objects, Some(7)); } + #[test] + fn accountinfo_scan_progress_projection_preserves_wire_contract() { + for (stage, objects, size, sources_observed) in [ + (BucketScanStage::Enumerating, None, None, 2), + (BucketScanStage::Scanning, Some(7), Some(71), 2), + (BucketScanStage::AwaitingSources, Some(0), Some(0), 1), + (BucketScanStage::Completed, Some(7), Some(71), 2), + ] { + let progress = rustfs_data_usage::BucketScanProgress { + stage, + updated_at: 1000, + objects, + size, + sources_observed, + sources_total: 2, + }; + let expected = serde_json::to_value(&progress).expect("domain progress wire contract"); + let response = bucket_scan_progress_response(progress); + assert_eq!(serde_json::to_value(response).expect("SDK progress response"), expected); + } + } + + #[test] + fn accountinfo_usage_provenance_projection_preserves_wire_contract() { + for state in [ + BucketUsageValueState::Certified, + BucketUsageValueState::Carried, + BucketUsageValueState::Observed, + BucketUsageValueState::Unknown, + ] { + let usage = BucketUsageInfo { + usage_value_state: Some(state), + ..Default::default() + }; + let mut bucket = rustfs_madmin::BucketAccessInfo::default(); + apply_usage_to_bucket_access_info(&mut bucket, Some(&usage)); + assert_eq!( + serde_json::to_value(bucket).expect("SDK usage provenance")["usage_value_state"], + serde_json::to_value(state).expect("domain provenance wire contract") + ); + } + } + #[test] fn accountinfo_data_usage_error_message_is_generic() { let err = map_data_usage_result::<&str>(Err("sensitive disk path")).expect_err("load failure must reach the response");