fix(scanner): preserve progress age and bound backup work

Retain accepted measurement times across cold source handoffs and checkpoint resaves. Bound preparation, reconciliation and reclamation with the same adaptive best-effort backup deadline.

Keep the Admin SDK a leaf by projecting server progress into SDK-owned DTOs without changing the JSON contract. Cover backup deadlines, retained compacted receipts, legacy codecs and future diagnostic labels with regressions.
This commit is contained in:
cxymds
2026-11-21 22:10:37 +08:00
parent 29921163e3
commit 7dc759f9cf
18 changed files with 564 additions and 35 deletions
Generated
-1
View File
@@ -10821,7 +10821,6 @@ dependencies = [
"jiff 1.0.1",
"reqwest",
"rmp-serde",
"rustfs-data-usage",
"rustfs-signer",
"serde",
"serde_json",
-1
View File
@@ -42,7 +42,6 @@ http.workspace = true
hyper = { workspace = true, features = ["http2", "http1", "server"] }
reqwest = { workspace = true, features = ["json"] }
rustfs-signer.workspace = true
rustfs-data-usage.workspace = true
jiff = { workspace = true, features = ["serde"] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true, features = ["raw_value"] }
+44 -2
View File
@@ -413,9 +413,10 @@ pub struct BucketAccessInfo {
pub name: String,
/// Display-only accepted scan progress, never a replacement for total usage.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scan_progress: Option<rustfs_data_usage::BucketScanProgress>,
pub scan_progress: Option<BucketScanProgress>,
/// Value provenance: certified, carried, observed or unknown. Preserve future labels.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage_value_state: Option<rustfs_data_usage::BucketUsageValueState>,
pub usage_value_state: Option<String>,
/// Unix milliseconds of the original complete statistic, when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage_updated_at: Option<u64>,
@@ -437,6 +438,20 @@ pub struct BucketAccessInfo {
pub access: AccountAccess,
}
/// Accepted scan coverage for display, independent of total usage and quotas.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct BucketScanProgress {
/// Observation stage. Clients must preserve unfamiliar diagnostic labels.
pub stage: String,
/// Oldest contributing measurement time in Unix milliseconds.
pub updated_at: u64,
/// Enumeration alone leaves both counters absent.
pub objects: Option<u64>,
pub size: Option<u64>,
pub sources_observed: u64,
pub sources_total: u64,
}
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct BucketDetails {
pub versioning: bool,
@@ -762,6 +777,33 @@ mod tests {
assert!(!obj.contains_key("objects"));
assert!(!obj.contains_key("object_sizes_histogram"));
assert!(!obj.contains_key("object_versions_histogram"));
assert!(!obj.contains_key("scan_progress"));
assert!(!obj.contains_key("usage_value_state"));
assert!(!obj.contains_key("usage_updated_at"));
}
#[test]
fn bucket_access_info_scan_progress_does_not_fill_total_usage() {
let mut wire = serde_json::to_value(BucketAccessInfo::default()).expect("empty account bucket response");
wire["scan_progress"] = serde_json::json!({
"stage": "scanning", "updated_at": 1000, "objects": 7, "size": 71,
"sources_observed": 1, "sources_total": 2,
});
let bucket: BucketAccessInfo = serde_json::from_value(wire.clone()).expect("decode scan progress response");
let value = serde_json::to_value(bucket).expect("encode scan progress response");
assert_eq!(value, wire, "scan coverage must round-trip without synthesizing total usage");
}
#[test]
fn bucket_access_info_preserves_future_progress_and_provenance_values() {
let mut wire = serde_json::to_value(BucketAccessInfo::default()).expect("empty account bucket response");
wire["scan_progress"] = serde_json::json!({
"stage": "future_stage", "updated_at": 1000, "objects": null, "size": null,
"sources_observed": 1, "sources_total": 2,
});
wire["usage_value_state"] = serde_json::json!("future_provenance");
let bucket: BucketAccessInfo = serde_json::from_value(wire.clone()).expect("SDK accepts future diagnostic labels");
assert_eq!(serde_json::to_value(bucket).expect("preserve diagnostic labels"), wire);
}
/// Wire pin (rustfs/backlog#1306): populated usage stats keep their
+26
View File
@@ -524,6 +524,10 @@ pub struct DataUsageScanCoverageReceipt {
pub digest: [u8; 32],
}
pub(crate) fn scan_measurement_time_millis(time: SystemTime) -> Option<u64> {
u64::try_from(time.duration_since(SystemTime::UNIX_EPOCH).ok()?.as_millis()).ok()
}
impl Serialize for DataUsageScanCoverageReceipt {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut map = serializer.serialize_map(Some(2))?;
@@ -1063,6 +1067,9 @@ pub struct DataUsageCacheInfo {
pub scan_bucket_observations: HashMap<String, DataUsageBucketObservation>,
#[serde(default, skip_serializing_if = "HashMap::is_empty")]
pub scan_bucket_value_origins: HashMap<String, DataUsageBucketValueOrigin>,
/// Original accepted-frontier measurement time; absent on legacy checkpoints.
#[serde(default)]
pub scan_coverage_measured_at: Option<u64>,
}
impl Serialize for DataUsageCacheInfo {
@@ -1083,6 +1090,7 @@ impl Serialize for DataUsageCacheInfo {
+ usize::from(self.scan_progress.is_some())
+ usize::from(self.scan_usage_watch.is_some())
+ usize::from(self.scan_coverage_receipt.is_some())
+ usize::from(self.scan_coverage_measured_at.is_some())
+ usize::from(self.scan_coverage_digest.is_some())
+ usize::from(self.tier_registry_generation.is_some())
+ usize::from(!self.size_reconciliation.is_empty())
@@ -1138,6 +1146,9 @@ impl Serialize for DataUsageCacheInfo {
if let Some(receipt) = &self.scan_coverage_receipt {
state.serialize_entry("scan_coverage_receipt", receipt)?;
}
if let Some(measured_at) = self.scan_coverage_measured_at {
state.serialize_entry("scan_coverage_measured_at", &measured_at)?;
}
state.serialize_entry("pending_heals", &self.pending_heals)?;
state.serialize_entry("object_lock", &crate::object_lock_cache::ObjectLockCache(&self.object_lock))?;
state.serialize_entry("source", &self.source)?;
@@ -1569,6 +1580,7 @@ impl DataUsageCache {
self.info.scan_directory_source = None;
self.info.scan_source_trees.clear();
self.info.scan_coverage_receipt = None;
self.info.scan_coverage_measured_at = None;
}
// Old readers do not understand coverage sweeps. An absent plan makes
// their existing prepare path rebuild instead of promoting mixed data.
@@ -1748,9 +1760,22 @@ impl DataUsageCache {
pub(crate) fn seal_scan_frontier(&mut self, frontier: Option<&str>) -> Result<(), serde_json::Error> {
if self.info.scan_progress.is_none() {
self.info.scan_coverage_receipt = None;
self.info.scan_coverage_measured_at = None;
return Ok(());
}
let frontier = frontier.filter(|path| path.len() <= 16 * 1024 && self.find(path).is_some());
let measured_at = frontier.and_then(|through| {
if self
.info
.scan_coverage_receipt
.as_ref()
.is_some_and(|previous| previous.through == through)
{
self.info.scan_coverage_measured_at
} else {
scan_measurement_time_millis(SystemTime::now())
}
});
self.info.scan_coverage_receipt = match frontier {
Some(through) => Some(DataUsageScanCoverageReceipt {
through: clone_cache_value(through),
@@ -1758,6 +1783,7 @@ impl DataUsageCache {
}),
None => None,
};
self.info.scan_coverage_measured_at = measured_at;
self.info.scan_resume_after = frontier.map(clone_cache_value);
let reason = self
.info
@@ -132,13 +132,16 @@ impl DataUsageCache {
.filter(|observation| Some(observation.completed_at) == self.info.last_update && !self.has_scan_position())
.map(|observation| observation.entry.clone())
};
let mut accepted = completed.as_ref().map(|entry| (entry.objects, entry.size));
let mut accepted = completed.as_ref().map(|entry| (entry.objects, entry.size, updated_at));
if completed.is_none() {
let frontier = self.validated_scan_frontier();
let mut replicas = BTreeMap::<&str, Vec<_>>::new();
for checkpoint in &self.info.scan_directory_work {
if let Some((coverage, entry)) = checkpoint.display_usage(&self.info) {
replicas.entry(checkpoint.endpoint()).or_default().push((coverage, entry));
if let Some((coverage, entry, measured_at)) = checkpoint.display_usage(&self.info) {
replicas
.entry(checkpoint.endpoint())
.or_default()
.push((coverage, entry, measured_at));
}
}
if let Some(active) = self.info.scan_directory_source.as_deref() {
@@ -150,7 +153,9 @@ impl DataUsageCache {
receipts.sort_by(|a, b| a.0.parent.split('/').cmp(b.0.parent.split('/')));
let mut selected = Vec::new();
let mut subtotal: Option<(usize, usize)> = None;
for (coverage, entry) in &receipts {
let mut measured_at: Option<u64> = None;
let mut known_time = true;
for (coverage, entry, time) in &receipts {
if selected.iter().any(|parent: &&crate::scanner_folder::DirectoryProgress| {
parent.parent == coverage.parent || parent.covers(&coverage.parent)
}) {
@@ -163,6 +168,11 @@ impl DataUsageCache {
}
let (objects, size) = subtotal.unwrap_or_default();
subtotal = Some((objects.checked_add(entry.objects)?, size.checked_add(entry.size)?));
if let Some(time) = time {
measured_at = Some(measured_at.map_or(*time, |previous| previous.min(*time)));
} else {
known_time = false;
}
selected.push(coverage);
}
if Some(endpoint) == self.info.scan_directory_source.as_deref()
@@ -171,16 +181,30 @@ impl DataUsageCache {
// A validated receipt signs this prefix. Sum each scalar
// once, excluding the ranges represented by compacted work.
self.checked_flatten_complete_scope(&self.info.name)?;
let mut contributed = false;
for (path, entry) in &self.cache {
if scan_frontier_covers_path(path, through) && !selected.iter().any(|proof| proof.covers(path)) {
let (objects, size) = subtotal.unwrap_or_default();
subtotal = Some((objects.checked_add(entry.objects)?, size.checked_add(entry.size)?));
contributed = true;
}
}
if contributed {
if let Some(time) = self.info.scan_coverage_measured_at {
measured_at = Some(measured_at.map_or(time, |previous| previous.min(time)));
} else {
known_time = false;
}
}
}
// Replicas overlap. Select one sample; never sum or append them.
if subtotal.is_some_and(|sample| accepted.is_none_or(|previous| sample > previous)) {
accepted = subtotal;
// Legacy coverage without a measurement time cannot borrow a
// newer checkpoint time. Keep the prior companion instead.
if known_time
&& let (Some((objects, size)), Some(time)) = (subtotal, measured_at)
&& accepted.is_none_or(|previous| (objects, size, time) > previous)
{
accepted = Some((objects, size, time));
}
}
}
@@ -195,7 +219,7 @@ impl DataUsageCache {
movement_generation: Some(movement_generation),
scanner_epoch: self.info.leader_epoch,
scanner_cycle: self.info.next_cycle,
updated_at,
updated_at: accepted.map_or(updated_at, |(_, _, time)| time),
stage: if completed.is_some() {
BucketScanStage::Completed
} else if accepted.is_some() {
@@ -203,8 +227,8 @@ impl DataUsageCache {
} else {
BucketScanStage::Enumerating
},
objects: accepted.map(|(objects, _)| u64::try_from(objects)).transpose().ok()?,
size: accepted.map(|(_, size)| u64::try_from(size)).transpose().ok()?,
objects: accepted.map(|(objects, _, _)| u64::try_from(objects)).transpose().ok()?,
size: accepted.map(|(_, size, _)| u64::try_from(size)).transpose().ok()?,
})
}
@@ -480,6 +504,53 @@ mod tests {
cache
}
#[test]
fn display_progress_frontier_resealing_and_cold_source_handoff_preserve_time() {
let mut cache = checkpoint();
cache.replace("bucket", "", DataUsageEntry::default());
cache.replace(
"bucket/a",
"bucket",
DataUsageEntry {
objects: 1,
size: 11,
..Default::default()
},
);
cache.seal_scan_frontier(Some("bucket/a")).expect("seal measured frontier");
cache.info.scan_coverage_measured_at = Some(10_000);
cache.info.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(70));
cache.seal_scan_frontier(Some("bucket/a")).expect("reseal unchanged frontier");
assert_eq!(cache.display_scan_progress(0).expect("unchanged observation").updated_at, 10_000);
let endpoints = std::collections::HashSet::from(["source".to_owned(), "other".to_owned()]);
cache.activate_directory_source("other".into(), &endpoints);
cache.replace("bucket", "", DataUsageEntry::default());
let bytes = cache.marshal_msg().expect("save inactive source with its measurement time");
let mut cold = DataUsageCache::unmarshal(&bytes).expect("cold source handoff");
cold.activate_directory_source("source".into(), &endpoints);
let sample = cold.display_scan_progress(0).expect("restored measured frontier");
assert_eq!((sample.objects, sample.size, sample.updated_at), (Some(1), Some(11), 10_000));
}
#[test]
fn display_progress_legacy_frontier_cannot_borrow_a_new_checkpoint_time() {
let mut cache = checkpoint();
cache.replace("bucket", "", DataUsageEntry::default());
cache.replace(
"bucket/a",
"bucket",
DataUsageEntry {
objects: 1,
size: 11,
..Default::default()
},
);
cache.seal_scan_frontier(Some("bucket/a")).expect("legacy coverage");
cache.info.scan_coverage_measured_at = None;
let sample = cache.display_scan_progress(0).expect("enumeration remains observable");
assert_eq!((sample.objects, sample.size), (None, None));
}
#[test]
fn display_progress_excludes_old_tail_and_uncommitted_ancestors() {
let mut cache = checkpoint();
@@ -813,7 +813,11 @@ impl DataUsageCache {
}
Ok::<_, StorageError>(())
};
if let Err(e) = save_backup.await {
let backup_result = match timeout(backup_timeout_duration, save_backup).await {
Ok(result) => result,
Err(_) => Err(StorageError::other("scanner cache backup save timed out")),
};
if let Err(e) = backup_result {
warn!(
target: "rustfs::scanner::data_usage",
event = EVENT_SCANNER_CACHE_SAVE_STATE,
@@ -5,12 +5,11 @@ use super::*;
/// A linked tree paused on a different physical replica. It contains only
/// traversal fields, so dormant sources cannot recursively contain each other.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[derive(Clone, Debug, Deserialize)]
pub struct SourceTree {
bucket: String,
identity: DataUsageScanIdentity,
started_plan: DataUsageScanPlanDigest,
#[serde(serialize_with = "ordered_entries")]
entries: HashMap<String, DataUsageEntry>,
resume_after: Option<String>,
checkpoint: Option<DataUsageScanCheckpoint>,
@@ -18,6 +17,36 @@ pub struct SourceTree {
cursor: Option<DataUsageRawEnumerationCursor>,
index: Option<RawEnumerationPageIndex>,
ancestors: Vec<RawEnumerationPageIndex>,
#[serde(default)]
coverage_measured_at: Option<u64>,
}
impl Serialize for SourceTree {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut map = serializer.serialize_map(Some(10 + usize::from(self.coverage_measured_at.is_some())))?;
map.serialize_entry("bucket", &self.bucket)?;
map.serialize_entry("identity", &self.identity)?;
map.serialize_entry("started_plan", &self.started_plan)?;
map.serialize_entry("entries", &OrderedEntries(&self.entries))?;
map.serialize_entry("resume_after", &self.resume_after)?;
map.serialize_entry("checkpoint", &self.checkpoint)?;
map.serialize_entry("coverage", &self.coverage)?;
map.serialize_entry("cursor", &self.cursor)?;
map.serialize_entry("index", &self.index)?;
map.serialize_entry("ancestors", &self.ancestors)?;
if let Some(measured_at) = self.coverage_measured_at {
map.serialize_entry("coverage_measured_at", &measured_at)?;
}
map.end()
}
}
struct OrderedEntries<'a>(&'a HashMap<String, DataUsageEntry>);
impl Serialize for OrderedEntries<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
ordered_entries(self.0, serializer)
}
}
impl SourceTree {
@@ -89,6 +118,7 @@ impl DataUsageCache {
resume_after: self.info.scan_resume_after.take(),
checkpoint: self.info.scan_checkpoint.take(),
coverage: self.info.scan_coverage_receipt.take(),
coverage_measured_at: self.info.scan_coverage_measured_at.take(),
cursor: self.info.scan_raw_enumeration_cursor.take(),
index: self.info.scan_raw_enumeration_page_index.take(),
ancestors: std::mem::take(&mut self.info.scan_raw_enumeration_path_page_indexes),
@@ -104,6 +134,7 @@ impl DataUsageCache {
self.cache = tree.entries;
self.info.scan_resume_after = tree.resume_after;
self.info.scan_coverage_receipt = tree.coverage;
self.info.scan_coverage_measured_at = tree.coverage_measured_at;
self.info.scan_checkpoint = tree.checkpoint;
self.info.scan_raw_enumeration_cursor = tree.cursor;
self.info.scan_raw_enumeration_page_index = tree.index;
@@ -122,6 +153,7 @@ impl DataUsageCache {
self.info.scan_resume_after = None;
self.info.scan_checkpoint = None;
self.info.scan_coverage_receipt = None;
self.info.scan_coverage_measured_at = None;
self.info.scan_raw_enumeration_cursor = None;
self.info.scan_raw_enumeration_page_index = None;
self.info.scan_raw_enumeration_path_page_indexes.clear();
@@ -131,3 +163,53 @@ impl DataUsageCache {
fn directory_source_location(source: &str) -> &str {
source.split_once('|').map_or(source, |(_, endpoint)| endpoint)
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Serialize, Deserialize)]
struct LegacySourceTree {
bucket: String,
identity: DataUsageScanIdentity,
started_plan: DataUsageScanPlanDigest,
entries: HashMap<String, DataUsageEntry>,
resume_after: Option<String>,
checkpoint: Option<DataUsageScanCheckpoint>,
coverage: Option<DataUsageScanCoverageReceipt>,
cursor: Option<DataUsageRawEnumerationCursor>,
index: Option<RawEnumerationPageIndex>,
ancestors: Vec<RawEnumerationPageIndex>,
}
#[test]
fn source_tree_measurement_metadata_preserves_legacy_codecs() {
let tree = SourceTree {
bucket: "bucket".into(),
identity: DataUsageScanIdentity {
version: DATA_USAGE_SCAN_CHECKPOINT_VERSION,
bucket_incarnation: uuid::Uuid::from_u128(1),
set_layout: DataUsageScanPlanDigest([2; 32]),
publication_epoch: 0,
tier_registry_generation: 0,
scan_mode: HealScanMode::Normal,
},
started_plan: DataUsageScanPlanDigest([3; 32]),
entries: HashMap::from([("bucket".into(), DataUsageEntry::default())]),
resume_after: None,
checkpoint: None,
coverage: None,
cursor: None,
index: None,
ancestors: Vec::new(),
coverage_measured_at: Some(10_000),
};
let bytes = rmp_serde::to_vec(&tree).expect("encode additive named source tree");
let old: LegacySourceTree = rmp_serde::from_slice(&bytes).expect("old reader ignores diagnostic time");
let bytes = rmp_serde::to_vec(&old).expect("encode old positional source tree");
let recovered: SourceTree = rmp_serde::from_slice(&bytes).expect("new reader accepts old positional source tree");
assert_eq!(recovered.bucket, tree.bucket);
assert_eq!(recovered.entries.len(), 1);
assert_eq!(recovered.coverage_measured_at, None);
}
}
@@ -2924,7 +2924,15 @@ async fn test_checkpoint_budget_covers_retries_and_reconciliation() {
.expect("a successful or reconciled primary must survive a best-effort backup failure");
assert_eq!(store.put_attempts[0].load(Ordering::SeqCst), 3);
assert_eq!(store.put_attempts[1].load(Ordering::SeqCst), 1, "backup writes must not retry");
assert_eq!(*store.backup_reads.lock().await, 1, "backup must finish its bounded reconciliation");
assert_eq!(
*store.backup_reads.lock().await,
0,
"backup reconciliation must not outlive the total backup budget"
);
assert!(
store.puts.lock().await.iter().all(|put| !put.object.ends_with(".bkp")),
"a timed-out backup must not replace the previous fallback"
);
assert!(started.elapsed() < DataUsageCache::persistence_timeout());
}
})
+4
View File
@@ -992,6 +992,7 @@ impl Drop for ScannerDefaultCycleGuard {
#[derive(Debug, Default)]
struct MemoryConfigStore {
display_progress_generation: Option<u64>,
#[cfg(feature = "diagnostic-read-version-caller")]
entry_sources: Mutex<Vec<Option<crate::storage_api::ecstore_diagnostic::ReadVersionEntrySource>>>,
#[cfg(feature = "diagnostic-read-version-caller")]
@@ -5028,6 +5029,9 @@ async fn scanner_backup_sync_distinguishes_movement_from_missing_or_corrupt_prim
#[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
async fn scanner_data_usage_movement_generation(&self) -> Option<u64> {
self.display_progress_generation
}
async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult<ObjectInfo> {
let key = memory_config_key(bucket, object);
let mut objects = self.objects.lock().await;
@@ -3,6 +3,98 @@
use super::*;
#[tokio::test(start_paused = true)]
async fn paged_backup_preparation_stays_inside_the_backup_budget() {
let store = Arc::new(MemoryConfigStore::default());
let mut cache = seeded();
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("initial heads");
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
.await
.expect("seed primary and backup");
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("committed heads");
cache.cache.get_mut("bucket/prefix-04095").expect("last leaf").size += 17;
let backup = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}.bkp"));
store.delayed_gets.lock().await.insert(backup, Duration::from_secs(60));
let started = tokio::time::Instant::now();
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
.await
.expect("primary commit survives a slow best-effort backup");
assert!(
started.elapsed() <= Duration::from_secs(5),
"backup preparation exceeded its entire budget"
);
let mut loaded = DataUsageCache::default();
loaded
.load(store, NAME)
.await
.expect("primary remains readable after backup cancellation");
assert_eq!(loaded.checked_flatten("bucket").expect("primary graph").size, 4096 * 4097 / 2 + 17);
}
#[tokio::test]
async fn display_progress_retained_compacted_handoff_preserves_measurement_time() {
let store = Arc::new(MemoryConfigStore {
display_progress_generation: Some(0),
..Default::default()
});
let mut cache = empty_bucket_checkpoint();
let plan = DataUsageScanPlanDigest([1; 32]);
cache.info.scan_progress = Some(crate::DataUsageScanProgress {
started_plan: plan,
requested_plan: plan,
});
let first_source = format!("{}|disk-a", uuid::Uuid::from_u128(1));
cache.info.scan_directory_source = Some(first_source);
cache.info.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10));
let [receipt, _] = crate::scanner_folder::DirectoryCheckpoint::storage_fixture(&cache.info, 4, [2, 3]).await;
cache.info.scan_directory_work = vec![receipt];
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("initial heads");
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
.await
.expect("save compacted measurement");
let progress_path = memory_config_key(RUSTFS_META_BUCKET, &rustfs_data_usage::bucket_scan_progress_path("bucket"));
let original: rustfs_data_usage::BucketScanProgressSource =
serde_json::from_slice(&store.objects.lock().await[&progress_path]).expect("original compacted sample");
assert_eq!(original.objects, Some(2));
for turn in 1..=2 {
let mut cold = DataUsageCache::default();
let revisions = cold
.load_with_revisions(store.clone(), NAME)
.await
.expect("cold shared handoff");
cold.activate_directory_source(
format!("{}|disk-b", uuid::Uuid::from_u128(2)),
&HashSet::from(["disk-a".into(), "disk-b".into()]),
);
cold.replace("bucket", "", DataUsageEntry::default());
cold.info.leader_epoch += 1;
cold.info.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10 + turn * 60));
assert!(
!cold.info.scan_directory_work.is_empty(),
"handoff must retain the previous compacted receipt"
);
cold.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0, CancellationToken::new())
.await
.expect("save enumerating executor");
let sample: rustfs_data_usage::BucketScanProgressSource =
serde_json::from_slice(&store.objects.lock().await[&progress_path]).expect("retained compacted sample");
assert_eq!((sample.objects, sample.size), (original.objects, original.size));
assert_eq!(
sample.updated_at, original.updated_at,
"checkpoint saves cannot refresh retained measurements"
);
}
}
#[tokio::test]
async fn indefinite_partial_checkpoints_keep_unreachable_bytes_bounded() {
let store = Arc::new(MemoryConfigStore::default());
+3
View File
@@ -3937,6 +3937,7 @@ pub(crate) async fn scan_data_folder_scoped(
new_cache.info.scan_directory_source = None;
new_cache.info.scan_source_trees.clear();
new_cache.info.scan_coverage_receipt = None;
new_cache.info.scan_coverage_measured_at = None;
if had_scan_checkpoint {
global_metrics().record_scanner_checkpoint_cleared();
}
@@ -3997,6 +3998,7 @@ pub(crate) async fn scan_data_folder_scoped(
new_cache.info.scan_checkpoint = None;
new_cache.info.scan_resume_after = None;
new_cache.info.scan_coverage_receipt = None;
new_cache.info.scan_coverage_measured_at = None;
}
if raw_enumeration_page_index.is_some() {
new_cache.info.scan_raw_enumeration_page_index = raw_enumeration_page_index;
@@ -4004,6 +4006,7 @@ pub(crate) async fn scan_data_folder_scoped(
new_cache.info.scan_checkpoint = None;
new_cache.info.scan_resume_after = None;
new_cache.info.scan_coverage_receipt = None;
new_cache.info.scan_coverage_measured_at = None;
}
if partial_cache_is_useful(&root, pending_heals_changed)
|| !new_cache.info.scan_directory_work.is_empty()
@@ -195,6 +195,7 @@ pub(super) struct DirectoryWork {
// The final compaction threshold caps how many paths this list can hold.
pub(super) linked: Vec<String>,
pub(super) compacted_total: super::DataUsageEntry,
pub(super) measured_at: Option<u64>,
pub(super) gap: bool,
compacted: bool,
recorder: Option<Recorder>,
@@ -225,6 +226,9 @@ impl DirectoryWork {
pub(super) fn acknowledge(&mut self, name: &str, linked: bool, compacted: bool) {
if self.names.acknowledge() {
self.compacted = compacted;
if compacted {
self.measured_at = crate::data_usage_define::scan_measurement_time_millis(std::time::SystemTime::now());
}
if linked && !compacted {
self.linked.push(name.to_owned());
}
@@ -236,6 +240,7 @@ impl DirectoryWork {
self.names.rewind();
self.linked.clear();
self.compacted_total = super::DataUsageEntry::default();
self.measured_at = None;
self.gap = false;
}
}
@@ -467,6 +472,7 @@ impl ScannerDirectory {
if let Some(aggregate) = &saved.aggregate {
work.compacted = true;
work.compacted_total = aggregate.value.clone();
work.measured_at = saved.measured_at;
}
let physical_path: Arc<Path> = Arc::from(Path::new(path));
if let Some(part) = &saved.part {
@@ -9,6 +9,7 @@ use super::{DirectoryScope, DirectoryStamp, DirectoryWork, FolderScanMode, nativ
use crate::data_usage_define::{
DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageScanIdentity, DataUsageScanProgress,
};
use serde::ser::{SerializeMap, SerializeStruct};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
@@ -109,7 +110,7 @@ pub(super) struct Aggregate {
/// Immutable directory membership and the last fully counted compacted child.
/// The active child's provisional contribution is never part of this receipt.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[derive(Clone, Debug, Deserialize)]
pub struct DirectoryCheckpoint {
version: u8,
pub(super) parent: String,
@@ -124,6 +125,53 @@ pub struct DirectoryCheckpoint {
pub(super) aggregate: Option<Aggregate>,
endpoint: String,
digest: [u8; 32],
/// Diagnostic time of the last acknowledged aggregate, never a save time.
#[serde(default)]
pub(super) measured_at: Option<u64>,
}
impl Serialize for DirectoryCheckpoint {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
if self.measured_at.is_none() {
// Enumeration-only receipts retain the compact version-one bytes
// so optional diagnostics do not enlarge each name-budget quantum.
let mut state = serializer.serialize_struct("DirectoryCheckpoint", 13)?;
state.serialize_field("version", &self.version)?;
state.serialize_field("parent", &self.parent)?;
state.serialize_field("scope", &self.scope)?;
state.serialize_field("physical", &self.physical)?;
state.serialize_field("pages", &self.pages)?;
state.serialize_field("anchor", &self.anchor)?;
state.serialize_field("entries_read", &self.entries_read)?;
state.serialize_field("sealed", &self.sealed)?;
state.serialize_field("part", &self.part)?;
state.serialize_field("metadata", &self.metadata)?;
state.serialize_field("aggregate", &self.aggregate)?;
state.serialize_field("endpoint", &self.endpoint)?;
state.serialize_field("digest", &self.digest)?;
return state.end();
}
// Version-one readers accept named fields and ignore this additive
// diagnostic field. Keep the existing coverage checksum unchanged.
let mut map = serializer.serialize_map(Some(13 + usize::from(self.measured_at.is_some())))?;
map.serialize_entry("version", &self.version)?;
map.serialize_entry("parent", &self.parent)?;
map.serialize_entry("scope", &self.scope)?;
map.serialize_entry("physical", &self.physical)?;
map.serialize_entry("pages", &self.pages)?;
map.serialize_entry("anchor", &self.anchor)?;
map.serialize_entry("entries_read", &self.entries_read)?;
map.serialize_entry("sealed", &self.sealed)?;
map.serialize_entry("part", &self.part)?;
map.serialize_entry("metadata", &self.metadata)?;
map.serialize_entry("aggregate", &self.aggregate)?;
map.serialize_entry("endpoint", &self.endpoint)?;
map.serialize_entry("digest", &self.digest)?;
if let Some(measured_at) = self.measured_at {
map.serialize_entry("measured_at", &measured_at)?;
}
map.end()
}
}
/// Physical progress orders only callbacks from the same directory generation.
@@ -252,8 +300,8 @@ impl DirectoryCheckpoint {
})
}
/// A display contribution contains only acknowledged compacted children.
pub(crate) fn display_usage(&self, info: &DataUsageCacheInfo) -> Option<(DirectoryProgress, &DataUsageEntry)> {
Some((self.progress(info)?, &self.aggregate.as_ref()?.value))
pub(crate) fn display_usage(&self, info: &DataUsageCacheInfo) -> Option<(DirectoryProgress, &DataUsageEntry, Option<u64>)> {
Some((self.progress(info)?, &self.aggregate.as_ref()?.value, self.measured_at))
}
pub(crate) fn endpoint(&self) -> &str {
&self.endpoint
@@ -352,6 +400,7 @@ impl Recorder {
aggregate,
endpoint: self.endpoint.clone(),
digest: [0; 32],
measured_at: work.measured_at,
};
checkpoint.checksum().map(|digest| {
checkpoint.digest = digest;
@@ -5,6 +5,10 @@ use super::*;
impl DirectoryCheckpoint {
pub(crate) async fn wide_storage_fixture(info: &DataUsageCacheInfo) -> [Self; 2] {
Self::storage_fixture(info, 300_000, [121_034, 121_035]).await
}
pub(crate) async fn storage_fixture(info: &DataUsageCacheInfo, width: usize, completed: [usize; 2]) -> [Self; 2] {
let scope = Scope {
bucket: info.name.clone(),
source: info.source.expect("fixture source"),
@@ -24,20 +28,23 @@ impl DirectoryCheckpoint {
modified: SystemTime::UNIX_EPOCH,
};
let mut recorder = Recorder::new(slot.clone(), &info.name, scope, endpoint, Some(physical), None);
recorder.entries_read = 300_000;
recorder.entries_read = u64::try_from(width).expect("fixture width");
let mut work = DirectoryWork {
compacted: true,
..Default::default()
};
for i in 0..300_000 {
for i in 0..width {
work.names.add(format!("obj-{i:06}"));
work.names.flush_full().await.expect("record real immutable name page");
}
work.names.finish();
[121_034, 121_035].map(|objects| {
completed.map(|objects| {
work.names.seek_after(Some(&format!("obj-{:06}", objects - 1)));
work.compacted_total.objects = objects;
work.compacted_total.size = objects * 1024;
work.measured_at = info
.last_update
.and_then(crate::data_usage_define::scan_measurement_time_millis);
recorder.save(&work);
slot.lock()
.expect("fixture slot")
@@ -46,3 +53,60 @@ impl DirectoryCheckpoint {
})
}
}
#[derive(Serialize, Deserialize)]
struct LegacyCheckpoint {
version: u8,
parent: String,
scope: Scope,
physical: Option<Physical>,
pages: Vec<NamePage>,
anchor: Option<native_io::DirectoryAnchor>,
entries_read: u64,
sealed: bool,
part: Option<String>,
metadata: bool,
aggregate: Option<Aggregate>,
endpoint: String,
digest: [u8; 32],
}
#[tokio::test]
async fn display_progress_measurement_metadata_keeps_version_one_checkpoint_compatibility() {
let plan = crate::DataUsageScanPlanDigest([1; 32]);
let info = DataUsageCacheInfo {
name: "bucket".into(),
source: Some(DataUsageCacheSource::new(0, 0)),
scan_identity: Some(DataUsageScanIdentity {
version: crate::data_usage_define::DATA_USAGE_SCAN_CHECKPOINT_VERSION,
bucket_incarnation: uuid::Uuid::from_u128(1),
set_layout: crate::DataUsageScanPlanDigest([7; 32]),
publication_epoch: 0,
tier_registry_generation: 1,
scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Normal,
}),
scan_progress: Some(DataUsageScanProgress {
started_plan: plan,
requested_plan: plan,
}),
scan_directory_source: Some("disk-a".into()),
last_update: Some(SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(10)),
..Default::default()
};
let [checkpoint, _] = DirectoryCheckpoint::storage_fixture(&info, 4, [2, 3]).await;
assert_eq!(checkpoint.measured_at, Some(10_000));
let bytes = rmp_serde::to_vec(&checkpoint).expect("encode additive named checkpoint");
let legacy: LegacyCheckpoint = rmp_serde::from_slice(&bytes).expect("version-one reader ignores measurement metadata");
let old_bytes = rmp_serde::to_vec(&legacy).expect("encode legacy positional checkpoint");
let recovered: DirectoryCheckpoint = rmp_serde::from_slice(&old_bytes).expect("new reader accepts legacy receipt");
assert_eq!(recovered.measured_at, None);
assert_eq!(
rmp_serde::to_vec(&recovered).expect("encode receipt without measurement metadata"),
old_bytes,
"receipts without diagnostic metadata keep the compact legacy encoding"
);
assert!(
recovered.is_bound_to(&info),
"diagnostic metadata must not change the version-one coverage checksum"
);
}
@@ -358,6 +358,7 @@ async fn display_progress_keeps_reference_during_executor_enumeration_then_repla
let (mut cache, _) = bound_checkpoint();
let original_time = SystemTime::UNIX_EPOCH + Duration::from_secs(10);
cache.info.last_update = Some(original_time);
cache.info.scan_coverage_measured_at = Some(10_000);
cache.info.scan_directory_source = Some("first-executor".into());
let mut revisions = DataUsageCache::default()
.load_with_revisions(writer.clone(), CACHE_NAME)
@@ -36,7 +36,16 @@ async fn scan(
(outcome, budget)
}
async fn read_display_progress(store: &FixtureStore) -> Option<rustfs_data_usage::BucketScanProgressSource> {
match tokio::fs::read(store.root.join("progress")).await {
Ok(bytes) => Some(serde_json::from_slice(&bytes).expect("decode independent display progress")),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
Err(error) => panic!("read durable display progress: {error}"),
}
}
async fn save_reload(store: &Arc<FixtureStore>, cache: &DataUsageCache) -> DataUsageCache {
let previous = read_display_progress(store).await;
let revisions = DataUsageCache::default()
.load_with_revisions(store.clone(), CACHE_NAME)
.await
@@ -47,12 +56,12 @@ async fn save_reload(store: &Arc<FixtureStore>, cache: &DataUsageCache) -> DataU
.expect("save compacted checkpoint through real codec and revision checks");
let loaded = store.strict_load().await;
let restarted_store = FixtureStore::reopen(store.root.clone());
let progress_bytes = tokio::fs::read(restarted_store.root.join("progress"))
.await
.expect("durable display progress survives a cold store reopen");
let progress: rustfs_data_usage::BucketScanProgressSource =
serde_json::from_slice(&progress_bytes).expect("decode independent display progress");
assert_eq!(Some(&progress), loaded.display_scan_progress(0).as_ref());
let progress = read_display_progress(&restarted_store).await;
if let Some(expected) = loaded.display_scan_progress(0) {
assert_eq!(progress, Some(expected), "accepted progress survives a cold store reopen");
} else {
assert_eq!(progress, previous, "unverified legacy coverage must not publish a new sample");
}
assert_eq!(loaded.info.snapshot_complete, cache.info.snapshot_complete);
assert_eq!(loaded.info.scan_checkpoint, cache.info.scan_checkpoint);
assert_eq!(loaded.info.scan_identity, cache.info.scan_identity);
+3 -1
View File
@@ -41,10 +41,12 @@ The companion [auto-testing scanner usage suite](https://github.com/rustfs/auto-
## Console bucket scan progress
`GET /rustfs/admin/v3/accountinfo` may attach `scan_progress` to each authorized bucket. Its `objects` and `size` describe accepted scanned coverage, not the bucket total, a lower bound on current contents, or a percentage. Concurrent changes can invalidate any such comparison. `stage` distinguishes `enumerating`, `scanning`, `awaiting_sources` and `completed`; completion means all logical sources supplied a completed traversal sample, without granting a usage certificate. Enumeration alone leaves both counters absent. `sources_observed` and `sources_total` make missing sets explicit. `updated_at` is the oldest contributing checkpoint time in Unix milliseconds.
`GET /rustfs/admin/v3/accountinfo` may attach `scan_progress` to each authorized bucket. Its `objects` and `size` describe accepted scanned coverage, not the bucket total, a lower bound on current contents, or a percentage. Concurrent changes can invalidate any such comparison. `stage` distinguishes `enumerating`, `scanning`, `awaiting_sources` and `completed`; completion means all logical sources supplied a completed traversal sample, without granting a usage certificate. Enumeration alone leaves both counters absent. `sources_observed` and `sources_total` make missing sets explicit. `updated_at` is the oldest contributing measurement time in Unix milliseconds. Saving an unchanged frontier or retained compacted receipt, including after a cold executor handoff, does not refresh that time. Legacy receipts without measurement metadata remain reusable for scanning but cannot supply a new numeric display sample with a fabricated age.
The scanner saves compact progress companions on their contributing erasure sets after committing bucket checkpoints. Retries replace a sample instead of adding deltas, and physical replicas are alternative views of one logical source. A new executor that has not yet measured another range retains the prior accepted sample and its original timestamp. Reader-local admission epochs and wakeup counters never fence a shared diagnostic sample; its movement scope comes from persisted pool/rebalance metadata. Missing, unsupported or corrupt companions do not affect checkpoint recovery or complete usage. Admin reads bind companions to the current bucket incarnation, set layout and durable movement generation, use bounded concurrent reads, and never list S3 objects or load the full checkpoint graph. The response keeps existing `objects` and `size` fields independent; Console retains their prior values and shows scan progress separately. Optional `usage_updated_at` preserves the original bucket certificate time, independently of the progress timestamp. Older servers and clients can omit or ignore these additive fields.
The two-second Admin read budget reserves its final quarter for movement-scope validation. Each validated source contributes immediately, so a stalled source or another bucket's identity lookup cannot erase completed reads. A bucket with missing sources reports `awaiting_sources`; unavailable final scope validation still omits the diagnostic view.
Progress extraction shares the checkpoint's serialized buffers, including paged payloads, with at most one dedicated CPU worker per process. It releases movement admission after capturing the durable generation and leaves native I/O workers available for primary saves. Waiting for the worker and preparing its input are included in the two-second diagnostic preparation budget; cancellation is checked during decoding. A nonpreemptible validation may finish after its waiter expires, retaining one decoded checkpoint until it exits. Its resource permit remains held, preventing repeated saves from accumulating workers or decoded copies, and its expired result is not published. Primary and companion publication retain their existing epoch and revision checks; scanner cycle budgets are unchanged.
The adaptive best-effort backup budget covers preparation, manifest publication, reconciliation and page reclamation together. Exhausting it stops waiting for the backup without undoing a successful primary checkpoint. An uncertain backup outcome can be recovered from storage on a later turn; it does not extend the current turn's budget.
+73 -5
View File
@@ -30,7 +30,7 @@ use http::header::CONTENT_TYPE;
use http::{HeaderMap, HeaderValue};
use hyper::{Method, StatusCode};
use matchit::Params;
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
use rustfs_data_usage::{BucketScanStage, BucketUsageInfo, BucketUsageValueState, DataUsageInfo};
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
@@ -91,7 +91,15 @@ fn apply_usage_to_bucket_access_info(bucket_info: &mut rustfs_madmin::BucketAcce
bucket_info.size = Some(usage.size);
bucket_info.objects = Some(usage.objects_count);
bucket_info.usage_value_state = usage.usage_value_state;
bucket_info.usage_value_state = usage.usage_value_state.map(|state| {
match state {
BucketUsageValueState::Certified => "certified",
BucketUsageValueState::Carried => "carried",
BucketUsageValueState::Observed => "observed",
BucketUsageValueState::Unknown => "unknown",
}
.to_owned()
});
bucket_info.usage_updated_at = usage.usage_certification.and_then(|certificate| {
u64::try_from(
certificate
@@ -106,6 +114,23 @@ fn apply_usage_to_bucket_access_info(bucket_info: &mut rustfs_madmin::BucketAcce
bucket_info.object_versions_histogram = Some(usage.object_versions_histogram.clone());
}
fn bucket_scan_progress_response(progress: rustfs_data_usage::BucketScanProgress) -> rustfs_madmin::BucketScanProgress {
let stage = match progress.stage {
BucketScanStage::Enumerating => "enumerating",
BucketScanStage::Scanning => "scanning",
BucketScanStage::AwaitingSources => "awaiting_sources",
BucketScanStage::Completed => "completed",
};
rustfs_madmin::BucketScanProgress {
stage: stage.to_owned(),
updated_at: progress.updated_at,
objects: progress.objects,
size: progress.size,
sources_observed: progress.sources_observed,
sources_total: progress.sources_total,
}
}
fn object_lock_config_enabled(config: &ObjectLockConfiguration) -> bool {
config
.object_lock_enabled
@@ -323,7 +348,7 @@ impl Operation for AccountInfoHandler {
.collect::<Vec<_>>();
let mut progress = load_bucket_scan_progress_for_admin(store, &visible_buckets, &data_usage_info).await;
for bucket in &mut account_info.buckets {
bucket.scan_progress = progress.remove(&bucket.name);
bucket.scan_progress = progress.remove(&bucket.name).map(bucket_scan_progress_response);
}
let data = serde_json::to_vec(&AccountInfoResponse {
@@ -442,14 +467,14 @@ mod tests {
fn accountinfo_partial_progress_never_fills_total_usage_fields() {
let mut bucket_info = rustfs_madmin::BucketAccessInfo {
name: "scanning".into(),
scan_progress: Some(rustfs_data_usage::BucketScanProgress {
scan_progress: Some(bucket_scan_progress_response(rustfs_data_usage::BucketScanProgress {
stage: rustfs_data_usage::BucketScanStage::Scanning,
updated_at: 1000,
objects: Some(7),
size: Some(71),
sources_observed: 1,
sources_total: 2,
}),
})),
..Default::default()
};
apply_usage_to_bucket_access_info(&mut bucket_info, None);
@@ -474,6 +499,49 @@ mod tests {
assert_eq!(bucket_info.scan_progress.expect("separate partial view").objects, Some(7));
}
#[test]
fn accountinfo_scan_progress_projection_preserves_wire_contract() {
for (stage, objects, size, sources_observed) in [
(BucketScanStage::Enumerating, None, None, 2),
(BucketScanStage::Scanning, Some(7), Some(71), 2),
(BucketScanStage::AwaitingSources, Some(0), Some(0), 1),
(BucketScanStage::Completed, Some(7), Some(71), 2),
] {
let progress = rustfs_data_usage::BucketScanProgress {
stage,
updated_at: 1000,
objects,
size,
sources_observed,
sources_total: 2,
};
let expected = serde_json::to_value(&progress).expect("domain progress wire contract");
let response = bucket_scan_progress_response(progress);
assert_eq!(serde_json::to_value(response).expect("SDK progress response"), expected);
}
}
#[test]
fn accountinfo_usage_provenance_projection_preserves_wire_contract() {
for state in [
BucketUsageValueState::Certified,
BucketUsageValueState::Carried,
BucketUsageValueState::Observed,
BucketUsageValueState::Unknown,
] {
let usage = BucketUsageInfo {
usage_value_state: Some(state),
..Default::default()
};
let mut bucket = rustfs_madmin::BucketAccessInfo::default();
apply_usage_to_bucket_access_info(&mut bucket, Some(&usage));
assert_eq!(
serde_json::to_value(bucket).expect("SDK usage provenance")["usage_value_state"],
serde_json::to_value(state).expect("domain provenance wire contract")
);
}
}
#[test]
fn accountinfo_data_usage_error_message_is_generic() {
let err = map_data_usage_result::<&str>(Err("sensitive disk path")).expect_err("load failure must reach the response");