Compare commits

...

7 Commits

Author SHA1 Message Date
cxymds b77323f24b chore(scanner): integrate current 1.0.6 release updates 2026-11-16 09:10:36 +08:00
cxymds 5433d2f947 test(admin): bind snapshot fixtures to current bucket identities
(cherry picked from commit 73895b37e8)
2026-11-15 15:47:37 +08:00
cxymds f4dcda08e6 feat(scanner): observe bucket reuse and production proof gates
(cherry picked from commit 32f7f70a29)
2026-11-15 15:47:01 +08:00
cxymds b17fb942ce fix(usage): validate cached statistics against bucket identity
(cherry picked from commit 5d32827af5)
2026-11-15 15:47:01 +08:00
cxymds 0e054ec619 fix(scanner): cap superseded retries at the nominal cycle
(cherry picked from commit 04ef836aa3)
2026-11-15 15:38:59 +08:00
cxymds 89f99245ec fix(scanner): preserve baselines during backup catchup
Require decoded-primary provenance before accepting an unchanged baseline after its backup catches up. Keep current-revision CAS and publication fences, and align the real checkpoint fixture with the bounded-cycle carry contract.

(cherry picked from commit 03850c1cd6)
2026-11-15 15:38:59 +08:00
cxymds 409c8aa8af feat(scanner): expose scan work and reuse observations
(cherry picked from commit 654be96186)
2026-11-15 15:38:59 +08:00
33 changed files with 2882 additions and 241 deletions
@@ -171,6 +171,70 @@ impl DataUsageBucketObservation {
}
impl DataUsageInfo {
/// Hide values that cannot be bound to the current storage-owned identity.
///
/// `incarnations` contains canonical non-nil identities read from storage;
/// a missing entry means identity was unavailable. Neither legacy values
/// without identity nor metadata read failures establish confirmed zero.
/// Changed responses are display-only and cannot authorize quota or writes.
/// If retained counters overflow, all values remain unknown.
/// Returns the number of canonical values removed.
pub fn retain_current_bucket_usage_for_admin(&mut self, incarnations: &HashMap<String, String>) -> usize {
let original_count = self.buckets_usage.len();
let original_observation_count = self.usage_snapshot_bucket_observations.len();
let matches_current = |bucket: &String, usage: &BucketUsageInfo| {
incarnations.get(bucket).is_some_and(|current| {
!current.is_empty()
&& current != "00000000-0000-0000-0000-000000000000"
&& usage.bucket_incarnation.as_ref() == Some(current)
})
};
self.buckets_usage.retain(|bucket, usage| matches_current(bucket, usage));
self.usage_snapshot_bucket_observations.retain(|bucket, observation| {
matches_current(bucket, &observation.usage)
&& observation
.sources
.iter()
.all(|source| incarnations.get(bucket) == Some(&source.bucket_incarnation))
});
if original_count == self.buckets_usage.len()
&& original_observation_count == self.usage_snapshot_bucket_observations.len()
{
return 0;
}
let totals = self.buckets_usage.values().try_fold([0_u64; 4], |total, usage| {
Some([
total[0].checked_add(usage.size)?,
total[1].checked_add(usage.objects_count)?,
total[2].checked_add(usage.versions_count)?,
total[3].checked_add(usage.delete_markers_count)?,
])
});
let [size, objects, versions, markers] = totals.unwrap_or_else(|| {
self.buckets_usage.clear();
self.usage_snapshot_bucket_observations.clear();
[0; 4]
});
self.bucket_sizes = self
.buckets_usage
.iter()
.map(|(bucket, usage)| (bucket.clone(), usage.size))
.collect();
self.buckets_count = u64::try_from(self.buckets_usage.len()).unwrap_or(u64::MAX);
self.objects_total_size = size;
self.objects_total_count = objects;
self.versions_total_count = versions;
self.delete_markers_total_count = markers;
self.replication_info.clear();
self.tier_stats = None;
self.unknown_tier_stats = None;
self.usage_snapshot_complete = false;
self.usage_snapshot_partial = true;
self.usage_snapshot_converged = Some(false);
self.usage_snapshot_admin_projection = true;
original_count.saturating_sub(self.buckets_usage.len())
}
/// Overlay validated companions on an owned admin response only.
///
/// `incarnations` must come from current storage metadata, not the companion.
@@ -321,3 +321,110 @@ fn provenance_and_companions_change_candidate_bytes_and_round_trip_as_maps() {
let restored: DataUsageInfo = rmp_serde::from_slice(&named).expect("read named map with additive fields");
assert_eq!(restored, canonical);
}
#[test]
fn current_identity_filter_preserves_an_unchanged_complete_snapshot() {
let mut candidate = snapshot();
let original = candidate.clone();
assert_eq!(candidate.retain_current_bucket_usage_for_admin(&incarnations()), 0);
assert_eq!(candidate, original);
assert!(candidate.is_complete_bucket_usage_snapshot());
}
#[test]
fn current_identity_filter_hides_unbound_values_without_confirming_zero() {
for identity in [
None,
Some(""),
Some("corrupt"),
Some("00000000-0000-0000-0000-000000000000"),
Some("22222222-2222-4222-8222-222222222222"),
] {
let mut candidate = snapshot();
candidate.usage_snapshot_bucket_observations.clear();
candidate
.buckets_usage
.get_mut("bucket")
.expect("canonical")
.bucket_incarnation = identity.map(str::to_string);
assert_eq!(
candidate.retain_current_bucket_usage_for_admin(&incarnations()),
1,
"identity {identity:?}"
);
assert!(!candidate.buckets_usage.contains_key("bucket"));
assert!(!candidate.bucket_sizes.contains_key("bucket"));
assert_eq!(candidate.usage_snapshot_converged, Some(false));
assert!(candidate.usage_snapshot_admin_projection);
assert!(!candidate.is_complete_bucket_usage_snapshot());
assert!(!candidate.is_valid_partial_snapshot());
}
for current in ["", "00000000-0000-0000-0000-000000000000"] {
let mut candidate = snapshot();
candidate
.buckets_usage
.get_mut("bucket")
.expect("canonical")
.bucket_incarnation = Some(current.to_string());
assert_eq!(
candidate.retain_current_bucket_usage_for_admin(&HashMap::from([("bucket".to_string(), current.to_string())])),
1
);
assert!(candidate.usage_snapshot_admin_projection);
}
let mut candidate = snapshot();
assert_eq!(candidate.retain_current_bucket_usage_for_admin(&HashMap::new()), 1);
assert!(candidate.usage_snapshot_bucket_observations.is_empty());
assert_eq!(candidate.usage_snapshot_converged, Some(false));
}
#[test]
fn current_identity_filter_keeps_healthy_buckets_and_recalculates_partial_totals() {
let mut candidate = snapshot();
candidate.buckets_usage.insert("cold".to_string(), usage(7));
candidate.buckets_count = 2;
candidate.bucket_sizes.insert("cold".to_string(), 7);
candidate.calculate_totals();
candidate.replication_info.insert("target".to_string(), Default::default());
candidate.tier_stats = Some(Default::default());
candidate.unknown_tier_stats = Some(Default::default());
assert_eq!(
candidate.retain_current_bucket_usage_for_admin(&HashMap::from([("cold".to_string(), INCARNATION.to_string())])),
1
);
assert_eq!(candidate.buckets_usage.len(), 1);
assert_eq!(candidate.buckets_usage["cold"].size, 7);
assert_eq!(candidate.bucket_sizes, HashMap::from([("cold".to_string(), 7)]));
assert_eq!(
(
candidate.buckets_count,
candidate.objects_total_size,
candidate.objects_total_count,
candidate.versions_total_count
),
(1, 7, 7, 7)
);
assert!(candidate.replication_info.is_empty());
assert!(candidate.tier_stats.is_none());
assert!(candidate.unknown_tier_stats.is_none());
assert!(candidate.usage_snapshot_admin_projection);
assert_eq!(candidate.usage_snapshot_converged, Some(false));
}
#[test]
fn current_identity_filter_overflow_leaves_totals_unknown() {
let mut candidate = snapshot();
candidate.buckets_usage.insert("large".to_string(), usage(u64::MAX));
candidate.buckets_usage.insert("small".to_string(), usage(1));
candidate.buckets_count = 3;
let current = HashMap::from([
("large".to_string(), INCARNATION.to_string()),
("small".to_string(), INCARNATION.to_string()),
]);
assert_eq!(candidate.retain_current_bucket_usage_for_admin(&current), 3);
assert!(candidate.buckets_usage.is_empty());
assert!(candidate.bucket_sizes.is_empty());
assert_eq!(candidate.usage_snapshot_converged, Some(false));
assert!(candidate.usage_snapshot_admin_projection);
assert!(!candidate.is_complete_bucket_usage_snapshot());
}
+2 -2
View File
@@ -440,8 +440,8 @@ pub mod data_usage {
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
load_bucket_usage_from_backend_cached, load_compression_total_from_memory, load_data_usage_from_backend,
load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory,
load_admin_data_usage_with_memory_overlay, load_bucket_usage_from_backend_cached, load_compression_total_from_memory,
load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory,
record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
+119 -21
View File
@@ -1463,22 +1463,36 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
let observed = load_observed_data_usage_snapshot(store.clone()).await.ok().flatten();
let (selected, selected_is_current_format) =
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
let mut response = normalize_loaded_data_usage(selected, selected_is_current_format).await.0;
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
}
async fn project_admin_data_usage_for_current_buckets(
store: Arc<ECStore>,
mut response: DataUsageInfo,
with_memory_overlay: bool,
) -> DataUsageInfo {
use futures::StreamExt as _;
let snapshot_measured = response.is_complete_bucket_usage_snapshot() || response.is_valid_partial_snapshot();
let overlay_authoritative = with_memory_overlay && !runtime_sources::setup_is_dist_erasure().await;
let deadline = tokio::time::Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT;
let mut incarnations = HashMap::new();
{
// Display-only identity reads must not serialize every bucket behind a
// slow metadata lock while holding the shared admin refresh permit.
// The common deadline bounds their total added latency; dropping this
// stream cancels all unfinished reads without spawning detached work.
// Own the names and store across awaits so this future remains Send
// when composed into spawned metrics collectors.
let buckets = response
.usage_snapshot_bucket_observations
// Identity reads happen after releasing snapshot and refresh locks,
// including on cache hits. Metadata transaction reads linearize each
// bucket against creation/deletion without relying on peer notices.
// The common deadline cancels unfinished work and bounds added latency.
let mut buckets = response
.buckets_usage
.keys()
.chain(response.usage_snapshot_bucket_observations.keys())
.cloned()
.collect::<Vec<_>>();
.collect::<HashSet<_>>();
if overlay_authoritative {
// Copy only names while holding the memory lock, then release it
// before storage I/O. A later new entry has no identity proof and
// will remain unknown in this response.
buckets.extend(memory_cache().read().await.keys().cloned());
}
let mut identities = futures::stream::iter(buckets)
.map(move |bucket| {
let store = store.clone();
@@ -1492,7 +1506,6 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
let Ok(Some((bucket, identity))) = tokio::time::timeout_at(deadline, identities.next()).await else {
break;
};
// Unavailable, timed-out or recreated buckets retain canonical values.
if let Ok(incarnation) = identity
&& !incarnation.is_nil()
{
@@ -1501,7 +1514,20 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
}
}
response.project_bucket_observations_for_admin(&incarnations);
Ok(response)
if with_memory_overlay
&& apply_bucket_usage_memory_overlay_if_authoritative(&mut response, overlay_authoritative).await
&& snapshot_measured
{
response.usage_snapshot_complete = false;
response.usage_snapshot_partial = true;
response.usage_snapshot_converged = Some(false);
response.usage_snapshot_admin_projection = true;
response.replication_info.clear();
response.tier_stats = None;
response.unknown_tier_stats = None;
}
response.retain_current_bucket_usage_for_admin(&incarnations);
response
}
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
@@ -1690,11 +1716,7 @@ async fn project_data_usage_from_backend_cached<T>(
}
}
/// Load the freshest structurally complete snapshot for authenticated admin
/// observability. A scan raced by namespace activity may be selected here, but
/// never by [`load_data_usage_from_backend_cached`], which remains the
/// converged source for quota admission.
pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
async fn load_admin_data_usage_snapshot_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
loop {
@@ -1743,6 +1765,38 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
}
}
/// Load cached scanner statistics for authenticated admin observability.
///
/// Each response verifies bucket identities against current storage metadata,
/// including cache hits. Missing, failed, or timed-out identity reads leave that
/// bucket's usage absent (unknown). Legacy values without an incarnation stay
/// unknown until a scanner snapshot certifies their identity. Per-bucket reads
/// linearize independently; this is not an atomic namespace snapshot.
/// Identity reads have a shared deadline and do not outlive cancellation. The
/// unchanged cached baseline remains independent of quota and persistence.
///
/// # Errors
/// Returns the cached snapshot loading error when no baseline can be loaded.
pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let response = load_admin_data_usage_snapshot_cached(store.clone()).await?;
Ok(project_admin_data_usage_for_current_buckets(store, response, false).await)
}
/// Load the admin view with the existing authoritative single-process overlay.
///
/// The overlay is applied before current-identity projection so it cannot
/// reintroduce statistics from a retired or unverified bucket generation.
/// Distributed nodes continue to use scanner statistics without local totals.
/// Cancellation, unknown values, and per-bucket read semantics are the same as
/// [`load_admin_data_usage_from_backend_cached`]. No overlay is persisted.
///
/// # Errors
/// Returns the cached snapshot loading error when no baseline can be loaded.
pub async fn load_admin_data_usage_with_memory_overlay(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let response = load_admin_data_usage_snapshot_cached(store.clone()).await?;
Ok(project_admin_data_usage_for_current_buckets(store, response, true).await)
}
/// Invalidate the process-local persisted usage snapshot cache after a durable
/// scanner write.
pub async fn invalidate_data_usage_snapshot_cache() {
@@ -2805,16 +2859,17 @@ async fn replace_bucket_usage_memory_from_info_if_generation(data_usage_info: &D
*cache = next_cache;
}
async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mut DataUsageInfo, authoritative: bool) {
async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mut DataUsageInfo, authoritative: bool) -> bool {
if !authoritative {
return;
return false;
}
let cache = memory_cache().read().await;
if cache.is_empty() {
return;
return false;
}
let was_complete = data_usage_info.usage_snapshot_complete;
let persisted_update = data_usage_info.last_update;
let mut changed = false;
let mut added_bucket = false;
@@ -2851,6 +2906,7 @@ async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mu
data_usage_info.buckets_count = data_usage_info.buckets_usage.len() as u64;
data_usage_info.calculate_totals();
}
changed || was_complete != data_usage_info.usage_snapshot_complete
}
pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageInfo) {
@@ -3744,6 +3800,23 @@ mod tests {
*cache_updating().write().await = false;
}
async fn bind_admin_usage_test_identities(store: &Arc<ECStore>, snapshot: &mut DataUsageInfo) {
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
for (bucket, usage) in &mut snapshot.buckets_usage {
store
.make_bucket(bucket, &crate::storage_api_contracts::bucket::MakeBucketOptions::default())
.await
.expect("create a real bucket for current admin identity validation");
usage.bucket_incarnation = Some(
store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("read the storage-owned fixture identity")
.to_string(),
);
}
}
fn data_usage_info_for_test(bucket: &str, objects_count: u64, size: u64, last_update: SystemTime) -> DataUsageInfo {
let mut info = DataUsageInfo {
last_update: Some(last_update),
@@ -4279,7 +4352,11 @@ mod tests {
use crate::config::com::save_config;
let (_dirs, store) = isolated_store_over_temp_disks().await;
let (authoritative, observed) = usage_snapshots_with_missing_observed_set();
let (mut authoritative, mut observed) = usage_snapshots_with_missing_observed_set();
bind_admin_usage_test_identities(&store, &mut authoritative).await;
for (bucket, usage) in &mut observed.buckets_usage {
usage.bucket_incarnation = authoritative.buckets_usage[bucket].bucket_incarnation.clone();
}
let primary_bytes = serde_json::to_vec(&authoritative).expect("encode complete usage");
let observed_bytes = serde_json::to_vec(&observed).expect("encode partial observation");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), primary_bytes.clone())
@@ -4328,6 +4405,11 @@ mod tests {
reconciled.scanner_epoch = Some(4);
reconciled.scanner_cycle = Some(12);
reconciled.usage_snapshot_converged = Some(true);
reconciled
.buckets_usage
.get_mut("large-bucket")
.expect("reconciled bucket")
.bucket_incarnation = authoritative.buckets_usage["large-bucket"].bucket_incarnation.clone();
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
@@ -4849,6 +4931,12 @@ mod tests {
backup.scanner_epoch = Some(4);
backup.scanner_cycle = Some(10);
backup.usage_snapshot_converged = Some(true);
bind_admin_usage_test_identities(&store, &mut current).await;
backup
.buckets_usage
.get_mut("existing")
.expect("backup bucket")
.bucket_incarnation = current.buckets_usage["existing"].bucket_incarnation.clone();
for (path, info) in [
(DATA_USAGE_OBJ_NAME_PATH.as_str(), &current),
(DATA_USAGE_OBJ_BACKUP_PATH.as_str(), &backup),
@@ -4871,6 +4959,11 @@ mod tests {
after_delete.scanner_epoch = current.scanner_epoch;
after_delete.scanner_cycle = current.scanner_cycle;
after_delete.usage_snapshot_converged = Some(true);
after_delete
.buckets_usage
.get_mut("existing")
.expect("post-delete bucket")
.bucket_incarnation = current.buckets_usage["existing"].bucket_incarnation.clone();
let mut stale_observation =
data_usage_info_for_test("retired-incarnation", 99, 999, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
stale_observation.scanner_epoch = Some(4);
@@ -4932,6 +5025,7 @@ mod tests {
authoritative.scanner_epoch = Some(7);
authoritative.scanner_cycle = Some(10);
authoritative.usage_snapshot_converged = Some(true);
bind_admin_usage_test_identities(&store, &mut authoritative).await;
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
@@ -4946,6 +5040,10 @@ mod tests {
info.scanner_cycle = Some(cycle);
info.usage_snapshot_converged = Some(false);
info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
info.buckets_usage
.get_mut("bucket")
.expect("observed bucket")
.bucket_incarnation = authoritative.buckets_usage["bucket"].bucket_incarnation.clone();
info
};
let first = observation(7, 11, 200, 11, 1_100);
@@ -221,7 +221,7 @@ async fn companion_admin_cache_isolated_from_quota_memory_and_persistence() {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buckets() {
async fn blocked_companion_identity_stays_unknown_without_delaying_healthy_buckets() {
use crate::bucket::metadata_sys::{
acquire_metadata_transaction_guard, init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
};
@@ -280,8 +280,11 @@ async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buc
.await
.expect("optional projection must return before the blocked metadata lock's long timeout")
.expect("canonical admin response stays available");
assert_eq!(response.buckets_usage[blocked].size, 100);
assert_eq!(response.buckets_usage[blocked].usage_value_state, Some(BucketUsageValueState::Carried));
assert!(
!response.buckets_usage.contains_key(blocked),
"an unavailable current identity cannot bind the canonical value"
);
assert!(!response.bucket_sizes.contains_key(blocked));
assert_eq!(response.buckets_usage[healthy].size, 250);
assert_eq!(response.buckets_usage[healthy].usage_value_state, Some(BucketUsageValueState::Observed));
assert!(response.usage_snapshot_admin_projection);
@@ -293,13 +296,12 @@ async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buc
.is_ok()
);
drop(blocked_guard);
invalidate_data_usage_snapshot_cache().await;
let recovered = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("refresh after lock recovery");
assert_eq!(
recovered.buckets_usage[blocked].size, 250,
"unavailable identity can recover on the next refresh"
"unavailable identity can recover on a cache hit"
);
assert_eq!(recovered.buckets_usage[healthy].size, 250);
let quota = load_data_usage_from_backend_cached(store)
@@ -308,3 +310,201 @@ async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buc
assert_eq!(quota.buckets_usage[blocked].size, 100);
assert_eq!(quota.buckets_usage[healthy].size, 100);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn cached_canonical_admin_usage_rechecks_current_identity_without_peer_invalidation() {
use crate::bucket::metadata::{load_bucket_incarnation, save_bucket_incarnation};
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "usage-recreated-cache";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create identity fixture");
let old_identity = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("original identity");
let mut canonical = companion_snapshot(bucket, old_identity);
canonical.usage_snapshot_bucket_observations.clear();
canonical.usage_snapshot_converged = Some(true);
canonical.usage_snapshot_pending_buckets.clear();
canonical.usage_snapshot_pending_bucket_count = 0;
canonical
.buckets_usage
.get_mut(bucket)
.expect("canonical bucket")
.objects_count = 107;
canonical.calculate_totals();
let original_bytes = serde_json::to_vec(&canonical).expect("encode canonical old bucket");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), original_bytes.clone())
.await
.expect("persist canonical root");
invalidate_data_usage_snapshot_cache().await;
let first = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("prime raw admin cache");
assert_eq!(first.buckets_usage[bucket].objects_count, 107);
assert!(first.is_complete_bucket_usage_snapshot());
let cache_generation = admin_data_usage_snapshot_generation();
let new_identity = uuid::Uuid::new_v4();
// Simulate the storage-owned identity changing on another node while this
// process has not received its invalidation notice. Do not clear the cache.
save_bucket_incarnation(store.clone(), bucket, new_identity)
.await
.expect("publish changed disk identity");
assert_eq!(
load_bucket_incarnation(store.clone(), bucket)
.await
.expect("read new sidecar"),
Some(new_identity)
);
let second = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("read raw cache with fresh identity");
assert_eq!(admin_data_usage_snapshot_generation(), cache_generation);
assert!(
!second.buckets_usage.contains_key(bucket),
"same name cannot retain the predecessor's 107"
);
assert!(!second.bucket_sizes.contains_key(bucket));
assert_eq!(second.usage_snapshot_converged, Some(false));
assert!(second.usage_snapshot_admin_projection);
assert!(reject_admin_usage_projection(&second).is_err());
assert!(certified_bucket_usage(&second, bucket, new_identity).is_none());
assert!(parse_usage_snapshot(&serde_json::to_vec(&second).expect("encode display response")).is_err());
let quota = load_data_usage_from_backend_cached(store.clone())
.await
.expect("independent quota baseline");
assert_eq!(quota.buckets_usage[bucket].objects_count, 107);
assert_eq!(
read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("unchanged persisted root"),
original_bytes
);
// Rebinding a cached raw baseline is transient: recovery does not need to
// reload or permanently blank a healthy bucket's statistics.
save_bucket_incarnation(store.clone(), bucket, old_identity)
.await
.expect("restore fixture identity");
let recovered = load_admin_data_usage_from_backend_cached(store)
.await
.expect("recover on another cache hit");
assert_eq!(recovered.buckets_usage[bucket].objects_count, 107);
assert!(recovered.is_complete_bucket_usage_snapshot());
assert_eq!(admin_data_usage_snapshot_generation(), cache_generation);
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn admin_memory_overlay_cannot_reintroduce_a_predecessor_identity() {
use crate::bucket::metadata::save_bucket_incarnation;
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "usage-recreated-overlay";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create overlay identity fixture");
let old_identity = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("original identity");
let mut canonical = companion_snapshot(bucket, old_identity);
canonical.usage_snapshot_bucket_observations.clear();
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&canonical).expect("encode canonical root"),
)
.await
.expect("persist overlay fixture");
invalidate_data_usage_snapshot_cache().await;
let cached_usage = canonical.buckets_usage[bucket].clone();
replace_bucket_usage_memory_from_authoritative(bucket, cached_usage, SystemTime::now()).await;
let new_identity = uuid::Uuid::new_v4();
save_bucket_incarnation(store.clone(), bucket, new_identity)
.await
.expect("publish a replacement identity");
let response = load_admin_data_usage_with_memory_overlay(store.clone())
.await
.expect("project after memory overlay");
assert!(!response.buckets_usage.contains_key(bucket));
assert_eq!(response.usage_snapshot_converged, Some(false));
assert!(response.usage_snapshot_admin_projection);
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn admin_overlay_flags_only_change_keeps_identity_bound_cold_usage() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
assert!(
!runtime_sources::setup_is_dist_erasure().await,
"exercise the production single-process overlay"
);
let cold = "usage-overlay-cold";
let unverified = "usage-overlay-unverified";
for bucket in [cold, unverified] {
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create overlay fixture bucket");
}
let identity = store
.bucket_incarnation_id_from_disk(cold)
.await
.expect("cold bucket identity");
let mut canonical = companion_snapshot(cold, identity);
canonical.usage_snapshot_bucket_observations.clear();
canonical.usage_snapshot_converged = Some(true);
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&canonical).expect("encode cold canonical value"),
)
.await
.expect("persist cold baseline");
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
memory_cache().write().await.insert(
unverified.to_string(),
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
);
record_bucket_object_write_memory(unverified, None, 42).await;
{
let cache = memory_cache().read().await;
assert!(cache[unverified].dirty);
assert!(!cache[unverified].authoritative);
}
let response = load_admin_data_usage_with_memory_overlay(store)
.await
.expect("identity-filtered flags-only overlay");
assert_eq!(response.buckets_usage[cold].size, 100);
assert_eq!(response.objects_total_size, 100);
assert_eq!(response.bucket_sizes[cold], 100);
assert!(!response.buckets_usage.contains_key(unverified));
assert!(response.usage_snapshot_admin_projection);
assert_eq!(response.usage_snapshot_converged, Some(false));
assert!(!response.is_complete_bucket_usage_snapshot());
assert!(!response.is_valid_partial_snapshot());
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
}
@@ -1033,16 +1033,26 @@ mod tests {
#[tokio::test]
#[serial]
async fn server_info_reads_complete_observation_without_promoting_authority() {
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::{read_config, save_config};
use crate::data_usage::{
DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH, invalidate_data_usage_snapshot_cache,
load_data_usage_from_backend_cached,
};
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
use std::time::{Duration, SystemTime};
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
store
.make_bucket("bucket", &MakeBucketOptions::default())
.await
.expect("create a real observed bucket");
let incarnation = store
.bucket_incarnation_id_from_disk("bucket")
.await
.expect("read current storage-owned incarnation");
let primary = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
scanner_epoch: Some(1),
@@ -1066,6 +1076,7 @@ mod tests {
versions_count: 4,
delete_markers_count: 1,
size: 6,
bucket_incarnation: Some(incarnation.to_string()),
..Default::default()
},
)]
@@ -1117,6 +1128,46 @@ mod tests {
invalidate_data_usage_snapshot_cache().await;
}
#[test]
fn identity_filtered_usage_remains_unknown_in_server_info() {
let mut info = rustfs_data_usage::DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
buckets_count: 1,
objects_total_count: 107,
objects_total_size: 1024,
buckets_usage: [(
"bucket".to_string(),
rustfs_data_usage::BucketUsageInfo {
objects_count: 107,
size: 1024,
bucket_incarnation: Some("11111111-1111-4111-8111-111111111111".to_string()),
..Default::default()
},
)]
.into_iter()
.collect(),
usage_snapshot_complete: true,
..Default::default()
};
let current =
std::collections::HashMap::from([("bucket".to_string(), "22222222-2222-4222-8222-222222222222".to_string())]);
assert_eq!(info.retain_current_bucket_usage_for_admin(&current), 1);
assert!(info.buckets_usage.is_empty());
assert!(info.usage_snapshot_admin_projection);
assert_eq!(info.usage_snapshot_converged, Some(false));
let mut buckets = rustfs_madmin::Buckets::default();
let mut objects = rustfs_madmin::Objects::default();
let mut versions = rustfs_madmin::Versions::default();
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
let mut usage = rustfs_madmin::Usage::default();
apply_data_usage_result(Ok(info), &mut buckets, &mut objects, &mut versions, &mut delete_markers, &mut usage);
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(objects.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(versions.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(delete_markers.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(usage.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
}
#[test]
fn incomplete_data_usage_is_unavailable_in_server_info() {
let mut buckets = rustfs_madmin::Buckets::default();
+1
View File
@@ -15,6 +15,7 @@
pub mod init_time;
pub mod last_minute;
pub mod metrics;
pub mod scan_observation;
pub mod startup;
pub mod usage_candidate;
+40 -1
View File
@@ -13,6 +13,9 @@
// limitations under the License.
use crate::last_minute::{AccElem, LastMinuteLatency};
use crate::scan_observation::{
ScannerMetadataReadGuard, ScannerMetadataReadOutcome, ScannerScanObservation, ScannerScanObservationSnapshot,
};
use crate::usage_candidate::{
ScannerUsageCandidateCount, ScannerUsageCandidateKind, ScannerUsageCandidateOutcome, USAGE_CANDIDATE_COUNTER_SLOTS,
};
@@ -999,6 +1002,7 @@ impl ScannerUsagePublicationObservation {
pub struct Metrics {
startup: crate::startup::ScannerStartupObservation,
scan_observation: ScannerScanObservation,
operations: Vec<AtomicU64>,
latency: Vec<LockedLastMinuteLatency>,
actions: Vec<AtomicU64>,
@@ -1653,6 +1657,9 @@ pub struct ScannerMetricsReport {
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
#[serde(default)]
pub usage_freshness: ScannerUsageFreshnessSnapshot,
/// Node-local, process-lifetime work observations. Older nodes have no evidence.
#[serde(default)]
pub scan_observation: ScannerScanObservationSnapshot,
/// Process-lifetime candidate-construction results. Missing on older reports
/// means unobserved, not accepted or healthy. Publication is tracked separately.
#[serde(default)]
@@ -2147,9 +2154,38 @@ impl Metrics {
&self.startup
}
pub fn scan_observation(&self) -> &ScannerScanObservation {
&self.scan_observation
}
/// Start after scanner node admission, before polling the disk read future.
/// Disk-permit and RPC waits remain included; decoding is excluded. Dropping
/// an unfinished guard observes cancellation, not physical I/O completion.
pub fn start_scanner_metadata_read(&self) -> ScannerMetadataReadGuard<'_> {
self.scan_observation.start_metadata_read();
ScannerMetadataReadGuard {
metrics: self,
started: std::time::Instant::now(),
outcome: ScannerMetadataReadOutcome::Cancelled,
bytes: 0,
}
}
pub(crate) fn finish_scanner_metadata_read(&self, outcome: ScannerMetadataReadOutcome, bytes: u64, elapsed: Duration) {
let bytes = if outcome == ScannerMetadataReadOutcome::Success {
bytes
} else {
0
};
self.scan_observation.finish_metadata_read(outcome, bytes, elapsed);
self.operations[Metric::ReadMetadata as usize].fetch_add(1, Ordering::Relaxed);
self.latency[Metric::ReadMetadata as usize].add_size(elapsed, bytes);
}
pub fn new() -> Self {
Self {
startup: crate::startup::ScannerStartupObservation::default(),
scan_observation: ScannerScanObservation::default(),
operations: (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
// Each slot gets its own fresh LockedLastMinuteLatency so that
// different metrics never accidentally share state.
@@ -2755,6 +2791,8 @@ impl Metrics {
/// walked the whole namespace, so an operator can tell why scoped
/// convergence is not happening.
pub fn record_scanner_scope_fallback(&self, reason: impl Into<String>) {
let reason = reason.into();
self.scan_observation.record_scope_fallback(&reason);
self.scanner_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
self.scanner_scope_fallback_last_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
@@ -2762,7 +2800,7 @@ impl Metrics {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*last_reason = reason.into();
*last_reason = reason;
}
pub fn record_scanner_usage_durable_success(&self) {
@@ -3897,6 +3935,7 @@ impl Metrics {
failed: self.scanner_transition_failed.load(Ordering::Relaxed),
};
m.usage_candidates = self.usage_candidate_counts();
m.scan_observation = self.scan_observation.snapshot();
let usage_save_result = ScannerUsageSaveResult::from_code(self.scanner_usage_last_save_result.load(Ordering::Relaxed));
m.usage_freshness = ScannerUsageFreshnessSnapshot {
dirty_pending_buckets: self.scanner_dirty_usage_pending_buckets.load(Ordering::Relaxed),
@@ -0,0 +1,675 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Node-local, process-lifetime observations of scanner work. Counters are
//! independently loaded, so concurrent reports need not balance exactly.
//! These observations never grant cache reuse or publication authority.
use crate::metrics::Metrics;
use serde::{Deserialize, Serialize};
use std::{
collections::BTreeMap,
sync::atomic::{AtomicU64, Ordering},
time::{Duration, Instant},
};
pub const PREFIX_REUSE_BLOCKERS: [&str; 8] = [
"missing_producer_identity",
"missing_durable_journal_replay",
"restart_gap",
"generation_gap",
"overflow",
"stale_ack_generation",
"missing_cold_zero_walk_oracle",
"distributed_without_peer_invalidation",
];
const SCOPE_FALLBACK_REASONS: [&str; 6] = [
"dirty_snapshot_incomplete",
"dirty_generation_exhausted",
"dirty_bucket_overflow",
"no_listed_dirty_buckets",
"baseline_unavailable",
"peer_scope_unavailable",
];
/// A storage read result, independent of later metadata decoding or ILM work.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerMetadataReadOutcome {
Success,
NotFound,
Error,
Cancelled,
}
/// The existing path that actually copied a cached subtree instead of walking it.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerCacheReuseKind {
CleanPrefix,
CoveredNamespace,
SampledNamespace,
ClassifiedNamespace,
}
impl ScannerCacheReuseKind {
const ALL: [Self; 4] = [
Self::CleanPrefix,
Self::CoveredNamespace,
Self::SampledNamespace,
Self::ClassifiedNamespace,
];
fn label(self) -> &'static str {
match self {
Self::CleanPrefix => "clean_prefix",
Self::CoveredNamespace => "covered_namespace",
Self::SampledNamespace => "sampled_namespace",
Self::ClassifiedNamespace => "classified_namespace",
}
}
}
/// Completed bucket-level reuse work, independent of durable publication.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerBucketReuseKind {
ScopedColdBucket,
CurrentRoot,
}
impl ScannerBucketReuseKind {
const ALL: [Self; 2] = [Self::ScopedColdBucket, Self::CurrentRoot];
fn label(self) -> &'static str {
match self {
Self::ScopedColdBucket => "scoped_cold_bucket",
Self::CurrentRoot => "current_root",
}
}
}
/// A production folder gate, not a diagnostic preflight.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerCacheReuseGate {
CleanPrefix,
CoveredNamespace,
NamespaceSampling,
}
impl ScannerCacheReuseGate {
const ALL: [Self; 3] = [Self::CleanPrefix, Self::CoveredNamespace, Self::NamespaceSampling];
fn label(self) -> &'static str {
match self {
Self::CleanPrefix => "clean_prefix",
Self::CoveredNamespace => "covered_namespace",
Self::NamespaceSampling => "namespace_sampling",
}
}
}
/// The first existing condition reached by a folder reuse gate.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerCacheReuseDecision {
MissingScope,
NoResumeSweep,
DeepScan,
ForwardSweep,
ActiveReplication,
NotRoot,
CompactedParent,
MissingCachedChild,
InvalidPrefix,
DirtyPrefix,
ErasureHealthRequired,
MissingCoverage,
CachedFailures,
UncompactedChild,
SelectedForWalk,
Eligible,
}
impl ScannerCacheReuseDecision {
const ALL: [Self; 16] = [
Self::MissingScope,
Self::NoResumeSweep,
Self::DeepScan,
Self::ForwardSweep,
Self::ActiveReplication,
Self::NotRoot,
Self::CompactedParent,
Self::MissingCachedChild,
Self::InvalidPrefix,
Self::DirtyPrefix,
Self::ErasureHealthRequired,
Self::MissingCoverage,
Self::CachedFailures,
Self::UncompactedChild,
Self::SelectedForWalk,
Self::Eligible,
];
fn label(self) -> &'static str {
match self {
Self::MissingScope => "missing_scope",
Self::NoResumeSweep => "no_resume_sweep",
Self::DeepScan => "deep_scan",
Self::ForwardSweep => "forward_sweep",
Self::ActiveReplication => "active_replication",
Self::NotRoot => "not_root",
Self::CompactedParent => "compacted_parent",
Self::MissingCachedChild => "missing_cached_child",
Self::InvalidPrefix => "invalid_prefix",
Self::DirtyPrefix => "dirty_prefix",
Self::ErasureHealthRequired => "erasure_health_required",
Self::MissingCoverage => "missing_coverage",
Self::CachedFailures => "cached_failures",
Self::UncompactedChild => "uncompacted_child",
Self::SelectedForWalk => "selected_for_walk",
Self::Eligible => "eligible",
}
}
fn not_evaluated(self) -> bool {
matches!(
self,
Self::MissingScope | Self::NoResumeSweep | Self::DeepScan | Self::ForwardSweep | Self::ActiveReplication
)
}
}
/// Scope admission and end-of-cycle evidence are separate observations. An
/// activated completion proof does not mean this cycle used prefix reuse.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerPrefixReuseStage {
ScopeResolution,
CycleCompletion,
}
/// The first existing baseline validation check that rejected a snapshot.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerBaselineRejection {
Missing,
Decode,
Incomplete,
Partial,
Marks,
Epoch,
SourceCount,
PlanMissing,
PlanMismatch,
SourceIndex,
SourceMembership,
SourceDuplicate,
SetIncomplete,
SetTombstone,
SetEpoch,
SetCycle,
SetPlan,
SourceCoverage,
AuthoritativeIdentity,
NotNewer,
}
impl ScannerBaselineRejection {
const ALL: [Self; 20] = [
Self::Missing,
Self::Decode,
Self::Incomplete,
Self::Partial,
Self::Marks,
Self::Epoch,
Self::SourceCount,
Self::PlanMissing,
Self::PlanMismatch,
Self::SourceIndex,
Self::SourceMembership,
Self::SourceDuplicate,
Self::SetIncomplete,
Self::SetTombstone,
Self::SetEpoch,
Self::SetCycle,
Self::SetPlan,
Self::SourceCoverage,
Self::AuthoritativeIdentity,
Self::NotNewer,
];
fn label(self) -> &'static str {
match self {
Self::Missing => "missing",
Self::Decode => "decode",
Self::Incomplete => "incomplete",
Self::Partial => "partial",
Self::Marks => "marks",
Self::Epoch => "epoch",
Self::SourceCount => "source_count",
Self::PlanMissing => "plan_missing",
Self::PlanMismatch => "plan_mismatch",
Self::SourceIndex => "source_index",
Self::SourceMembership => "source_membership",
Self::SourceDuplicate => "source_duplicate",
Self::SetIncomplete => "set_incomplete",
Self::SetTombstone => "set_tombstone",
Self::SetEpoch => "set_epoch",
Self::SetCycle => "set_cycle",
Self::SetPlan => "set_plan",
Self::SourceCoverage => "source_coverage",
Self::AuthoritativeIdentity => "authoritative_identity",
Self::NotNewer => "not_newer",
}
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerMetadataReadSnapshot {
pub attempts: u64,
pub succeeded: u64,
pub not_found: u64,
pub errors: u64,
pub cancelled: u64,
/// Bytes returned by successful storage reads, including corrupt metadata.
pub bytes: u64,
/// Elapsed read-future time, including failed and cancelled reads.
pub elapsed_nanos: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerCacheReuseSnapshot {
/// Completed subtree reuse decisions; not cache entries or objects skipped.
pub count: u64,
/// Time spent copying and attaching the reused subtree, excluding listing.
pub elapsed_nanos: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerBucketReuseSnapshot {
/// Cold bucket copies or current roots successfully handed to the collector.
pub count: u64,
/// Cold copy/attachment time or current-root channel-send time; excludes validation.
pub elapsed_nanos: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerCacheReuseDecisionSnapshot {
/// Policy prerequisites that stopped the gate before cache proof checks.
pub not_evaluated: BTreeMap<String, u64>,
/// First failed cache/work condition; later conditions are not evaluated.
pub blocked: BTreeMap<String, u64>,
/// Admitted candidates, which may still need EC classification before copying.
pub eligible: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerPrefixReuseSnapshot {
pub evaluated: u64,
pub activated: u64,
/// All failed checks are counted; one evaluation can have several blockers.
pub blockers: BTreeMap<String, u64>,
}
/// Missing on an older node means unobserved, not zero work or healthy reuse.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerScanObservationSnapshot {
pub metadata_reads: ScannerMetadataReadSnapshot,
pub cache_reuse: BTreeMap<String, ScannerCacheReuseSnapshot>,
pub bucket_reuse: BTreeMap<String, ScannerBucketReuseSnapshot>,
pub reuse_decisions: BTreeMap<String, ScannerCacheReuseDecisionSnapshot>,
/// Clean root children still walked because EC scans also check health.
pub erasure_clean_prefix_walks: u64,
pub prefix_scope_resolution: ScannerPrefixReuseSnapshot,
pub prefix_cycle_completion: ScannerPrefixReuseSnapshot,
pub scope_fallbacks: BTreeMap<String, u64>,
/// Rejections only for cycles with a baseline_unavailable scope fallback.
pub authoritative_baseline_rejections: BTreeMap<String, u64>,
pub observed_baseline_rejections: BTreeMap<String, u64>,
}
#[derive(Default)]
pub struct ScannerScanObservation {
metadata: [AtomicU64; 7],
cache_count: [AtomicU64; 4],
cache_nanos: [AtomicU64; 4],
bucket_count: [AtomicU64; 2],
bucket_nanos: [AtomicU64; 2],
reuse_decisions: [[AtomicU64; ScannerCacheReuseDecision::ALL.len()]; 3],
erasure_clean_prefix_walks: AtomicU64,
prefix_evaluated: [AtomicU64; 2],
prefix_activated: [AtomicU64; 2],
prefix_blockers: [[AtomicU64; PREFIX_REUSE_BLOCKERS.len()]; 2],
scope_fallbacks: [AtomicU64; SCOPE_FALLBACK_REASONS.len()],
baseline_rejections: [[AtomicU64; ScannerBaselineRejection::ALL.len()]; 2],
}
fn duration_nanos(duration: Duration) -> u64 {
u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
}
fn labelled_counts(labels: impl IntoIterator<Item = &'static str>, counters: &[AtomicU64]) -> BTreeMap<String, u64> {
labels
.into_iter()
.zip(counters)
.map(|(label, count)| (label.to_owned(), count.load(Ordering::Relaxed)))
.collect()
}
impl ScannerScanObservation {
pub(crate) fn start_metadata_read(&self) {
self.metadata[0].fetch_add(1, Ordering::Relaxed);
}
pub(crate) fn finish_metadata_read(&self, outcome: ScannerMetadataReadOutcome, bytes: u64, elapsed: Duration) {
self.metadata[1 + outcome as usize].fetch_add(1, Ordering::Relaxed);
if outcome == ScannerMetadataReadOutcome::Success {
self.metadata[5].fetch_add(bytes, Ordering::Relaxed);
}
self.metadata[6].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
}
pub fn record_cache_reuse(&self, kind: ScannerCacheReuseKind, elapsed: Duration) {
self.cache_count[kind as usize].fetch_add(1, Ordering::Relaxed);
self.cache_nanos[kind as usize].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
}
/// Records completed reuse work only; it grants no publication authority.
pub fn record_bucket_reuse(&self, kind: ScannerBucketReuseKind, count: u64, elapsed: Duration) {
self.bucket_count[kind as usize].fetch_add(count, Ordering::Relaxed);
self.bucket_nanos[kind as usize].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
}
/// Records one first-condition decision without inspecting later prerequisites.
pub fn record_cache_reuse_decision(&self, gate: ScannerCacheReuseGate, decision: ScannerCacheReuseDecision) {
self.reuse_decisions[gate as usize][decision as usize].fetch_add(1, Ordering::Relaxed);
}
pub fn record_erasure_clean_prefix_walk(&self) {
self.erasure_clean_prefix_walks.fetch_add(1, Ordering::Relaxed);
}
pub fn record_prefix_reuse(&self, stage: ScannerPrefixReuseStage, activated: bool, blockers: &[Option<&str>; 8]) {
let stage = stage as usize;
self.prefix_evaluated[stage].fetch_add(1, Ordering::Relaxed);
if activated {
self.prefix_activated[stage].fetch_add(1, Ordering::Relaxed);
}
for blocker in blockers.iter().flatten() {
if let Some(index) = PREFIX_REUSE_BLOCKERS.iter().position(|label| label == blocker) {
self.prefix_blockers[stage][index].fetch_add(1, Ordering::Relaxed);
}
}
}
pub(crate) fn record_scope_fallback(&self, reason: &str) {
if let Some(index) = SCOPE_FALLBACK_REASONS.iter().position(|label| *label == reason) {
self.scope_fallbacks[index].fetch_add(1, Ordering::Relaxed);
}
}
pub fn record_baseline_rejection(&self, authoritative: ScannerBaselineRejection, observed: Option<ScannerBaselineRejection>) {
self.baseline_rejections[0][authoritative as usize].fetch_add(1, Ordering::Relaxed);
if let Some(observed) = observed {
self.baseline_rejections[1][observed as usize].fetch_add(1, Ordering::Relaxed);
}
}
pub fn snapshot(&self) -> ScannerScanObservationSnapshot {
let prefix = |stage: ScannerPrefixReuseStage| ScannerPrefixReuseSnapshot {
evaluated: self.prefix_evaluated[stage as usize].load(Ordering::Relaxed),
activated: self.prefix_activated[stage as usize].load(Ordering::Relaxed),
blockers: labelled_counts(PREFIX_REUSE_BLOCKERS, &self.prefix_blockers[stage as usize]),
};
ScannerScanObservationSnapshot {
metadata_reads: ScannerMetadataReadSnapshot {
attempts: self.metadata[0].load(Ordering::Relaxed),
succeeded: self.metadata[1].load(Ordering::Relaxed),
not_found: self.metadata[2].load(Ordering::Relaxed),
errors: self.metadata[3].load(Ordering::Relaxed),
cancelled: self.metadata[4].load(Ordering::Relaxed),
bytes: self.metadata[5].load(Ordering::Relaxed),
elapsed_nanos: self.metadata[6].load(Ordering::Relaxed),
},
cache_reuse: ScannerCacheReuseKind::ALL
.into_iter()
.map(|kind| {
(
kind.label().to_owned(),
ScannerCacheReuseSnapshot {
count: self.cache_count[kind as usize].load(Ordering::Relaxed),
elapsed_nanos: self.cache_nanos[kind as usize].load(Ordering::Relaxed),
},
)
})
.collect(),
bucket_reuse: ScannerBucketReuseKind::ALL
.into_iter()
.map(|kind| {
(
kind.label().to_owned(),
ScannerBucketReuseSnapshot {
count: self.bucket_count[kind as usize].load(Ordering::Relaxed),
elapsed_nanos: self.bucket_nanos[kind as usize].load(Ordering::Relaxed),
},
)
})
.collect(),
reuse_decisions: ScannerCacheReuseGate::ALL
.into_iter()
.map(|gate| {
let counts = &self.reuse_decisions[gate as usize];
let decisions = |not_evaluated| {
ScannerCacheReuseDecision::ALL
.into_iter()
.filter(|decision| {
*decision != ScannerCacheReuseDecision::Eligible && decision.not_evaluated() == not_evaluated
})
.map(|decision| (decision.label().to_owned(), counts[decision as usize].load(Ordering::Relaxed)))
.collect()
};
(
gate.label().to_owned(),
ScannerCacheReuseDecisionSnapshot {
not_evaluated: decisions(true),
blocked: decisions(false),
eligible: counts[ScannerCacheReuseDecision::Eligible as usize].load(Ordering::Relaxed),
},
)
})
.collect(),
erasure_clean_prefix_walks: self.erasure_clean_prefix_walks.load(Ordering::Relaxed),
prefix_scope_resolution: prefix(ScannerPrefixReuseStage::ScopeResolution),
prefix_cycle_completion: prefix(ScannerPrefixReuseStage::CycleCompletion),
scope_fallbacks: labelled_counts(SCOPE_FALLBACK_REASONS, &self.scope_fallbacks),
authoritative_baseline_rejections: labelled_counts(
ScannerBaselineRejection::ALL.map(ScannerBaselineRejection::label),
&self.baseline_rejections[0],
),
observed_baseline_rejections: labelled_counts(
ScannerBaselineRejection::ALL.map(ScannerBaselineRejection::label),
&self.baseline_rejections[1],
),
}
}
}
/// Dropping an unfinished read records cancellation. The guard observes the
/// read future only; it cannot cancel it or change its storage result.
#[must_use]
pub struct ScannerMetadataReadGuard<'a> {
pub(crate) metrics: &'a Metrics,
pub(crate) started: Instant,
pub(crate) outcome: ScannerMetadataReadOutcome,
pub(crate) bytes: u64,
}
impl ScannerMetadataReadGuard<'_> {
pub fn finish(mut self, outcome: ScannerMetadataReadOutcome, bytes: u64) {
self.outcome = outcome;
self.bytes = bytes;
}
}
impl Drop for ScannerMetadataReadGuard<'_> {
fn drop(&mut self) {
self.metrics
.finish_scanner_metadata_read(self.outcome, self.bytes, self.started.elapsed());
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
#[tokio::test]
async fn metadata_read_reports_storage_results_and_existing_read_metric() {
let metrics = Metrics::new();
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::Success, 127);
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::Success, 0);
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::NotFound, 0);
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::Error, 0);
let report = metrics.report().await;
let reads = &report.scan_observation.metadata_reads;
assert_eq!(
(reads.attempts, reads.succeeded, reads.not_found, reads.errors, reads.cancelled),
(4, 2, 1, 1, 0)
);
assert_eq!(reads.bytes, 127);
assert_eq!(report.life_time_ops["read_metadata"], 4);
assert_eq!(report.last_minute.actions["read_metadata"].count, 4);
assert_eq!(report.last_minute.actions["read_metadata"].bytes, 127);
}
#[tokio::test]
async fn metadata_read_cancellation_balances_an_actually_polled_future_once() {
let metrics = Arc::new(Metrics::new());
let worker_metrics = Arc::clone(&metrics);
let (started, admitted) = tokio::sync::oneshot::channel();
let worker = tokio::spawn(async move {
let read = worker_metrics.start_scanner_metadata_read();
started.send(()).expect("observe started read");
std::future::pending::<()>().await;
read.finish(ScannerMetadataReadOutcome::Success, 99);
});
tokio::time::timeout(Duration::from_secs(5), admitted)
.await
.expect("read starts within deadline")
.expect("read started");
let in_flight = metrics.scan_observation().snapshot().metadata_reads;
assert_eq!((in_flight.attempts, in_flight.cancelled, in_flight.succeeded), (1, 0, 0));
worker.abort();
assert!(worker.await.expect_err("read task was aborted").is_cancelled());
let report = metrics.report().await;
let reads = &report.scan_observation.metadata_reads;
assert_eq!((reads.attempts, reads.cancelled, reads.succeeded, reads.bytes), (1, 1, 0, 0));
assert_eq!(report.life_time_ops["read_metadata"], 1);
assert_eq!(report.last_minute.actions["read_metadata"].count, 1);
assert_eq!(
Duration::from_nanos(reads.elapsed_nanos).as_secs(),
report.last_minute.actions["read_metadata"].acc_time
);
}
#[tokio::test]
async fn observation_report_keeps_stages_reasons_and_copy_costs_separate() {
let metrics = Metrics::new();
let observation = metrics.scan_observation();
observation.record_cache_reuse(ScannerCacheReuseKind::CleanPrefix, Duration::from_nanos(321));
observation.record_bucket_reuse(ScannerBucketReuseKind::ScopedColdBucket, 2, Duration::from_nanos(654));
observation.record_bucket_reuse(ScannerBucketReuseKind::CurrentRoot, 1, Duration::from_nanos(987));
observation.record_cache_reuse_decision(ScannerCacheReuseGate::CleanPrefix, ScannerCacheReuseDecision::MissingScope);
observation
.record_cache_reuse_decision(ScannerCacheReuseGate::CleanPrefix, ScannerCacheReuseDecision::ErasureHealthRequired);
observation.record_cache_reuse_decision(ScannerCacheReuseGate::CoveredNamespace, ScannerCacheReuseDecision::Eligible);
observation.record_erasure_clean_prefix_walk();
observation.record_prefix_reuse(
ScannerPrefixReuseStage::ScopeResolution,
false,
&[
Some("missing_producer_identity"),
Some("missing_durable_journal_replay"),
None,
None,
None,
None,
None,
None,
],
);
observation.record_prefix_reuse(ScannerPrefixReuseStage::CycleCompletion, true, &[None; 8]);
metrics.record_scanner_scope_fallback("baseline_unavailable");
observation.record_baseline_rejection(ScannerBaselineRejection::Epoch, Some(ScannerBaselineRejection::Missing));
let report = metrics.report().await;
let snapshot = &report.scan_observation;
assert_eq!(snapshot.cache_reuse["clean_prefix"].count, 1);
assert_eq!(snapshot.cache_reuse["clean_prefix"].elapsed_nanos, 321);
assert_eq!(snapshot.cache_reuse["sampled_namespace"].count, 0);
assert_eq!(snapshot.bucket_reuse["scoped_cold_bucket"].count, 2);
assert_eq!(snapshot.bucket_reuse["scoped_cold_bucket"].elapsed_nanos, 654);
assert_eq!(snapshot.bucket_reuse["current_root"].count, 1);
assert_eq!(snapshot.bucket_reuse["current_root"].elapsed_nanos, 987);
assert_eq!(snapshot.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"], 1);
assert_eq!(snapshot.reuse_decisions["clean_prefix"].blocked["erasure_health_required"], 1);
assert_eq!(snapshot.reuse_decisions["covered_namespace"].eligible, 1);
let mut prior_observations = serde_json::to_value(snapshot).expect("encode observations");
let fields = prior_observations.as_object_mut().expect("observation object");
fields.remove("bucket_reuse");
fields.remove("reuse_decisions");
let prior: ScannerScanObservationSnapshot =
serde_json::from_value(prior_observations).expect("read earlier observations");
assert!(prior.bucket_reuse.is_empty());
assert!(prior.reuse_decisions.is_empty());
assert_eq!(prior.cache_reuse["clean_prefix"].count, 1);
assert_eq!(snapshot.erasure_clean_prefix_walks, 1);
assert_eq!(
(snapshot.prefix_scope_resolution.evaluated, snapshot.prefix_scope_resolution.activated),
(1, 0)
);
assert_eq!(snapshot.prefix_scope_resolution.blockers["missing_producer_identity"], 1);
assert_eq!(snapshot.prefix_scope_resolution.blockers["missing_durable_journal_replay"], 1);
assert_eq!(
(snapshot.prefix_cycle_completion.evaluated, snapshot.prefix_cycle_completion.activated),
(1, 1)
);
assert_eq!(
snapshot.scope_fallbacks["baseline_unavailable"],
report.usage_freshness.scope_fallback_total
);
assert_eq!(snapshot.authoritative_baseline_rejections["epoch"], 1);
assert_eq!(snapshot.observed_baseline_rejections["missing"], 1);
let mut legacy = serde_json::to_value(&report).expect("encode report");
legacy
.as_object_mut()
.expect("report is an object")
.remove("scan_observation");
let decoded: crate::metrics::ScannerMetricsReport = serde_json::from_value(legacy).expect("read legacy report");
assert!(decoded.scan_observation.cache_reuse.is_empty(), "legacy counters are unobserved");
assert_eq!(decoded.scan_observation.metadata_reads.attempts, 0);
}
}
+26
View File
@@ -205,6 +205,32 @@ pub(crate) struct DataUsageCacheRevisions {
backup: Option<DataUsageCacheRevision>,
}
/// In-memory provenance of the decoded cache and the revisions observed by its loader.
#[derive(Clone, Debug)]
pub(crate) struct DataUsageCacheBaselineRevisions {
revisions: DataUsageCacheRevisions,
loaded_from_main: bool,
}
impl DataUsageCacheBaselineRevisions {
pub(crate) fn revisions(&self) -> &DataUsageCacheRevisions {
&self.revisions
}
/// A backup may catch up to an unchanged primary without changing the decoded baseline.
/// Both reads must have decoded that primary; an ETag on a corrupt primary is insufficient.
pub(crate) fn permits_unchanged_baseline_since(&self, initial: &Self) -> bool {
if self.revisions == initial.revisions {
return true;
}
self.loaded_from_main
&& initial.loaded_from_main
&& matches!(&initial.revisions.main, DataUsageCacheRevision::Etag(etag) if !etag.is_empty())
&& self.revisions.main == initial.revisions.main
&& self.revisions.backup.as_ref() == Some(&self.revisions.main)
}
}
pub static DATA_USAGE_BUCKET: LazyLock<String> =
LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{BUCKET_META_PREFIX}"));
@@ -70,6 +70,7 @@ struct DataUsageCacheLoadResult {
cache: DataUsageCache,
main_revision: DataUsageCacheRevision,
backup_revision: Option<DataUsageCacheRevision>,
loaded_from_main: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -106,6 +107,18 @@ impl DataUsageCache {
store: Arc<S>,
name: &str,
) -> StorageResult<DataUsageCacheRevisions> {
self.load_with_baseline_revisions(store, name)
.await
.map(|baseline| baseline.revisions)
}
/// Loads cache revisions with provenance for unchanged-primary backup reconciliation.
/// Missing or rebuilt caches and a decoded backup do not establish a decoded primary.
pub(crate) async fn load_with_baseline_revisions<S: ScannerObjectIO>(
&mut self,
store: Arc<S>,
name: &str,
) -> StorageResult<DataUsageCacheBaselineRevisions> {
let backup_name = format!("{name}.bkp");
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
let loaded = Self::load_cache(store.clone(), name).await?;
@@ -133,7 +146,10 @@ impl DataUsageCache {
let main = loaded.main_revision;
*self = loaded.cache;
Ok(DataUsageCacheRevisions { main, backup })
Ok(DataUsageCacheBaselineRevisions {
revisions: DataUsageCacheRevisions { main, backup },
loaded_from_main: loaded.loaded_from_main,
})
}
pub(crate) async fn read_revisions<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheRevisions> {
@@ -156,10 +172,13 @@ impl DataUsageCache {
cache,
revision: Some(main_revision),
} => {
// Legacy-location fallback carries Missing for the primary path.
let loaded_from_main = matches!(&main_revision, DataUsageCacheRevision::Etag(_));
return Ok(DataUsageCacheLoadResult {
cache: *cache,
main_revision,
backup_revision: None,
loaded_from_main,
});
}
DataUsageCacheLoadAttempt::Loaded { revision: None, .. } => {
@@ -197,6 +216,7 @@ impl DataUsageCache {
cache: *cache,
main_revision,
backup_revision: Some(backup_revision),
loaded_from_main: false,
});
}
DataUsageCacheLoadState::Loaded
@@ -220,6 +240,7 @@ impl DataUsageCache {
main_revision: main_revision
.ok_or_else(|| Error::other(format!("scanner cache missing state has no revision: {name}")))?,
backup_revision,
loaded_from_main: false,
});
}
(DataUsageCacheLoadState::Corrupt, DataUsageCacheLoadState::Missing)
@@ -239,6 +260,7 @@ impl DataUsageCache {
main_revision: main_revision
.ok_or_else(|| Error::other(format!("scanner cache corrupt state has no revision: {name}")))?,
backup_revision,
loaded_from_main: false,
});
}
_ => {}
@@ -772,6 +772,102 @@ async fn corrupt_primary_cache_loads_valid_backup() {
assert_eq!(store.puts.load(Ordering::SeqCst), 0);
}
#[test]
fn baseline_revisions_allow_only_a_decoded_unchanged_primary_backup_catchup() {
let etag = |value: &str| DataUsageCacheRevision::Etag(value.to_owned());
let initial = DataUsageCacheBaselineRevisions {
revisions: DataUsageCacheRevisions {
main: etag("primary"),
backup: Some(etag("old-backup")),
},
loaded_from_main: true,
};
assert!(initial.permits_unchanged_baseline_since(&initial));
let mut caught_up = initial.clone();
caught_up.revisions.backup = Some(etag("primary"));
assert!(caught_up.permits_unchanged_baseline_since(&initial));
assert!(
!initial.permits_unchanged_baseline_since(&caught_up),
"backup rollback changes the baseline"
);
for backup in [None, Some(DataUsageCacheRevision::Missing)] {
let mut missing_backup = initial.clone();
missing_backup.revisions.backup = backup;
assert!(caught_up.permits_unchanged_baseline_since(&missing_backup));
assert!(!missing_backup.permits_unchanged_baseline_since(&initial));
}
for (initial_from_main, current_from_main) in [(false, true), (true, false), (false, false)] {
let mut baseline = initial.clone();
baseline.loaded_from_main = initial_from_main;
let mut current = caught_up.clone();
current.loaded_from_main = current_from_main;
assert!(
!current.permits_unchanged_baseline_since(&baseline),
"backup fallback grants no primary proof"
);
current.revisions = baseline.revisions.clone();
assert!(
current.permits_unchanged_baseline_since(&baseline),
"exact revisions retain existing admission"
);
}
let mut changed_main = caught_up.clone();
changed_main.revisions.main = etag("new-primary");
changed_main.revisions.backup = Some(etag("new-primary"));
assert!(!changed_main.permits_unchanged_baseline_since(&initial));
let mut changed_backup = caught_up;
changed_backup.revisions.backup = Some(etag("unrelated-backup"));
assert!(!changed_backup.permits_unchanged_baseline_since(&initial));
for main in [DataUsageCacheRevision::Missing, etag("")] {
let mut baseline = initial.clone();
baseline.revisions.main = main.clone();
let mut current = baseline.clone();
current.revisions.backup = Some(main);
assert!(
!current.permits_unchanged_baseline_since(&baseline),
"missing or empty primary ETag is not proof"
);
}
}
#[tokio::test]
async fn baseline_revisions_record_the_actual_decoded_cache_source() {
let mut expected = DataUsageCache::default();
expected.info.name = "decoded".to_owned();
let valid = expected.marshal_msg().expect("encode valid cache");
for (main, backup, from_main, name) in [
(valid.clone(), Some(valid.clone()), true, "decoded"),
(b"corrupt".to_vec(), Some(valid.clone()), false, "decoded"),
(b"corrupt".to_vec(), None, false, ""),
(Vec::new(), None, false, ""),
] {
let store = Arc::new(CacheReadStore::new(CacheReadBody::Bytes(main), backup));
let mut loaded = DataUsageCache::default();
let baseline = loaded
.load_with_baseline_revisions(store.clone(), DATA_USAGE_CACHE_NAME)
.await
.expect("load actual cache source");
assert_eq!(baseline.loaded_from_main, from_main);
assert_eq!(loaded.info.name, name);
let mut raw_loaded = DataUsageCache::default();
let raw = raw_loaded
.load_with_revisions(store, DATA_USAGE_CACHE_NAME)
.await
.expect("load compatible raw revisions");
assert_eq!(baseline.revisions(), &raw, "raw loader retains its revision contract");
}
let mut legacy_store = CacheReadStore::new(CacheReadBody::Bytes(Vec::new()), None);
legacy_store.legacy = Some(valid);
let mut legacy_cache = DataUsageCache::default();
let legacy = legacy_cache
.load_with_baseline_revisions(Arc::new(legacy_store), "legacy-cache")
.await
.expect("load cache from the legacy location");
assert_eq!(legacy_cache.info.name, "decoded");
assert!(!legacy.loaded_from_main);
assert_eq!(legacy.revisions.main, DataUsageCacheRevision::Missing);
}
#[tokio::test]
async fn corrupt_cache_without_valid_backup_rebuilds_with_cas_revisions() {
for backup in [None, Some(b"also-not-msgpack".to_vec())] {
+8 -9
View File
@@ -248,13 +248,13 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
/// otherwise idle instance whose clean-idle backoff had doubled a 60 s
/// interval), which defeats the fast path it is meant to protect.
///
/// The exponential growth in [`ScannerRetryBackoff::retry_interval`] is
/// what protects against a persistently hot bucket driving an unbroken
/// full-scan loop, so it can start small: 5 s, 10 s, 20 s … capped by
/// [`SCANNER_RETRY_MAX_INTERVAL`]. A one-off race recovers in seconds; a
/// genuinely hot bucket still reaches minute-scale backoff within a handful of
/// cycles. Preflight deferrals use the same bounded schedule so a temporarily
/// unavailable peer cannot drive a tight retry loop.
/// The exponential growth in [`ScannerRetryBackoff::retry_interval`] protects
/// against a persistently hot bucket driving an unbroken full-scan loop, so
/// retries can start small: 5 s, 10 s, 20 s … Superseded retries cap this
/// schedule at the ordinary cycle interval and preserve an operator-selected
/// nominal cadence. Preflight deferrals retain the generic
/// [`SCANNER_RETRY_MAX_INTERVAL`] cap so a temporarily unavailable peer cannot
/// drive a tight retry loop.
const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
/// Probe cadence while data movement pauses the scanner. Recovery retries use
@@ -3536,8 +3536,7 @@ where
));
let mut wait_plan =
scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for);
let superseded_retry_interval =
scanner_superseded_retry_interval(superseded_backoff, &runtime_config, maintenance_features);
let superseded_retry_interval = scanner_superseded_retry_interval(superseded_backoff, &runtime_config);
let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval);
let scope_fallback_retry_interval = scanner_scope_fallback_retry_interval(scope_fallback_backoff, &runtime_config);
let mut convergence_retry_interval = superseded_retry_interval
+7 -10
View File
@@ -192,19 +192,16 @@ impl ScannerRetryBackoff {
pub(super) fn scanner_superseded_retry_interval(
backoff: ScannerRetryBackoff,
runtime_config: &ScannerRuntimeConfig,
maintenance_features: ScannerMaintenanceFeatures,
) -> Option<Duration> {
let mut retry_interval = backoff.retry_interval(runtime_config.cycle_interval)?;
// Preserve an operator-selected duty-cycle floor during convergence.
let cycle_interval = runtime_config.cycle_interval.max(Duration::from_secs(1));
let retry_interval = backoff.retry_interval(cycle_interval)?;
// Preserve an operator-selected nominal cadence during convergence.
if runtime_config.cycle_interval_source != ScannerRuntimeConfigSource::Default {
retry_interval = retry_interval.max(runtime_config.cycle_interval);
return Some(cycle_interval);
}
// A complete but non-authoritative walk must not postpone lifecycle or
// replication evaluation beyond its ordinary maintenance cadence.
if maintenance_features.needs_regular_cycle() {
retry_interval = retry_interval.min(runtime_config.cycle_interval.max(Duration::from_secs(1)));
}
Some(retry_interval)
// A complete but non-authoritative walk must not postpone convergence
// beyond the ordinary cadence, even without lifecycle or replication.
Some(retry_interval.min(cycle_interval))
}
/// Pacing for the cycle after a full-namespace scope fallback while dirty
+73 -38
View File
@@ -8949,7 +8949,7 @@ fn clean_idle_backoff_grows_to_cap() {
}
#[test]
fn superseded_retry_backoff_grows_caps_and_resets_after_convergence() {
fn retry_backoff_grows_caps_and_resets_after_convergence() {
let mut backoff = ScannerRetryBackoff::default();
assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None);
@@ -8973,7 +8973,7 @@ fn superseded_retry_backoff_grows_caps_and_resets_after_convergence() {
}
#[test]
fn superseded_retry_backoff_respects_a_faster_configured_cycle() {
fn retry_backoff_respects_a_faster_configured_cycle() {
let mut backoff = ScannerRetryBackoff::default();
backoff.record_retryable_cycle(true);
@@ -8985,7 +8985,7 @@ fn superseded_retry_backoff_respects_a_faster_configured_cycle() {
}
#[test]
fn superseded_retry_backoff_grows_from_the_default_cycle() {
fn retry_backoff_grows_from_the_default_cycle() {
let mut backoff = ScannerRetryBackoff::default();
// The first race after a write burst retries in seconds, not a whole
// cycle, while repeated supersedes still climb toward the cap.
@@ -9005,7 +9005,7 @@ fn superseded_retry_preserves_explicit_cycle_cadence() {
..Default::default()
};
assert_eq!(
scanner_superseded_retry_interval(backoff, &default_config, ScannerMaintenanceFeatures::default()),
scanner_superseded_retry_interval(backoff, &default_config),
Some(SCANNER_RETRY_BASE_INTERVAL),
"the default adaptive cadence must retain fast convergence"
);
@@ -9015,24 +9015,24 @@ fn superseded_retry_preserves_explicit_cycle_cadence() {
ScannerRuntimeConfigSource::Config,
ScannerRuntimeConfigSource::ScannerCompatConfig,
] {
let runtime_config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(6 * 60 * 60),
cycle_interval_source: source,
..Default::default()
};
for features in [
ScannerMaintenanceFeatures::default(),
ScannerMaintenanceFeatures {
lifecycle: true,
for cycle_secs in [1, 3, 600, 6 * 60 * 60] {
let runtime_config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(cycle_secs),
cycle_interval_source: source,
..Default::default()
},
] {
assert_eq!(
scanner_superseded_retry_interval(backoff, &runtime_config, features),
Some(runtime_config.cycle_interval),
"{source:?} cycle cadence must not be shortened by convergence retries"
);
};
let mut backoff = ScannerRetryBackoff::default();
assert_eq!(scanner_superseded_retry_interval(backoff, &runtime_config), None);
for _ in 0..40 {
backoff.record_retryable_cycle(true);
assert_eq!(
scanner_superseded_retry_interval(backoff, &runtime_config),
Some(runtime_config.cycle_interval),
"{source:?} cycle cadence must hold throughout convergence retries"
);
}
backoff.record_retryable_cycle(false);
assert_eq!(scanner_superseded_retry_interval(backoff, &runtime_config), None);
}
}
}
@@ -9053,7 +9053,7 @@ fn superseded_retry_does_not_delay_required_maintenance() {
for consecutive_cycles in 1..=12 {
backoff.record_retryable_cycle(true);
assert_eq!(
scanner_superseded_retry_interval(backoff, &config, features),
scanner_superseded_retry_interval(backoff, &config),
Some(config.cycle_interval),
"superseded cycle {consecutive_cycles} must preserve regular maintenance"
);
@@ -9139,19 +9139,50 @@ fn scope_fallback_retry_preserves_explicit_and_fast_cycle_cadence() {
}
#[test]
fn superseded_retry_only_caps_regular_maintenance() {
fn superseded_retry_caps_at_the_default_cycle_and_resets_after_convergence() {
let config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(60),
..Default::default()
};
let mut backoff = ScannerRetryBackoff::default();
for _ in 0..6 {
assert_eq!(scanner_superseded_retry_interval(backoff, &config), None);
for expected in [5, 10, 20, 40, 60, 60, 60, 60] {
backoff.record_retryable_cycle(true);
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(Duration::from_secs(expected)));
}
backoff.consecutive_cycles = u32::MAX;
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(config.cycle_interval));
backoff.record_retryable_cycle(true);
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(config.cycle_interval));
backoff.record_retryable_cycle(false);
assert_eq!(scanner_superseded_retry_interval(backoff, &config), None);
}
#[test]
fn superseded_retry_caps_fast_cycles_without_changing_the_generic_retry_cap() {
let mut backoff = ScannerRetryBackoff::default();
for _ in 0..40 {
backoff.record_retryable_cycle(true);
}
assert_eq!(
scanner_superseded_retry_interval(backoff, &config, ScannerMaintenanceFeatures::default()),
Some(Duration::from_secs(160)),
backoff.retry_interval(Duration::from_secs(60)),
Some(Duration::from_secs(30 * 60)),
"deferred and transient retries must retain the generic cap"
);
for cycle_secs in [0, 1, 3, 60, 6 * 60 * 60] {
let config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(cycle_secs),
..Default::default()
};
assert_eq!(
scanner_superseded_retry_interval(backoff, &config),
Some(Duration::from_secs(cycle_secs.max(1))),
);
}
}
#[test]
fn superseded_retry_timer_keeps_required_maintenance_full_scans() {
for features in [
ScannerMaintenanceFeatures {
lifecycle: true,
@@ -9166,28 +9197,23 @@ fn superseded_retry_only_caps_regular_maintenance() {
..Default::default()
},
] {
assert_eq!(scanner_superseded_retry_interval(backoff, &config, features), Some(config.cycle_interval));
assert_eq!(scanner_superseded_retry_interval(ScannerRetryBackoff::default(), &config, features), None);
assert!(features.requires_full_scan(Some(7), 7, ScannerCycleWakeReason::Timer, true));
}
}
#[tokio::test(start_paused = true)]
#[serial]
async fn superseded_maintenance_wait_keeps_convergence_fencing() {
async fn superseded_retry_wait_keeps_convergence_fencing_at_the_default_cycle() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(60),
..Default::default()
};
let features = ScannerMaintenanceFeatures {
lifecycle: true,
..Default::default()
};
let mut backoff = ScannerRetryBackoff::default();
for _ in 0..12 {
backoff.record_retryable_cycle(true);
}
let interval = scanner_superseded_retry_interval(backoff, &config, features);
let interval = scanner_superseded_retry_interval(backoff, &config);
assert_eq!(interval, Some(config.cycle_interval));
let generations = ScannerCycleObservedGenerations::for_wait(
&config,
@@ -9202,19 +9228,28 @@ async fn superseded_maintenance_wait_keeps_convergence_fencing() {
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]));
let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]);
let started = tokio::time::Instant::now();
let reason = wait_for_next_scanner_cycle_with_activity(
let mut wait = Box::pin(wait_for_next_scanner_cycle_with_activity(
&ctx,
interval.expect("maintenance must retain its convergence interval"),
interval.expect("superseded retry must retain its convergence interval"),
Some(Duration::from_secs(1)),
&mut seen,
generations,
|| false,
|| std::future::ready(Ok(changed.clone())),
)
.await;
));
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
crate::scanner_io::record_dirty_usage_bucket("photos");
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
for _ in 0..3 {
tokio::time::advance(Duration::from_secs(15)).await;
crate::scanner_io::record_dirty_usage_bucket("photos");
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
}
let reason = wait.await;
assert_eq!(reason, ScannerCycleWakeReason::Timer);
assert_eq!(started.elapsed(), config.cycle_interval);
assert_eq!(seen, Some(changed));
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
}
#[test]
+105 -44
View File
@@ -59,6 +59,7 @@ use rustfs_scanner_metrics::metrics::{
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
UpdateCurrentPathFn, current_path_updater, global_metrics,
};
use rustfs_scanner_metrics::scan_observation::{ScannerCacheReuseDecision, ScannerCacheReuseGate, ScannerCacheReuseKind};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use sha2::{Digest, Sha256};
use time::OffsetDateTime;
@@ -1057,38 +1058,51 @@ impl FolderScanner {
child_hash: &DataUsageHash,
abandoned_children: &DataUsageHashMap,
) -> bool {
let Some(prefix_scan_scope) = &self.prefix_scan_scope else {
return false;
let decision = if let Some(prefix_scan_scope) = &self.prefix_scan_scope {
if folder.parent.is_some() || folder.name != self.old_cache.info.name {
ScannerCacheReuseDecision::NotRoot
} else if into.compacted {
ScannerCacheReuseDecision::CompactedParent
} else if !abandoned_children.contains(&child_hash.0) {
ScannerCacheReuseDecision::MissingCachedChild
} else if let Some(entry) = child
.name
.strip_prefix(folder.name.as_str())
.and_then(|entry| entry.strip_prefix('/'))
{
if entry.is_empty() || entry.contains('/') {
ScannerCacheReuseDecision::InvalidPrefix
} else if prefix_scan_scope.contains(entry) {
ScannerCacheReuseDecision::DirtyPrefix
} else if self.is_erasure_mode {
// EC scans must still perform independent object-health work.
global_metrics().scan_observation().record_erasure_clean_prefix_walk();
ScannerCacheReuseDecision::ErasureHealthRequired
} else {
ScannerCacheReuseDecision::Eligible
}
} else {
ScannerCacheReuseDecision::InvalidPrefix
}
} else {
ScannerCacheReuseDecision::MissingScope
};
// Erasure-mode usage scans also perform probabilistic object-health
// work. Reusing a clean subtree here would silently suppress that
// independent maintenance path, so prefix reuse is limited to the
// non-erasure data-usage scanner.
if self.is_erasure_mode
|| folder.parent.is_some()
|| folder.name != self.old_cache.info.name
|| into.compacted
|| !abandoned_children.contains(&clone_cache_value(&child_hash.0))
{
return false;
}
let Some(entry) = child
.name
.strip_prefix(folder.name.as_str())
.and_then(|entry| entry.strip_prefix('/'))
else {
return false;
};
!entry.is_empty() && !entry.contains('/') && !prefix_scan_scope.contains(entry)
global_metrics()
.scan_observation()
.record_cache_reuse_decision(ScannerCacheReuseGate::CleanPrefix, decision);
decision == ScannerCacheReuseDecision::Eligible
}
fn reuse_clean_root_child(&mut self, child: &CachedFolder, child_hash: &DataUsageHash, into: &mut DataUsageEntry) {
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
if let Some(updates) = &mut self.usage_updates {
updates.cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
}
into.add_child(child_hash);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::CleanPrefix, started.elapsed());
}
fn failed_retry_suppressed(&self, path: &str) -> bool {
@@ -1275,14 +1289,60 @@ impl FolderScanner {
}
fn can_reuse_covered_namespace(&self, name: &str, compacted: bool) -> bool {
self.old_cache.info.scan_progress.is_some()
&& !compacted
&& self
.resume_frontier
.as_deref()
.is_some_and(|frontier| scan_frontier_covers_path(name, frontier))
&& self.old_cache.find(name).is_some()
&& !self.resume_subtree_has_failures(name)
let decision = if self.old_cache.info.scan_progress.is_none() {
ScannerCacheReuseDecision::NoResumeSweep
} else if compacted {
ScannerCacheReuseDecision::CompactedParent
} else if !self
.resume_frontier
.as_deref()
.is_some_and(|frontier| scan_frontier_covers_path(name, frontier))
{
ScannerCacheReuseDecision::MissingCoverage
} else if self.old_cache.find(name).is_none() {
ScannerCacheReuseDecision::MissingCachedChild
} else if self.resume_subtree_has_failures(name) {
ScannerCacheReuseDecision::CachedFailures
} else {
ScannerCacheReuseDecision::Eligible
};
global_metrics()
.scan_observation()
.record_cache_reuse_decision(ScannerCacheReuseGate::CoveredNamespace, decision);
decision == ScannerCacheReuseDecision::Eligible
}
// None preserves work that cannot use sampling; false requests this cycle's rescan.
fn sampled_namespace_reuse(&self, hash: &DataUsageHash, compacted: bool, forward_sweep: bool) -> Option<bool> {
let decision = if self.scan_mode == HealScanMode::Deep {
ScannerCacheReuseDecision::DeepScan
} else if forward_sweep {
ScannerCacheReuseDecision::ForwardSweep
} else if self
.old_cache
.info
.replication
.as_ref()
.is_some_and(|config| config.has_active_rules("", true))
{
ScannerCacheReuseDecision::ActiveReplication
} else if compacted {
ScannerCacheReuseDecision::CompactedParent
} else if !self.old_cache.is_compacted(hash) {
ScannerCacheReuseDecision::UncompactedChild
} else if hash.mod_(self.old_cache.info.next_cycle as u32, data_usage_update_dir_cycles()) {
ScannerCacheReuseDecision::SelectedForWalk
} else {
ScannerCacheReuseDecision::Eligible
};
global_metrics()
.scan_observation()
.record_cache_reuse_decision(ScannerCacheReuseGate::NamespaceSampling, decision);
match decision {
ScannerCacheReuseDecision::Eligible => Some(true),
ScannerCacheReuseDecision::SelectedForWalk => Some(false),
_ => None,
}
}
fn resume_subtree_has_failures(&self, folder: &str) -> bool {
@@ -2072,7 +2132,11 @@ impl FolderScanner {
// The caller stitches this cached child into its
// parent; do not clone a growing child set into the
// unused provisional accumulator as well.
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, &this_hash, &folder.parent);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::ClassifiedNamespace, started.elapsed());
self.budget.record_entries_visited(pending_entry_progress);
self.finish_raw_enumeration_parent(&folder.name);
done_folder();
@@ -2659,8 +2723,12 @@ impl FolderScanner {
FolderScanMode::Namespace
};
if child_mode == FolderScanMode::Namespace && reuse_covered {
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
into.add_child(&h);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::CoveredNamespace, started.elapsed());
continue;
}
@@ -2700,22 +2768,15 @@ impl FolderScanner {
}
FolderScanSource::Existing => {
// Usage sampling is not proof that a Deep check ran.
if self.scan_mode != HealScanMode::Deep
&& !forward_sweep
&& !self
.old_cache
.info
.replication
.as_ref()
.is_some_and(|config| config.has_active_rules("", true))
&& !into.compacted
&& self.old_cache.is_compacted(&h)
{
let next_cycle = self.old_cache.info.next_cycle as u32;
if !h.mod_(next_cycle, data_usage_update_dir_cycles()) {
if let Some(reuse) = self.sampled_namespace_reuse(&h, into.compacted, forward_sweep) {
if reuse {
if child_mode == FolderScanMode::Namespace {
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
into.add_child(&h);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::SampledNamespace, started.elapsed());
self.record_scan_resume_hint(&folder_item.name);
continue;
}
+191
View File
@@ -3103,6 +3103,112 @@ async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
}
#[tokio::test]
#[serial]
async fn cache_reuse_gates_report_only_the_first_production_condition() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir);
scanner.old_cache.info.name = "bucket".to_string();
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
scanner.old_cache.replace(
"bucket/child",
"bucket",
DataUsageEntry {
compacted: true,
objects: 2,
..Default::default()
},
);
let hash = hash_path("bucket/child");
let folder = CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let child = CachedFolder {
name: "bucket/child".to_string(),
parent: Some(hash_path("bucket")),
object_heal_prob_div: 1,
};
let abandoned = HashSet::from([hash.0.clone()]);
let before = global_metrics().scan_observation().snapshot();
assert!(!scanner.should_reuse_clean_root_child(&folder, &DataUsageEntry::default(), &child, &hash, &abandoned));
let missing_scope = global_metrics().scan_observation().snapshot();
assert_eq!(
missing_scope.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"],
before.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"] + 1
);
scanner.prefix_scan_scope = ScannerBucketPrefixScanScope::from_dirty_top_level_entries(HashSet::from(["dirty".to_string()]));
scanner.is_erasure_mode = true;
assert!(!scanner.should_reuse_clean_root_child(&folder, &DataUsageEntry::default(), &child, &hash, &abandoned));
let erasure = global_metrics().scan_observation().snapshot();
assert_eq!(
erasure.reuse_decisions["clean_prefix"].blocked["erasure_health_required"],
before.reuse_decisions["clean_prefix"].blocked["erasure_health_required"] + 1
);
assert_eq!(erasure.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks + 1);
assert!(!scanner.can_reuse_covered_namespace("bucket/child", true));
scanner.old_cache.info.scan_progress = Some(crate::DataUsageScanProgress {
started_plan: crate::DataUsageScanPlanDigest([1; 32]),
requested_plan: crate::DataUsageScanPlanDigest([1; 32]),
});
assert!(!scanner.can_reuse_covered_namespace("bucket/child", true));
assert!(!scanner.can_reuse_covered_namespace("bucket/child", false));
scanner.resume_frontier = Some("bucket/child".to_string());
scanner.resume_failed_paths = vec!["bucket/child/failed".to_string()];
assert!(!scanner.can_reuse_covered_namespace("bucket/child", false));
scanner.resume_failed_paths.clear();
assert!(scanner.can_reuse_covered_namespace("bucket/child", false));
let covered = global_metrics().scan_observation().snapshot();
let reasons = &covered.reuse_decisions["covered_namespace"];
let prior = &before.reuse_decisions["covered_namespace"];
assert_eq!(reasons.not_evaluated["no_resume_sweep"], prior.not_evaluated["no_resume_sweep"] + 1);
assert_eq!(reasons.blocked["compacted_parent"], prior.blocked["compacted_parent"] + 1);
assert_eq!(reasons.blocked["missing_coverage"], prior.blocked["missing_coverage"] + 1);
assert_eq!(reasons.blocked["cached_failures"], prior.blocked["cached_failures"] + 1);
assert_eq!(reasons.eligible, prior.eligible + 1);
with_var(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, Some("16"), || {
scanner.scan_mode = HealScanMode::Deep;
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, true), None);
scanner.scan_mode = HealScanMode::Normal;
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, true), None);
let replication: ReplicationConfig = serde_json::from_value(serde_json::json!({
"config": { "role": "", "rules": [{ "destination": { "bucket": "arn:test" }, "id": "status", "prefix": "", "priority": 1, "status": "Enabled" }] },
"remotes": crate::storage_api::EcstoreBucketTargets::default(),
})).expect("active replication fixture");
scanner.old_cache.info.replication = Some(Arc::new(replication));
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, false), None);
scanner.old_cache.info.replication = None;
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, false), None);
let rescan_cycle = (0..16)
.find(|cycle| hash.mod_(*cycle, 16))
.expect("hash is visited in one sampling cycle");
scanner.old_cache.info.next_cycle = u64::from(rescan_cycle);
assert_eq!(scanner.sampled_namespace_reuse(&hash, false, false), Some(false));
scanner.old_cache.info.next_cycle = u64::from((rescan_cycle + 1) % 16);
assert_eq!(scanner.sampled_namespace_reuse(&hash, false, false), Some(true));
});
let after = global_metrics().scan_observation().snapshot();
let reasons = &after.reuse_decisions["namespace_sampling"];
let prior = &before.reuse_decisions["namespace_sampling"];
for label in ["deep_scan", "forward_sweep", "active_replication"] {
assert_eq!(
reasons.not_evaluated[label],
prior.not_evaluated[label] + 1,
"first sampling condition {label}"
);
}
assert_eq!(reasons.blocked["compacted_parent"], prior.blocked["compacted_parent"] + 1);
assert_eq!(reasons.blocked["selected_for_walk"], prior.blocked["selected_for_walk"] + 1);
assert_eq!(reasons.eligible, prior.eligible + 1);
for (kind, reuse) in &after.cache_reuse {
assert_eq!(reuse.count, before.cache_reuse[kind].count, "gate admission alone does not copy {kind}");
}
}
#[tokio::test]
#[serial]
async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entries() {
@@ -3144,12 +3250,21 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
parent: None,
object_heal_prob_div: 1,
};
let before = global_metrics().scan_observation().snapshot();
let mut root = DataUsageEntry::default();
scanner
.scan_folder(CancellationToken::new(), folder, &mut root)
.await
.expect("scoped root scan should finish successfully");
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(observed.cache_reuse["clean_prefix"].count, before.cache_reuse["clean_prefix"].count + 1);
assert_eq!(observed.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks);
assert_eq!(
observed.reuse_decisions["clean_prefix"].eligible,
before.reuse_decisions["clean_prefix"].eligible + 1
);
let clean = scanner
.new_cache
.size_recursive("bucket/clean")
@@ -3167,6 +3282,78 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
assert_eq!((bucket.size, bucket.objects), (17, 3));
}
#[tokio::test]
#[serial]
async fn scoped_cache_observations_distinguish_checkpoint_and_periodic_sampling() {
for covered in [true, false] {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
tokio::fs::create_dir_all(temp_dir.join("bucket/child"))
.await
.expect("create cached namespace");
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
scanner.old_cache.replace(
"bucket/child",
"bucket",
DataUsageEntry {
size: 17,
objects: 5,
compacted: true,
..Default::default()
},
);
let kind = if covered {
scanner.old_cache.info.scan_progress = Some(crate::DataUsageScanProgress {
started_plan: crate::DataUsageScanPlanDigest([1; 32]),
requested_plan: crate::DataUsageScanPlanDigest([1; 32]),
});
scanner.resume_frontier = Some("bucket/child".to_string());
"covered_namespace"
} else {
let hash = hash_path("bucket/child");
let cycle = (0..data_usage_update_dir_cycles())
.find(|cycle| !hash.mod_(*cycle, data_usage_update_dir_cycles()))
.expect("a cycle samples other work");
scanner.old_cache.info.next_cycle = u64::from(cycle);
"sampled_namespace"
};
let before = global_metrics().scan_observation().snapshot();
let mut root = DataUsageEntry::default();
scanner
.scan_folder(
CancellationToken::new(),
CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
},
&mut root,
)
.await
.expect("cached namespace reuses accounting");
let summary = scanner.new_cache.size_recursive("bucket").expect("complete bucket totals");
assert_eq!((summary.size, summary.objects), (17, 5));
let after = global_metrics().scan_observation().snapshot();
for label in [
"clean_prefix",
"covered_namespace",
"sampled_namespace",
"classified_namespace",
] {
assert_eq!(
after.cache_reuse[label].count,
before.cache_reuse[label].count + u64::from(label == kind),
"reuse reason {label}"
);
}
assert_eq!(after.metadata_reads.attempts, before.metadata_reads.attempts);
let gate = if covered { "covered_namespace" } else { "namespace_sampling" };
assert_eq!(after.reuse_decisions[gate].eligible, before.reuse_decisions[gate].eligible + 1);
}
}
async fn scan_hot_cold_segment_fixture(scoped: bool) -> (DataUsageEntry, Vec<String>) {
let (mut scanner, temp_dir) = build_test_scanner().await;
let visits = metadata_visits::observe(&mut scanner);
@@ -3278,6 +3465,7 @@ async fn scoped_root_scan_preserves_erasure_health_walks() {
parent: None,
object_heal_prob_div: 1,
};
let before = global_metrics().scan_observation().snapshot();
let mut root = DataUsageEntry::default();
scanner
.scan_folder(CancellationToken::new(), folder, &mut root)
@@ -3289,6 +3477,9 @@ async fn scoped_root_scan_preserves_erasure_health_walks() {
.size_recursive("bucket/clean")
.expect("erasure scan should visit the clean entry");
assert_eq!((clean.size, clean.objects), (0, 0));
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(observed.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks + 1);
assert_eq!(observed.cache_reuse["clean_prefix"].count, before.cache_reuse["clean_prefix"].count);
}
#[tokio::test]
@@ -633,6 +633,7 @@ async fn erasure_candidate_covered_namespace_reuses_totals_after_classification(
},
);
scanner.budget = budget.clone();
let before = global_metrics().scan_observation().snapshot();
scanner
.scan_folder(
budget.token(),
@@ -649,6 +650,15 @@ async fn erasure_candidate_covered_namespace_reuses_totals_after_classification(
assert_eq!(budget.progress(), (0, 1), "only the parent consumes namespace quota");
assert_eq!(charged_ops(&root), 4, "list both directories once without re-reading covered metadata");
assert_eq!(scanner.new_cache.find(&folder.name).expect("reused candidate").objects, 1);
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(
observed.cache_reuse["classified_namespace"].count,
before.cache_reuse["classified_namespace"].count + 1
);
assert_eq!(
observed.metadata_reads.attempts, before.metadata_reads.attempts,
"classification retains the existing zero metadata read path"
);
}
#[tokio::test]
@@ -728,6 +738,7 @@ async fn erasure_candidate_preserves_compacted_namespace_sampling_after_classifi
},
);
scanner.budget = budget.clone();
let before = global_metrics().scan_observation().snapshot();
scanner
.scan_folder(
budget.token(),
@@ -748,6 +759,15 @@ async fn erasure_candidate_preserves_compacted_namespace_sampling_after_classifi
"sampling still requires full current classification, without metadata re-read"
);
assert_eq!(scanner.new_cache.find(&folder.name).expect("sampled candidate").objects, 1);
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(
observed.cache_reuse["classified_namespace"].count,
before.cache_reuse["classified_namespace"].count + 1
);
assert_eq!(
observed.metadata_reads.attempts, before.metadata_reads.attempts,
"classification retains the existing zero metadata read path"
);
}
#[tokio::test]
+133 -46
View File
@@ -35,6 +35,7 @@ use rustfs_lock::{LockError, NamespaceLockGuard};
use rustfs_scanner_metrics::metrics::{
Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics,
};
use rustfs_scanner_metrics::scan_observation::{PREFIX_REUSE_BLOCKERS, ScannerBaselineRejection, ScannerPrefixReuseStage};
use rustfs_utils::path::path_join_buf;
use s3s::dto::{
BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration, VersioningConfiguration,
@@ -110,16 +111,7 @@ pub(crate) const SCANNER_SEGMENT_ACTIVATION_PROOF_INPUTS: [&str; 9] = [
"durable_dirty_producer_journal",
"ack_generation_guard",
];
pub(crate) const SCANNER_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS: [&str; 8] = [
"missing_producer_identity",
"missing_durable_journal_replay",
"restart_gap",
"generation_gap",
"overflow",
"stale_ack_generation",
"missing_cold_zero_walk_oracle",
"distributed_without_peer_invalidation",
];
pub(crate) const SCANNER_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS: [&str; 8] = PREFIX_REUSE_BLOCKERS;
pub type DirtyUsageBuckets = HashMap<String, u64>;
@@ -195,6 +187,7 @@ struct VerifiedRemoteDirtyUsage {
}
struct ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: Option<ScannerSegmentReuseActivationPreflight>,
scope: ScannerBucketScanScope,
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
distributed_segment_invalidation_evidence: Option<DistributedSegmentInvalidationEvidence>,
@@ -234,6 +227,13 @@ pub(crate) struct ScannerSegmentReuseActivationPreflight {
}
impl ScannerSegmentReuseActivationPreflight {
fn record_observation(&self, stage: ScannerPrefixReuseStage) {
global_metrics().scan_observation().record_prefix_reuse(
stage,
self.scanner_segment_reuse_activated,
&self.fail_closed_blockers,
);
}
#[cfg(test)]
pub(crate) fn fail_closed_blockers(&self) -> impl Iterator<Item = &'static str> + '_ {
self.fail_closed_blockers.iter().filter_map(|blocker| *blocker)
@@ -317,6 +317,7 @@ fn resolve_remote_dirty_usage_scope(
baseline_proof: ScannerCacheBaselineProof<'_>,
) -> ScannerBucketScopeResolutionResult {
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: None,
scope,
remote_dirty_usage_acknowledgements: Vec::new(),
distributed_segment_invalidation_evidence: None,
@@ -408,6 +409,7 @@ fn resolve_remote_dirty_usage_scope(
};
ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: Some(segment_reuse_activation_preflight),
scope,
remote_dirty_usage_acknowledgements: scoped_acknowledgements,
distributed_segment_invalidation_evidence,
@@ -425,43 +427,81 @@ fn complete_scanner_cache_snapshot_plan_digest(
proof: ScannerCacheBaselineProof<'_>,
expected_converged: Option<bool>,
) -> Option<DataUsageScanPlanDigest> {
validate_scanner_cache_snapshot_plan_digest(snapshot, proof, expected_converged).ok()
}
fn validate_scanner_cache_snapshot_plan_digest(
snapshot: &DataUsageInfo,
proof: ScannerCacheBaselineProof<'_>,
expected_converged: Option<bool>,
) -> std::result::Result<DataUsageScanPlanDigest, ScannerBaselineRejection> {
// The observational mark disqualifies only the authoritative slot: an
// observed candidate carries it by construction.
let marks_accepted = match expected_converged {
Some(expected) => snapshot.usage_snapshot_converged == Some(expected),
None => snapshot.usage_snapshot_converged.is_some() && !snapshot.usage_snapshot_observational,
};
if !snapshot.is_complete_bucket_usage_snapshot()
|| snapshot.usage_snapshot_partial
|| !marks_accepted
|| snapshot.scanner_epoch != Some(proof.leader_epoch)
|| snapshot.usage_snapshot_set_states.len() != proof.expected_sources.len()
{
return None;
if !snapshot.is_complete_bucket_usage_snapshot() {
return Err(ScannerBaselineRejection::Incomplete);
}
if snapshot.usage_snapshot_partial {
return Err(ScannerBaselineRejection::Partial);
}
if !marks_accepted {
return Err(ScannerBaselineRejection::Marks);
}
if snapshot.scanner_epoch != Some(proof.leader_epoch) {
return Err(ScannerBaselineRejection::Epoch);
}
if snapshot.usage_snapshot_set_states.len() != proof.expected_sources.len() {
return Err(ScannerBaselineRejection::SourceCount);
}
// Completed maintenance also covers ordinary usage. Keep its exact stored
// proof for cache reuse, and reject mixtures of different set work proofs.
let baseline_plan_digest = DataUsageScanPlanDigest(snapshot.usage_snapshot_set_states.first()?.scan_plan_digest?);
let baseline_plan_digest = DataUsageScanPlanDigest(
snapshot
.usage_snapshot_set_states
.first()
.and_then(|state| state.scan_plan_digest)
.ok_or(ScannerBaselineRejection::PlanMissing)?,
);
if !scanner_plan_digest_is_work_variant(proof.scan_plan_digest, baseline_plan_digest) {
return None;
return Err(ScannerBaselineRejection::PlanMismatch);
}
let mut states = HashSet::with_capacity(snapshot.usage_snapshot_set_states.len());
for state in &snapshot.usage_snapshot_set_states {
let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?);
if !proof.expected_sources.contains(&source)
|| !states.insert(source)
|| !state.complete
|| state.tombstone
|| state.scanner_epoch != Some(proof.leader_epoch)
|| state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle)
|| state.scan_plan_digest != Some(baseline_plan_digest.0)
{
return None;
let source = DataUsageCacheSource::new(
usize::try_from(state.pool_index).map_err(|_| ScannerBaselineRejection::SourceIndex)?,
usize::try_from(state.set_index).map_err(|_| ScannerBaselineRejection::SourceIndex)?,
);
if !proof.expected_sources.contains(&source) {
return Err(ScannerBaselineRejection::SourceMembership);
}
if !states.insert(source) {
return Err(ScannerBaselineRejection::SourceDuplicate);
}
if !state.complete {
return Err(ScannerBaselineRejection::SetIncomplete);
}
if state.tombstone {
return Err(ScannerBaselineRejection::SetTombstone);
}
if state.scanner_epoch != Some(proof.leader_epoch) {
return Err(ScannerBaselineRejection::SetEpoch);
}
if state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle) {
return Err(ScannerBaselineRejection::SetCycle);
}
if state.scan_plan_digest != Some(baseline_plan_digest.0) {
return Err(ScannerBaselineRejection::SetPlan);
}
}
(states == *proof.expected_sources).then_some(baseline_plan_digest)
if states != *proof.expected_sources {
return Err(ScannerBaselineRejection::SourceCoverage);
}
Ok(baseline_plan_digest)
}
/// Whether `digest` is `structural` under any work flag: the same bucket
@@ -491,10 +531,35 @@ fn scanner_baseline_pending_buckets(proof: &ScannerCacheBaselineProof<'_>) -> Ve
}
fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option<DataUsageScanPlanDigest> {
let authoritative = serde_json::from_slice::<DataUsageInfo>(proof.authoritative_data?).ok()?;
if let Some(validated_digest) = complete_scanner_cache_snapshot_plan_digest(&authoritative, proof, None) {
return Some(validated_digest);
}
validate_scanner_cache_baseline_plan_digest(proof).ok()
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct ScannerBaselineValidationFailure {
authoritative: ScannerBaselineRejection,
observed: Option<ScannerBaselineRejection>,
}
fn validate_scanner_cache_baseline_plan_digest(
proof: ScannerCacheBaselineProof<'_>,
) -> std::result::Result<DataUsageScanPlanDigest, ScannerBaselineValidationFailure> {
let authoritative_error = |authoritative| ScannerBaselineValidationFailure {
authoritative,
observed: None,
};
let authoritative_data = proof
.authoritative_data
.ok_or_else(|| authoritative_error(ScannerBaselineRejection::Missing))?;
let authoritative = serde_json::from_slice::<DataUsageInfo>(authoritative_data)
.map_err(|_| authoritative_error(ScannerBaselineRejection::Decode))?;
let authoritative_rejection = match validate_scanner_cache_snapshot_plan_digest(&authoritative, proof, None) {
Ok(validated_digest) => return Ok(validated_digest),
Err(reason) => reason,
};
let observed_error = |observed| ScannerBaselineValidationFailure {
authoritative: authoritative_rejection,
observed: Some(observed),
};
// A complete but superseded observation may reuse its per-set cache only
// when it was explicitly tied to the durable authoritative baseline. It
@@ -504,14 +569,18 @@ fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<
&& !authoritative.usage_snapshot_observational)
|| crate::scanner::data_usage_info_is_bootstrap_pending(&authoritative);
if !authoritative_has_identity {
return None;
return Err(observed_error(ScannerBaselineRejection::AuthoritativeIdentity));
}
let observed = serde_json::from_slice::<DataUsageInfo>(proof.observed_candidate_data?).ok()?;
let observed_data = proof
.observed_candidate_data
.ok_or_else(|| observed_error(ScannerBaselineRejection::Missing))?;
let observed =
serde_json::from_slice::<DataUsageInfo>(observed_data).map_err(|_| observed_error(ScannerBaselineRejection::Decode))?;
if !observed_data_usage_is_newer(&observed, &authoritative) {
return None;
return Err(observed_error(ScannerBaselineRejection::NotNewer));
}
complete_scanner_cache_snapshot_plan_digest(&observed, proof, Some(false))
validate_scanner_cache_snapshot_plan_digest(&observed, proof, Some(false)).map_err(observed_error)
}
fn scanner_segment_reuse_baseline_producer_evidence(
@@ -656,6 +725,12 @@ impl ScannerScopeFallbackReason {
}
}
#[derive(Clone, Copy)]
struct ScannerScopeFallback {
reason: ScannerScopeFallbackReason,
baseline_rejection: Option<ScannerBaselineValidationFailure>,
}
/// Classify a resolved full-namespace scope against the preconditions
/// `resolve_scanner_bucket_scan_scope` checks, in the same order. Returns
/// `None` when the cycle was required to walk everything or was scoped.
@@ -666,18 +741,24 @@ fn scanner_scope_fallback_reason(
all_buckets: &[BucketInfo],
baseline_proof: ScannerCacheBaselineProof<'_>,
distributed: bool,
) -> Option<ScannerScopeFallbackReason> {
) -> Option<ScannerScopeFallback> {
let fallback = |reason| {
Some(ScannerScopeFallback {
reason,
baseline_rejection: None,
})
};
if !scope_eligible || !resolved_scope.is_default() {
return None;
}
if dirty_usage_snapshot.generation == u64::MAX {
return Some(ScannerScopeFallbackReason::DirtyGenerationExhausted);
return fallback(ScannerScopeFallbackReason::DirtyGenerationExhausted);
}
if !dirty_usage_snapshot.covers_listed_pending {
return Some(ScannerScopeFallbackReason::DirtySnapshotIncomplete);
return fallback(ScannerScopeFallbackReason::DirtySnapshotIncomplete);
}
if dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES {
return Some(ScannerScopeFallbackReason::DirtyBucketOverflow);
return fallback(ScannerScopeFallbackReason::DirtyBucketOverflow);
}
// Peers may contribute dirty buckets the local snapshot does not hold.
if !distributed {
@@ -687,13 +768,19 @@ fn scanner_scope_fallback_reason(
.keys()
.any(|bucket| listed.contains(bucket.as_str()))
{
return Some(ScannerScopeFallbackReason::NoListedDirtyBuckets);
return fallback(ScannerScopeFallbackReason::NoListedDirtyBuckets);
}
}
if complete_scanner_cache_baseline_plan_digest(baseline_proof).is_none() {
return Some(ScannerScopeFallbackReason::BaselineUnavailable);
if let Err(failure) = validate_scanner_cache_baseline_plan_digest(baseline_proof) {
return Some(ScannerScopeFallback {
reason: ScannerScopeFallbackReason::BaselineUnavailable,
baseline_rejection: Some(failure),
});
}
distributed.then_some(ScannerScopeFallbackReason::PeerScopeUnavailable)
distributed.then_some(ScannerScopeFallback {
reason: ScannerScopeFallbackReason::PeerScopeUnavailable,
baseline_rejection: None,
})
}
fn scanner_segment_reuse_activation_preflight() -> ScannerSegmentReuseActivationPreflight {
+11 -7
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use super::*;
/// scanner cache locks and the cache snapshot persist/publish path.
use crate::data_usage_define::clone_cache_value;
use crate::data_usage_define::{DataUsageCacheBaselineRevisions, clone_cache_value};
use crate::{ScannerConfigObjectDelete, ScannerObjectIO};
use rustfs_scanner_metrics::usage_candidate::{
ScannerUsageCandidateKind as CandidateKind, ScannerUsageCandidateOutcome as CandidateOutcome,
@@ -1494,7 +1494,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
store: Arc<SetDisks>,
updates: &mpsc::Sender<DataUsageCache>,
mut cache_snapshot: DataUsageCache,
initial_revisions: Option<&DataUsageCacheRevisions>,
initial_revisions: Option<&DataUsageCacheBaselineRevisions>,
cache_cycle_floor: &AtomicU64,
expected_publication_epoch: u64,
) -> Option<SystemTime> {
@@ -1536,7 +1536,10 @@ pub(super) async fn persist_and_publish_cache_snapshot(
};
let mut persisted = DataUsageCache::default();
let revisions = match persisted.load_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME).await {
let current_baseline = match persisted
.load_with_baseline_revisions(store.clone(), DATA_USAGE_CACHE_NAME)
.await
{
Ok(revisions) => revisions,
Err(err) => {
error!(
@@ -1558,6 +1561,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
return None;
}
};
let revisions = current_baseline.revisions();
let Some(scan_plan_digest) = cache_snapshot.info.scan_plan_digest else {
log_missing_set_snapshot_metadata(&store, &cache_snapshot.info, "plan_digest");
return None;
@@ -1621,9 +1625,9 @@ pub(super) async fn persist_and_publish_cache_snapshot(
{
cache_snapshot = persisted;
} else {
// A later execution may have completed while this scan was walking.
// Only replace the cache revision from which this scan started.
if initial_revisions != Some(&revisions) {
// Another execution must not advance the baseline. A backup catching
// up to the same decoded primary changes no baseline content.
if !initial_revisions.is_some_and(|initial| current_baseline.permits_unchanged_baseline_since(initial)) {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
@@ -1662,7 +1666,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache_snapshot
.save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch)
.save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, revisions, expected_publication_epoch)
.await
{
error!(
+48 -3
View File
@@ -14,6 +14,7 @@
use super::*;
/// ScannerIOCache implementation for SetDisks: bucket ordering, worker fan-out, merge, and publish.
use crate::data_usage_define::clone_cache_value;
use rustfs_scanner_metrics::scan_observation::{ScannerBucketReuseKind, ScannerScanObservation};
pub(super) async fn collect_completed_bucket_results(
ctx: CancellationToken,
@@ -142,6 +143,32 @@ pub(super) struct PreparedScopedSetScan {
pub(super) buckets: Vec<BucketInfo>,
pub(super) cache: DataUsageCache,
pub(super) cold_bucket_reuse_proof: Option<ScopedColdBucketReuseProof>,
cold_bucket_reuse_count: u64,
cold_bucket_reuse_elapsed: Duration,
}
impl PreparedScopedSetScan {
pub(super) fn record_cold_bucket_reuse(&self, observation: &ScannerScanObservation) {
observation.record_bucket_reuse(
ScannerBucketReuseKind::ScopedColdBucket,
self.cold_bucket_reuse_count,
self.cold_bucket_reuse_elapsed,
);
}
}
/// Counts only roots delivered to the collector; cancellation or channel failure is not reuse.
pub(super) async fn send_current_cache_root_entry(
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
root: DataUsageEntryInfo,
cache: &DataUsageCache,
pending_maintenance_work: &AtomicBool,
observation: &ScannerScanObservation,
) -> std::result::Result<(), ScannerError> {
let started = Instant::now();
send_cache_root_entry(bucket_result_tx, root, cache, pending_maintenance_work).await?;
observation.record_bucket_reuse(ScannerBucketReuseKind::CurrentRoot, 1, started.elapsed());
Ok(())
}
pub(super) struct ScopedColdBucketReuseProof {
@@ -225,6 +252,8 @@ pub(super) fn prepare_scoped_set_scan(
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
let mut current_bucket_names = HashSet::with_capacity(all_buckets.len());
let mut cold_bucket_reuse_count = 0u64;
let mut cold_bucket_reuse_elapsed = Duration::ZERO;
for bucket in all_buckets {
if !current_bucket_names.insert(bucket.name.as_str()) {
return None;
@@ -233,11 +262,14 @@ pub(super) fn prepare_scoped_set_scan(
cache.info.scan_bucket_value_origins.remove(&bucket.name);
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
} else {
let started = Instant::now();
cache.copy_with_children(
old_cache,
&rustfs_data_usage::hash_path(&bucket.name),
&Some(rustfs_data_usage::hash_path(DATA_USAGE_ROOT)),
);
cold_bucket_reuse_count = cold_bucket_reuse_count.saturating_add(1);
cold_bucket_reuse_elapsed = cold_bucket_reuse_elapsed.saturating_add(started.elapsed());
}
}
@@ -249,6 +281,8 @@ pub(super) fn prepare_scoped_set_scan(
.collect(),
cache,
cold_bucket_reuse_proof,
cold_bucket_reuse_count,
cold_bucket_reuse_elapsed,
})
}
@@ -425,7 +459,10 @@ impl ScannerIOCache for SetDisks {
.ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?,
};
let mut old_cache = DataUsageCache::default();
let initial_revisions = match old_cache.load_with_revisions(self.clone(), DATA_USAGE_CACHE_NAME).await {
let initial_revisions = match old_cache
.load_with_baseline_revisions(self.clone(), DATA_USAGE_CACHE_NAME)
.await
{
Ok(revisions) => Some(revisions),
Err(e) => {
warn!(
@@ -555,6 +592,8 @@ impl ScannerIOCache for SetDisks {
})
});
let mut scoped_cache = scoped_scan.map(|mut prepared| {
// The copies are completed work even if this turn later fails publication.
prepared.record_cold_bucket_reuse(global_metrics().scan_observation());
buckets = prepared.buckets;
prepared.cache.info.scan_coverage_digest = Some(bucket_coverage_digest);
prepared.cache
@@ -1558,8 +1597,14 @@ impl ScannerIOCache for SetDisks {
}
record_completed_usage_watch(&usage_watch_completions_clone, source, &bucket.name, bucket_usage_watch.as_ref()).await;
if let Err(e) =
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
.await
send_current_cache_root_entry(
&bucket_result_tx_clone,
*root,
&cache,
&pending_maintenance_work_clone,
global_metrics().scan_observation(),
)
.await
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
+20 -3
View File
@@ -357,6 +357,7 @@ where
S: ScannerStorage,
{
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: None,
scope,
remote_dirty_usage_acknowledgements: Vec::new(),
distributed_segment_invalidation_evidence: None,
@@ -506,7 +507,7 @@ where
distributed,
resolution.baseline_proof,
);
default_result(scoped_scan_scope_from_dirty_buckets(
let scope = scoped_scan_scope_from_dirty_buckets(
resolution.requested_scope,
dirty_buckets,
(!distributed).then_some(resolution.dirty_usage_snapshot.scopes.as_ref()),
@@ -514,7 +515,11 @@ where
segment_reuse_activation_preflight.scanner_segment_reuse_activated,
resolution.all_buckets,
resolution.baseline_proof,
))
);
ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: Some(segment_reuse_activation_preflight),
..default_result(scope)
}
}
/// Runs once between the bucket listing and the dirty-usage snapshot, the
@@ -759,6 +764,9 @@ where
},
)
.await;
if let Some(preflight) = scope_resolution.segment_reuse_activation_preflight {
preflight.record_observation(ScannerPrefixReuseStage::ScopeResolution);
}
let remote_dirty_usage_acknowledgements = scope_resolution.remote_dirty_usage_acknowledgements;
let distributed_segment_invalidation_evidence = scope_resolution.distributed_segment_invalidation_evidence;
let scan_scope = scope_resolution.scope;
@@ -770,8 +778,14 @@ where
baseline_proof,
distributed,
);
if let Some(reason) = scope_fallback {
if let Some(fallback) = scope_fallback {
let reason = fallback.reason;
global_metrics().record_scanner_scope_fallback(reason.as_str());
if let Some(failure) = fallback.baseline_rejection {
global_metrics()
.scan_observation()
.record_baseline_rejection(failure.authoritative, failure.observed);
}
tracing::info!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
@@ -785,6 +799,7 @@ where
"Scanner cycle walks every bucket because its dirty-bucket scope was not admitted"
);
}
let scope_fallback = scope_fallback.map(|fallback| fallback.reason);
let segment_invalidation_proof = dirty_usage_producer_evidence(&dirty_usage_snapshot).segment_invalidation_proof();
#[cfg(test)]
if let Some(observer) = resolved_scope_observer {
@@ -861,6 +876,7 @@ where
None,
false,
);
segment_reuse_activation_preflight.record_observation(ScannerPrefixReuseStage::CycleCompletion);
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_activity_digest(activity_digest)
@@ -1207,6 +1223,7 @@ where
distributed_segment_invalidation_evidence,
cold_zero_walk_oracle,
);
segment_reuse_activation_preflight.record_observation(ScannerPrefixReuseStage::CycleCompletion);
let pending_maintenance_work = pending_maintenance_work_for_cycle(&pending_maintenance_work, &results);
let observed_cycle_floor = cache_cycle_floor.load(Ordering::Acquire);
let required_cycle_floor = (observed_cycle_floor > want_cycle).then_some(observed_cycle_floor);
+15 -2
View File
@@ -15,6 +15,7 @@ use super::*;
use crate::UNKNOWN_TIER;
/// ScannerIODisk implementation for Disk: get_size and the per-disk bucket scan.
use crate::data_usage_define::clone_cache_value;
use rustfs_scanner_metrics::scan_observation::ScannerMetadataReadOutcome;
///
/// Seed [`SizeSummary::tier_stats`] from the cached tier-name list.
@@ -62,8 +63,20 @@ impl ScannerIODisk for Disk {
ensure_scanner_node_operation_active(cancellation.as_ref())?;
let metadata_object_path = item.object_path();
let metadata_result =
crate::workload_admission::with_scanner_metadata_io(self.read_metadata(&item.bucket, &metadata_object_path)).await;
let metadata_result = crate::workload_admission::with_scanner_metadata_io(async {
let read = global_metrics().start_scanner_metadata_read();
let result = self.read_metadata(&item.bucket, &metadata_object_path).await;
let (outcome, bytes) = match &result {
Ok(data) => (ScannerMetadataReadOutcome::Success, u64::try_from(data.len()).unwrap_or(u64::MAX)),
Err(e) if DiskError::is_err_object_not_found(e) || DiskError::is_err_version_not_found(e) => {
(ScannerMetadataReadOutcome::NotFound, 0)
}
Err(_) => (ScannerMetadataReadOutcome::Error, 0),
};
read.finish(outcome, bytes);
result
})
.await;
ensure_scanner_node_operation_active(cancellation.as_ref())?;
let data = match metadata_result {
Ok(data) => data,
+558 -7
View File
@@ -27,10 +27,10 @@ use crate::storage_api::owner::{
};
use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
use crate::{
DATA_USAGE_BLOOM_NAME_PATH, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints,
ScannerObjectOptions, ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests,
init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
save_config,
BUCKET_META_PREFIX, DATA_USAGE_BLOOM_NAME_PATH, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
InstanceContext, PoolEndpoints, ScannerObjectOptions, ScannerPutObjReader, UNKNOWN_TIER,
init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
new_disk, path2_bucket_object_with_base_path, save_config,
};
use rustfs_concurrency::{
AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider,
@@ -104,6 +104,7 @@ fn scanner_activity_preflight_defers_a_temporarily_offline_peer() {
#[test]
fn scanner_segment_reuse_activation_preflight_reports_release_gate_inputs() {
let before = global_metrics().scan_observation().snapshot();
let preflight = scanner_segment_reuse_activation_preflight();
assert!(preflight.production_activation);
@@ -124,6 +125,9 @@ fn scanner_segment_reuse_activation_preflight_reports_release_gate_inputs() {
"distributed_without_peer_invalidation",
]
);
let after = global_metrics().scan_observation().snapshot();
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated);
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated);
}
#[test]
@@ -2386,6 +2390,254 @@ async fn set_cache_send_failure_logs_receiver_and_snapshot_without_changing_retu
assert_set_cache_context(set_cache_event(&events, "publish_failed"), Some((0, 0)));
}
#[tokio::test]
#[serial]
async fn set_snapshot_accepts_backup_catchup_to_the_same_decoded_primary() {
use crate::data_usage_define::read_config_with_revision;
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
let set = Arc::clone(&store.pools[0].disk_set[0]);
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
let backup_path = format!("{primary_path}.bkp");
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read decoded primary bytes");
let primary_body = primary_body.expect("primary exists");
let mut older = cache.clone();
older.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
save_config(Arc::clone(&set), &backup_path, older.marshal_msg().expect("encode older backup"))
.await
.expect("seed a lagging backup");
let mut loaded = DataUsageCache::default();
let initial = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture primary with lagging backup");
save_config(Arc::clone(&set), &backup_path, primary_body)
.await
.expect("backup catches up without changing primary");
let current = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture the reconciled backup");
assert_ne!(initial.revisions(), current.revisions());
assert!(current.permits_unchanged_baseline_since(&initial));
let mut candidate = cache;
candidate.info.next_cycle += 1;
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
candidate.replace(
"photos",
DATA_USAGE_ROOT,
DataUsageEntry {
size: 20,
objects: 1,
..Default::default()
},
);
let cycle = candidate.info.next_cycle;
let cycle_floor = AtomicU64::new(cycle);
let (tx, mut rx) = mpsc::channel(1);
assert!(
persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, candidate, Some(&initial), &cycle_floor, epoch)
.await
.is_some(),
"content-preserving backup catchup must not veto the new snapshot"
);
let published = rx.try_recv().expect("new snapshot reached the collector");
assert_eq!(published.info.next_cycle, cycle);
assert_eq!(published.find("photos").expect("new bucket entry").size, 20);
let (primary, primary_revision) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read new primary");
let (backup, backup_revision) = read_config_with_revision(set, &backup_path).await.expect("read new backup");
assert_eq!(primary, backup, "backup CAS must use the reconciled current revision");
assert_eq!(primary_revision, backup_revision);
let persisted = DataUsageCache::unmarshal(&primary.expect("new primary body")).expect("decode new primary");
assert_eq!(persisted.info.next_cycle, cycle);
assert_eq!(persisted.find("photos").expect("persisted new entry").size, 20);
}
#[tokio::test]
#[serial]
async fn set_snapshot_backup_catchup_keeps_revision_provenance_and_epoch_fences() {
use crate::data_usage_define::read_config_with_revision;
for case in ["different_backup", "different_primary", "corrupt_primary", "changed_epoch"] {
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
let set = Arc::clone(&store.pools[0].disk_set[0]);
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
let backup_path = format!("{primary_path}.bkp");
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read original primary");
let primary_body = primary_body.expect("original primary exists");
let mut older = cache.clone();
older.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
save_config(Arc::clone(&set), &backup_path, older.marshal_msg().expect("encode lagging backup"))
.await
.expect("seed lagging backup");
if case == "corrupt_primary" {
save_config(Arc::clone(&set), &primary_path, b"corrupt-primary".to_vec())
.await
.expect("inject primary corruption before the initial load");
}
let mut loaded = DataUsageCache::default();
let initial = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture original baseline source and revisions");
let mut replacement = cache.clone();
replacement.info.scan_execution_digest = Some(DataUsageScanPlanDigest([9; 32]));
match case {
"different_backup" => {
save_config(Arc::clone(&set), &backup_path, replacement.marshal_msg().expect("encode changed backup"))
.await
.expect("backup changes to different contents");
}
"different_primary" => {
let body = replacement.marshal_msg().expect("encode changed primary");
save_config(Arc::clone(&set), &primary_path, body.clone())
.await
.expect("primary advances");
save_config(Arc::clone(&set), &backup_path, body)
.await
.expect("backup follows changed primary");
}
"corrupt_primary" => {
save_config(Arc::clone(&set), &backup_path, b"corrupt-primary".to_vec())
.await
.expect("backup matches the corrupt primary ETag without a decoded primary proof");
}
"changed_epoch" => {
save_config(Arc::clone(&set), &backup_path, primary_body)
.await
.expect("backup catches up before epoch refusal");
}
_ => unreachable!("fixed test cases"),
}
let before_primary = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("current primary");
let before_backup = read_config_with_revision(Arc::clone(&set), &backup_path)
.await
.expect("current backup");
let mut candidate = cache;
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
let cycle_floor = AtomicU64::new(candidate.info.next_cycle);
let requested_epoch = if case == "changed_epoch" {
epoch.checked_add(1).expect("fixture epoch can advance")
} else {
epoch
};
let (tx, mut rx) = mpsc::channel(1);
let (result, events) = capture_set_cache_logs(persist_and_publish_cache_snapshot(
Arc::clone(&set),
&tx,
candidate,
Some(&initial),
&cycle_floor,
requested_epoch,
))
.await;
assert!(result.is_none(), "{case} must still refuse publication");
assert!(
matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"{case} forwards no snapshot"
);
let expected_state = if case == "changed_epoch" {
"save_failed"
} else {
"scan_baseline_revision_changed"
};
assert_set_cache_context(set_cache_event(&events, expected_state), Some((0, 0)));
assert_eq!(
read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("protected primary"),
before_primary,
"{case} does not overwrite primary"
);
assert_eq!(
read_config_with_revision(set, &backup_path).await.expect("protected backup"),
before_backup,
"{case} does not overwrite backup"
);
}
}
#[tokio::test]
#[serial]
async fn set_cache_cas_preserves_updates_after_backup_catchup_validation() {
use crate::data_usage_define::read_config_with_revision;
for changed_copy in ["primary", "backup"] {
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
let set = Arc::clone(&store.pools[0].disk_set[0]);
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
let backup_path = format!("{primary_path}.bkp");
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read initial primary");
let mut lagging = cache.clone();
lagging.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
save_config(Arc::clone(&set), &backup_path, lagging.marshal_msg().expect("lagging backup bytes"))
.await
.expect("seed backup lag");
let mut loaded = DataUsageCache::default();
let initial = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("read initial baseline");
save_config(Arc::clone(&set), &backup_path, primary_body.expect("initial primary exists"))
.await
.expect("finish backup catchup");
let current = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("read reconciled baseline");
assert!(current.permits_unchanged_baseline_since(&initial));
let changed_path = if changed_copy == "primary" {
&primary_path
} else {
&backup_path
};
let mut concurrent = cache.clone();
concurrent.info.scan_execution_digest = Some(DataUsageScanPlanDigest([9; 32]));
let concurrent_body = concurrent.marshal_msg().expect("encode concurrent update");
save_config(Arc::clone(&set), changed_path, concurrent_body.clone())
.await
.expect("copy changes after the baseline validation");
let mut candidate = cache;
candidate.info.next_cycle += 1;
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
let result = candidate
.save_with_revisions_for_epoch(Arc::clone(&set), DATA_USAGE_CACHE_NAME, current.revisions(), epoch)
.await;
if changed_copy == "primary" {
assert!(result.is_err(), "primary CAS refuses the intervening update");
} else {
result.expect("primary remains durable when the best-effort backup CAS refuses an update");
}
let (protected, _) = read_config_with_revision(Arc::clone(&set), changed_path)
.await
.expect("read protected concurrent copy");
assert_eq!(
protected,
Some(concurrent_body),
"{changed_copy} CAS must not overwrite the concurrent update"
);
let (primary, _) = read_config_with_revision(set, &primary_path)
.await
.expect("read resulting primary");
let primary = DataUsageCache::unmarshal(&primary.expect("primary exists")).expect("primary remains decodable");
assert_eq!(
primary.info.scan_execution_digest,
Some(DataUsageScanPlanDigest([if changed_copy == "primary" { 9 } else { 3 }; 32])),
);
}
}
#[tokio::test]
#[serial]
async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers() {
@@ -2400,7 +2652,7 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers
.expect("seed legacy set cache");
let mut persisted = DataUsageCache::default();
let initial = persisted
.load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture the shared starting revision");
let mut fresh = legacy.clone();
@@ -2426,7 +2678,7 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers
assert_eq!(published.find("photos").expect("published bucket").size, 20);
assert_eq!(published.info.scan_execution_digest, fresh.info.scan_execution_digest);
let current = persisted
.load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture the current revision for the unidentified execution");
@@ -2717,6 +2969,130 @@ fn scoped_scan_requires_a_complete_authoritative_baseline_with_exact_set_provena
);
}
#[test]
fn scoped_scan_baseline_rejections_preserve_the_existing_first_failed_check() {
let source = DataUsageCacheSource::new(1, 2);
let expected_sources = HashSet::from([source]);
let digest = DataUsageScanPlanDigest([9; 32]);
let baseline = complete_usage_baseline(source, digest, 7, 11);
let snapshot: DataUsageInfo = serde_json::from_slice(&baseline).expect("decode complete baseline");
let proof = ScannerCacheBaselineProof {
authoritative_data: Some(&baseline),
observed_candidate_data: None,
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest: digest,
};
type RejectionCase = (ScannerBaselineRejection, fn(&mut DataUsageInfo));
let cases: [RejectionCase; 13] = [
(ScannerBaselineRejection::Incomplete, |s| s.usage_snapshot_complete = false),
(ScannerBaselineRejection::Partial, |s| s.usage_snapshot_partial = true),
(ScannerBaselineRejection::Marks, |s| s.usage_snapshot_converged = None),
(ScannerBaselineRejection::Epoch, |s| s.scanner_epoch = Some(10)),
(ScannerBaselineRejection::SourceCount, |s| s.usage_snapshot_set_states.clear()),
(ScannerBaselineRejection::PlanMissing, |s| {
s.usage_snapshot_set_states[0].scan_plan_digest = None
}),
(ScannerBaselineRejection::PlanMismatch, |s| {
s.usage_snapshot_set_states[0].scan_plan_digest = Some([8; 32])
}),
(ScannerBaselineRejection::SourceMembership, |s| {
s.usage_snapshot_set_states[0].set_index = 3
}),
(ScannerBaselineRejection::SetIncomplete, |s| {
s.usage_snapshot_set_states[0].complete = false
}),
(ScannerBaselineRejection::SetTombstone, |s| {
s.usage_snapshot_set_states[0].tombstone = true
}),
(ScannerBaselineRejection::SetEpoch, |s| {
s.usage_snapshot_set_states[0].scanner_epoch = None
}),
(ScannerBaselineRejection::SetCycle, |s| {
s.usage_snapshot_set_states[0].scanner_cycle = Some(9)
}),
(ScannerBaselineRejection::SetCycle, |s| {
s.usage_snapshot_set_states[0].scanner_cycle = None
}),
];
assert_eq!(validate_scanner_cache_snapshot_plan_digest(&snapshot, proof, None), Ok(digest));
for (expected, mutate) in cases {
let mut invalid = snapshot.clone();
mutate(&mut invalid);
assert_eq!(validate_scanner_cache_snapshot_plan_digest(&invalid, proof, None), Err(expected));
assert_eq!(complete_scanner_cache_snapshot_plan_digest(&invalid, proof, None), None);
}
let mut invalid = snapshot.clone();
invalid.usage_snapshot_partial = true;
invalid.scanner_epoch = Some(10);
assert_eq!(
validate_scanner_cache_snapshot_plan_digest(&invalid, proof, None),
Err(ScannerBaselineRejection::Partial)
);
let two_sources = HashSet::from([source, DataUsageCacheSource::new(1, 3)]);
let mut two_sets = snapshot.clone();
two_sets
.usage_snapshot_set_states
.push(two_sets.usage_snapshot_set_states[0].clone());
let two_set_proof = ScannerCacheBaselineProof {
expected_sources: &two_sources,
..proof
};
assert_eq!(
validate_scanner_cache_snapshot_plan_digest(&two_sets, two_set_proof, None),
Err(ScannerBaselineRejection::SourceDuplicate)
);
two_sets.usage_snapshot_set_states[1].set_index = 3;
two_sets.usage_snapshot_set_states[1].scan_plan_digest = Some([8; 32]);
assert_eq!(
validate_scanner_cache_snapshot_plan_digest(&two_sets, two_set_proof, None),
Err(ScannerBaselineRejection::SetPlan)
);
let undecodable = Bytes::from_static(b"not-json");
for (data, expected) in [
(None, ScannerBaselineRejection::Missing),
(Some(&undecodable), ScannerBaselineRejection::Decode),
] {
assert_eq!(
validate_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
authoritative_data: data,
..proof
}),
Err(ScannerBaselineValidationFailure {
authoritative: expected,
observed: None
})
);
}
let mut wrong_epoch = snapshot;
wrong_epoch.scanner_epoch = Some(10);
let wrong_epoch = Bytes::from(serde_json::to_vec(&wrong_epoch).expect("encode stale epoch baseline"));
let stale_proof = ScannerCacheBaselineProof {
authoritative_data: Some(&wrong_epoch),
..proof
};
assert_eq!(
validate_scanner_cache_baseline_plan_digest(stale_proof),
Err(ScannerBaselineValidationFailure {
authoritative: ScannerBaselineRejection::Epoch,
observed: Some(ScannerBaselineRejection::Missing),
})
);
assert_eq!(
validate_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
observed_candidate_data: Some(&undecodable),
..stale_proof
}),
Err(ScannerBaselineValidationFailure {
authoritative: ScannerBaselineRejection::Epoch,
observed: Some(ScannerBaselineRejection::Decode)
})
);
}
#[test]
fn scoped_scan_accepts_only_a_complete_observation_tied_to_the_authoritative_baseline() {
let source = DataUsageCacheSource::new(1, 2);
@@ -3150,6 +3526,12 @@ fn distributed_segment_reuse_activation_keeps_remote_dirty_buckets_at_bucket_sco
result.scope.prefix_scope_for("photos").is_some(),
"durable local producer proof may activate local segment reuse after distributed ACK capability evidence"
);
assert!(
result
.segment_reuse_activation_preflight
.expect("actual scope proof is retained for observation")
.scanner_segment_reuse_activated
);
assert!(
result.scope.prefix_scope_for("archive").is_none(),
"remote dirty usage has only bucket-granularity evidence and must not be narrowed to a local prefix"
@@ -3568,6 +3950,10 @@ fn scoped_set_scan_rebuilds_selected_buckets_and_drops_deleted_buckets() {
.checked_flatten("stable")
.expect("selected bucket placeholder should exist");
assert_eq!((stable.size, stable.objects), (0, 0));
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
prepared.record_cold_bucket_reuse(&observation);
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 0);
assert!(prepared.cache.find("stable/prefix").is_none());
assert_eq!(prepared.cache.find("dirty").map(|entry| (entry.size, entry.objects)), Some((0, 0)));
assert!(prepared.cache.find("deleted").is_none());
@@ -3579,6 +3965,124 @@ fn scoped_set_scan_rebuilds_selected_buckets_and_drops_deleted_buckets() {
assert_eq!(prepared.cache.info.lkg_scan_plan_digest, Some(baseline_digest));
}
#[test]
fn cold_bucket_reuse_observation_rejects_unbound_or_failed_preparation() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
let mut old_cache = complete_set_usage_cache(&[("stable", 10), ("dirty", 20)], baseline_digest);
old_cache.info.scan_bucket_incarnations = test_bucket_incarnations(&["stable", "dirty"]);
let incarnations = old_cache.info.scan_bucket_incarnations.clone();
let buckets = vec![
bucket_info_with_created_time("stable"),
bucket_info_with_created_time("dirty"),
];
let scope = ScannerBucketScanScope {
selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
selected_bucket_prefixes: None,
baseline_scan_plan_digest: Some(baseline_digest),
};
let generation = ScannerSetCacheGeneration {
want_cycle: 8,
leader_epoch: 11,
tier_registry_generation: 13,
source: DataUsageCacheSource::new(1, 2),
scan_plan_digest: DataUsageScanPlanDigest([2; 32]),
};
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
let prepared = prepare_scoped_set_scan(&old_cache, &buckets, &buckets, &scope, generation, Some(&incarnations))
.expect("bound complete cache permits a cold bucket copy");
prepared.record_cold_bucket_reuse(&observation);
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 1);
for case in ["incomplete", "duplicate", "nil_incarnation"] {
let mut invalid_cache = old_cache.clone();
let mut invalid_buckets = buckets.clone();
let mut invalid_incarnations = incarnations.clone();
match case {
"incomplete" => invalid_cache.info.snapshot_complete = false,
"duplicate" => invalid_buckets.push(buckets[0].clone()),
"nil_incarnation" => {
invalid_incarnations.insert("stable".to_string(), Uuid::nil());
}
_ => unreachable!("fixed test cases"),
}
let prepared = prepare_scoped_set_scan(
&invalid_cache,
&invalid_buckets,
&invalid_buckets,
&scope,
generation,
Some(&invalid_incarnations),
);
assert!(prepared.is_none(), "{case} must not return a reusable prepared cache");
if let Some(prepared) = prepared {
prepared.record_cold_bucket_reuse(&observation);
}
assert_eq!(
observation.snapshot().bucket_reuse["scoped_cold_bucket"].count,
1,
"{case} does not count completed reuse"
);
}
}
#[tokio::test]
async fn current_root_reuse_counts_only_successful_collector_delivery() {
use rustfs_scanner_metrics::scan_observation::ScannerScanObservation;
let observation = ScannerScanObservation::default();
let cache = DataUsageCache::default();
let pending = AtomicBool::new(false);
let root = || DataUsageEntryInfo {
name: "bucket".to_string(),
parent: DATA_USAGE_ROOT.to_string(),
entry: DataUsageEntry {
size: 10,
objects: 2,
..Default::default()
},
bucket_incarnation: Some(Uuid::from_u128(7)),
tier_registry_generation: None,
value_origin: None,
};
let (sender, mut receiver) = mpsc::channel(1);
super::io_cache::send_current_cache_root_entry(&sender, root(), &cache, &pending, &observation)
.await
.expect("current root is delivered to the collector");
let delivered = receiver.recv().await.expect("collector received the root");
assert_eq!((delivered.entry.size, delivered.entry.objects), (10, 2));
assert_eq!(observation.snapshot().bucket_reuse["current_root"].count, 1);
drop(receiver);
assert!(
super::io_cache::send_current_cache_root_entry(&sender, root(), &cache, &pending, &observation)
.await
.is_err()
);
assert_eq!(
observation.snapshot().bucket_reuse["current_root"].count,
1,
"closed collector is not completed reuse"
);
let (sender, _receiver) = mpsc::channel(1);
sender.send(root()).await.expect("fill bounded collector channel");
{
let mut send = Box::pin(super::io_cache::send_current_cache_root_entry(
&sender,
root(),
&cache,
&pending,
&observation,
));
assert!(futures::poll!(send.as_mut()).is_pending(), "full collector keeps reuse unfinished");
}
assert_eq!(
observation.snapshot().bucket_reuse["current_root"].count,
1,
"dropping unfinished delivery is not reuse"
);
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 0);
}
#[test]
fn scoped_set_scan_reuses_unselected_buckets_with_matching_incarnations() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
@@ -3620,6 +4124,12 @@ fn scoped_set_scan_reuses_unselected_buckets_with_matching_incarnations() {
)
.expect("matching bucket incarnations should authorize cold bucket reuse");
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
prepared.record_cold_bucket_reuse(&observation);
let observed = observation.snapshot();
assert_eq!(observed.bucket_reuse["scoped_cold_bucket"].count, 1);
assert_eq!(observed.bucket_reuse["current_root"].count, 0);
assert!(observed.cache_reuse.values().all(|reuse| reuse.count == 0));
assert_eq!(prepared.buckets.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(), ["dirty"]);
let stable = prepared
.cache
@@ -4625,12 +5135,36 @@ async fn get_size_treats_missing_metadata_as_skip_file() {
debug: false,
};
let before = global_metrics().scan_observation().snapshot().metadata_reads;
let err = disk
.get_size(item)
.get_size(item.clone())
.await
.expect_err("missing metadata should be skipped instead of reported as a scanner failure");
assert!(matches!(err, StorageError::Io(ref io) if io.to_string() == SCANNER_SKIP_FILE_ERROR));
let missing = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(missing.attempts, before.attempts + 1);
assert_eq!(missing.not_found, before.not_found + 1);
assert_eq!(missing.bytes, before.bytes);
#[cfg(unix)]
{
tokio::fs::create_dir_all(&object_dir)
.await
.expect("recreate object directory");
std::os::unix::fs::symlink(STORAGE_FORMAT_FILE, &metadata_path).expect("inject a metadata symlink loop");
let err = disk
.get_size(item)
.await
.expect_err("symlink loop must remain a read failure");
assert!(is_scanner_metadata_transient_error(&err), "read error: {err:?}");
let failed = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(failed.attempts, missing.attempts + 1);
assert_eq!(failed.errors, missing.errors + 1);
assert_eq!(failed.succeeded, before.succeeded);
assert_eq!(failed.bytes, before.bytes);
}
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
@@ -4681,6 +5215,7 @@ async fn get_size_marks_corrupt_metadata_for_heal() {
debug: false,
};
let before = global_metrics().scan_observation().snapshot().metadata_reads;
for contents in [b"".as_slice(), b"not-valid-filemeta".as_slice()] {
tokio::fs::write(&metadata_path, contents)
.await
@@ -4692,6 +5227,16 @@ async fn get_size_marks_corrupt_metadata_for_heal() {
assert!(is_scanner_metadata_corrupt_error(&err));
}
let reads = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(reads.attempts, before.attempts + 2);
assert_eq!(
reads.succeeded,
before.succeeded + 2,
"storage reads succeed before decoding rejects corruption"
);
assert_eq!(reads.errors, before.errors);
assert_eq!(reads.bytes, before.bytes + 18);
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
@@ -4764,11 +5309,17 @@ async fn get_size_counts_delete_markers_separately_from_versions() {
debug: false,
};
let before = global_metrics().scan_observation().snapshot().metadata_reads;
let expected_bytes = tokio::fs::metadata(&metadata_path).await.expect("stat stored metadata").len();
let summary = disk.get_size(item).await.expect("scanner should read versioned metadata");
assert_eq!(summary.versions, 2);
assert_eq!(summary.delete_markers, 1);
assert_eq!(summary.total_size, 30);
let reads = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(reads.attempts, before.attempts + 1);
assert_eq!(reads.succeeded, before.succeeded + 1);
assert_eq!(reads.bytes, before.bytes + expected_bytes);
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
@@ -183,9 +183,11 @@ async fn dirty_entry_removed_during_listing_still_forces_a_full_scan() {
);
assert_eq!(second.buckets_usage["hot-bucket"].objects_count, 2);
assert_eq!(second.buckets_usage["cold-bucket"].objects_count, 2);
let freshness = global_metrics().report().await.usage_freshness;
let report = global_metrics().report().await;
let freshness = report.usage_freshness;
assert_eq!(freshness.scope_fallback_total, fallbacks_before + 1);
assert_eq!(freshness.last_scope_fallback_reason, "dirty_snapshot_incomplete");
assert_eq!(report.scan_observation.scope_fallbacks["dirty_snapshot_incomplete"], 1);
clear_dirty_usage_buckets_for_tests();
}
@@ -199,11 +201,53 @@ async fn required_full_scan_is_not_reported_as_a_scope_fallback() {
wait_for_namespace_commit_tails(store.as_ref()).await;
super::io_cycle::set_bucket_listing_window_hook_for_tests(|| record_dirty_usage_bucket("created-after-listing"));
let before = global_metrics().scan_observation().snapshot();
let (result, _, scope) = run_cycle(&store, 2, Some(baseline), true).await;
assert!(scope.is_default());
assert_eq!(result.pending_dirty_usage_buckets(), 1);
assert_eq!(result.scope_fallback(), None, "scheduled maintenance walks are not degradations");
assert_eq!(result.completed_scope_fallback(), None);
let after = global_metrics().scan_observation().snapshot();
assert_eq!(after.scope_fallbacks, before.scope_fallbacks);
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated);
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated + 1);
clear_dirty_usage_buckets_for_tests();
}
#[tokio::test]
#[serial]
async fn baseline_unavailable_cycle_records_real_scope_and_baseline_rejections_once() {
let (_temp_dir, store) = setup_local_scanner_store(1, 1).await;
let _baseline = converged_baseline(&store).await;
put_fixture_object(&store, "hot-bucket", "second").await;
record_dirty_usage_bucket("hot-bucket");
wait_for_namespace_commit_tails(store.as_ref()).await;
let before = global_metrics().scan_observation().snapshot();
let (result, snapshot, scope) = run_cycle(&store, 2, Some(Bytes::from_static(b"not-json")), false).await;
assert!(scope.is_default());
assert_eq!(result.status, ScannerCycleStatus::Complete);
assert_eq!(result.scope_fallback(), Some(ScannerScopeFallbackReason::BaselineUnavailable));
assert_eq!(snapshot.buckets_usage["hot-bucket"].objects_count, 2);
let after = global_metrics().scan_observation().snapshot();
assert_eq!(
after.scope_fallbacks["baseline_unavailable"],
before.scope_fallbacks["baseline_unavailable"] + 1
);
assert_eq!(
after.authoritative_baseline_rejections["decode"],
before.authoritative_baseline_rejections["decode"] + 1
);
assert_eq!(
after.observed_baseline_rejections, before.observed_baseline_rejections,
"an undecodable authority never admits an observation"
);
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated + 1);
assert_eq!(after.prefix_scope_resolution.activated, before.prefix_scope_resolution.activated);
assert_eq!(
after.prefix_scope_resolution.blockers["missing_cold_zero_walk_oracle"],
before.prefix_scope_resolution.blockers["missing_cold_zero_walk_oracle"] + 1
);
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated + 1);
clear_dirty_usage_buckets_for_tests();
}
@@ -1,12 +1,12 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! A bucket whose walk stops early without exhausting the cycle budget (here
//! a sweep resumed under a newer plan, the `mixed_coverage` case of
//! rustfs/rustfs#8494) no longer makes the whole cycle `Incomplete`. The set
//! publishes the bucket's last certified value, the cycle completes with the
//! bucket pending, and the next cycle walks it again. A bucket that never
//! certified keeps the cycle incomplete.
//! A bucket whose walk stops early can publish its last certified value as
//! pending. This fixture tests usage-scan status, not the outer scanner cycle
//! outcome: budget exhaustion may leave complete validated usage while the
//! coordinator records `Partial`. A continuation under a newer plan exercises
//! `mixed_coverage` (rustfs/rustfs#8494). A bucket that never certified keeps
//! the usage scan incomplete.
use super::*;
@@ -38,6 +38,8 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
..Default::default()
},
);
// Real checkpoint commits need the runtime drain grace after the count budget stops the walk.
budget.use_production_drain_grace_for_tests();
let (updates, mut receiver) = mpsc::channel(1);
let result = tokio::time::timeout(
Duration::from_secs(60),
@@ -46,7 +48,7 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
ScannerCycleRequest {
owner_ctx: ctx.clone(),
ctx: budget.token(),
budget,
budget: Arc::clone(&budget),
updates,
want_cycle: cycle,
leader_epoch: LEADER_EPOCH,
@@ -66,6 +68,13 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
.await
.expect("cycle should finish within the test deadline")
.expect("cycle should succeed");
if options.max_objects.is_some() {
assert_eq!(
budget.reason(),
Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects),
"a count-limited fixture must stop for its object budget"
);
}
let snapshot = receiver.recv().await;
assert!(receiver.recv().await.is_none(), "a cycle publishes at most one terminal snapshot");
(result, snapshot)
@@ -122,14 +131,14 @@ async fn partial_bucket_with_a_certified_value_completes_the_cycle_as_pending()
assert!(result.clear_verified_usage().is_empty());
assert!(dirty_usage_buckets_for_tests().is_empty());
// Cycle 3: the object budget stops the hot bucket's walk and leaves a
// checkpoint under this cycle's plan; the cycle itself is incomplete.
// Cycle 3: the budget stops the walk and saves a checkpoint. Usage can
// carry the certified value as pending; the outer coordinator records Partial.
for object in ["p4/third", "p5/third"] {
put_fixture_object(&store, "hot-bucket", object).await;
}
record_dirty_usage_bucket("hot-bucket");
wait_for_namespace_commit_tails(store.as_ref()).await;
let (result, _) = run_cycle(
let (result, third) = run_cycle(
&store,
3,
CycleOptions {
@@ -140,7 +149,38 @@ async fn partial_bucket_with_a_certified_value_completes_the_cycle_as_pending()
},
)
.await;
assert_eq!(result.status, ScannerCycleStatus::Incomplete, "an exhausted budget stays incomplete");
assert_eq!(
result.status,
ScannerCycleStatus::Complete,
"certified carried usage remains complete after a budget stop"
);
let third = third.expect("validated carried usage publishes after the budget stop");
assert_eq!(result.pending_dirty_usage_buckets(), 1);
assert!(!third.usage_snapshot_observational);
assert_eq!(third.usage_snapshot_converged, Some(false));
assert_eq!(third.usage_snapshot_pending_buckets, vec!["hot-bucket".to_string()]);
assert_eq!(third.usage_snapshot_pending_bucket_count, 1);
assert!(third.is_complete_bucket_usage_snapshot());
let carried_hot = &third.buckets_usage["hot-bucket"];
let certified_hot = &second.buckets_usage["hot-bucket"];
assert_eq!(carried_hot.usage_value_state, Some(rustfs_data_usage::BucketUsageValueState::Carried));
assert!(certified_hot.usage_certification.is_some(), "cycle 2 certified the prior hot value");
assert_eq!(
carried_hot.usage_certification, certified_hot.usage_certification,
"carrying must not mint a new certificate"
);
assert_eq!(carried_hot.objects_count, 4);
assert_eq!(carried_hot.size, certified_hot.size);
assert_eq!(third.buckets_usage["cold-bucket"].objects_count, 1);
assert_eq!(third.buckets_usage["idle-bucket"].objects_count, 1);
assert_eq!(third.objects_total_count, 6);
// Only hot was dirty in this scoped scan; it received no fresh certification or ACK.
assert!(result.clear_verified_usage().is_empty());
assert_eq!(
dirty_usage_buckets_for_tests().keys().cloned().collect::<Vec<_>>(),
vec!["hot-bucket".to_string()],
"the budget-stopped hot bucket retains its dirty responsibility"
);
let set = Arc::clone(&store.pools[0].disk_set[0]);
let mut interrupted = DataUsageCache::default();
interrupted
@@ -154,7 +194,7 @@ async fn partial_bucket_with_a_certified_value_completes_the_cycle_as_pending()
// is mixed coverage and returns Partial without exhausting any budget.
// The set stands in with the last certified value and the cycle
// completes with the bucket pending instead of being incomplete.
let (result, fourth) = run_cycle(&store, 4, complete_cycle(true, Some(&second))).await;
let (result, fourth) = run_cycle(&store, 4, complete_cycle(true, Some(&third))).await;
let fourth = fourth.expect("a complete cycle publishes");
assert_eq!(
result.status,
@@ -898,6 +898,42 @@ done
The `aggregated.scanner` payload preserves the same scanner progress, checkpoint, pacing, source work, maintenance control, lifecycle expiry, and lifecycle transition fields used by the local scanner status, but only for the responding node; `by_host.*.scanner` keeps that node's host view. Compare the per-node artifacts externally to find old active paths, partial checkpoints, pacing pressure, source-level control pressure, or downstream queue admission problems across the deployment.
## Reading Cached Bucket Statistics
Admin usage reads cache the selected scanner baseline for the existing 30-second TTL, then verify every returned bucket against uncached storage-owned incarnation metadata. Identity reads use at most four concurrent operations and a shared two-second deadline after releasing snapshot/refresh locks. Each bucket read linearizes independently against metadata transactions; the response is not an atomic snapshot of the entire namespace. Delayed peer invalidation cannot authorize a predecessor's statistics for a recreated bucket.
Missing, failed, timed-out, nil, mismatched, or legacy absent identities leave the bucket value absent (unknown), rather than returning the predecessor's value or a confirmed zero. Legacy statistics become visible after a scanner snapshot supplies the current identity. Account-info and usage display apply the existing single-process memory overlay before the final identity filter; distributed local counters are not promoted. When a projection omits values or changes an overlay, totals are partial and `usage_snapshot_converged=false`, and per-tier/replication aggregates are unavailable. Verified cold bucket values remain visible. Such response-only projections cannot be persisted, used for quota admission, or acknowledge dirty work.
## Reading Scanner Work Observations
`GET /rustfs/admin/v3/scanner/status` exposes `metrics.scan_observation` for the responding node. Capture timestamped snapshots on every node, including the scanner leader, and compare their deltas over the same workload window. These process-lifetime counters reset on restart; a missing field on an older node means unobserved. They are loaded independently and may briefly disagree while operations finish. This telemetry does not change scan cadence, cached usage reads, quota authority, dirty acknowledgement, or maintenance requirements.
| Field | Meaning |
|---|---|
| `metadata_reads.attempts` | Disk metadata read futures started after scanner node admission. Invalid paths and operations cancelled before starting the read do not count. |
| `metadata_reads.succeeded`, `not_found`, `errors`, `cancelled` | Storage outcomes, including object/version disappearance and dropped read futures. Successful storage reads can still contain corrupt metadata; decoding and lifecycle failures do not become read errors. Cancellation observes the future, not proof that physical I/O stopped. |
| `metadata_reads.bytes` | Total returned buffer lengths for successful reads, including inline data and corrupt metadata. Failed reads contribute no known bytes; this is not physical disk traffic. |
| `metadata_reads.elapsed_nanos` | Sum of read-future elapsed time, including disk-permit or RPC waits, failures, and cancellation. Folder pacing, scanner node admission, decoding, and lifecycle work are outside this interval. Concurrent reads overlap, so the sum is not cycle wall time. |
| `cache_reuse.clean_prefix` | Clean root subtrees copied under an admitted prefix scope by the non-erasure scanner. |
| `cache_reuse.covered_namespace` | Namespace subtrees copied from validated checkpoint coverage. |
| `cache_reuse.sampled_namespace` | Cached namespace subtrees copied when the existing periodic sampling selected other work. |
| `cache_reuse.classified_namespace` | EC candidate namespace subtrees copied only after the current listing finished classifying them. Health/classification work still runs. |
| `cache_reuse.*.count`, `elapsed_nanos` | Completed subtree reuse decisions and time spent copying/attaching their cache entries. Counts are not objects skipped or a cache hit ratio; times exclude classification/listing and are not total cache maintenance cost. |
| `bucket_reuse.scoped_cold_bucket` | Whole cold buckets copied and attached to a successfully prepared scoped set cache. Count is copied buckets, not plans selected; later turn publication can still fail. Time covers copying/attachment only. |
| `bucket_reuse.current_root` | Existing current bucket roots successfully sent to the local collector after watch, namespace lease and publication epoch checks. Time covers channel send only; failure or cancellation contributes no completed reuse. Collector acceptance and durable publication remain separate. |
| `reuse_decisions.*.not_evaluated`, `blocked`, `eligible` | First conditions reached by production clean-prefix, checkpoint-coverage and namespace-sampling gates. Policy exclusions (`missing_scope`, `no_resume_sweep`, `deep_scan`, `forward_sweep`, `active_replication`) precede cache proof checks. `blocked` reports the first failed work/cache condition; later checks are not repeated for diagnostics. `eligible` does not certify a copy: an EC candidate can still require classification. |
| `erasure_clean_prefix_walks` | Clean root children that still require walking because EC scans perform independent object-health work. |
| `prefix_scope_resolution.evaluated`, `activated`, `blockers` | Proof evaluations retained by the final scope resolver, admitted proof gates, and every failed prerequisite. An admitted proof does not guarantee a reusable child exists; actual subtree copies appear in `cache_reuse`. Startup/required full scans, early scope rejection, and discarded peer/ACK resolutions do not count here. |
| `prefix_cycle_completion.evaluated`, `activated`, `blockers` | End-of-cycle proof evaluations for subsequent reuse. An activated completion proof does not mean this cycle reused prefixes or that its later publication/persistence succeeded. One evaluation can increment several blocker counters. |
| `scope_fallbacks` | Per-reason breakdown of the existing `usage_freshness.scope_fallback_total`; required full scans are excluded. |
| `authoritative_baseline_rejections`, `observed_baseline_rejections` | First failed validation checks for `baseline_unavailable` fallback cycles, preserving the authoritative rejection when the observed fallback also fails. Missing/undecodable authority never reaches observed validation; `authoritative_identity` means it could not anchor an observed fallback. |
`freshness` describes the responding node's completed-cycle age, not whether every bucket has incorporated new writes. Read `metrics.usage_freshness` for node-local live pending work, guardrail vetoes, durable success and publication state; read `metrics.usage_publication` for the confirmed snapshot's pending count and publication class/outcome, and `cycle_schedule` for effective retry intervals and consecutive superseded cycles. Missing or unknown publication data does not establish zero pending work. A follower or a single node's empty dirty map cannot establish cluster-wide convergence.
Whole cold bucket copies, current-root reuse and the four subtree categories have different prerequisites and are not interchangeable hit rates. EC clean prefixes retain health walks. Deep or required full work must run, but a completed cache matching that exact work digest may satisfy a retry of the same cycle. A normal workload need not exercise current-root reuse or every subtree category; use controlled tests for positive emission evidence and capture real gate reasons when a VM window has zero copies.
The existing `life_time_ops.read_metadata` and `last_minute.actions.read_metadata` now include finished and cancelled read attempts; the latter reports count, accumulated whole seconds, and returned bytes in its existing window. Use `metadata_reads.elapsed_nanos` for subsecond read costs. Use read-count and byte deltas to assess work growth, and elapsed-time deltas to assess read-future cost. Compare those with reused-subtree deltas, scope fallback reasons, cycle wall time, pacing waits, and source work before attributing freshness delays to metadata I/O or proposing a scan optimization.
## Reading Lifecycle Transition Status
`metrics.lifecycle_transition`:
+2 -3
View File
@@ -19,7 +19,7 @@ use crate::admin::service::caller_identity::oidc_profile_fields;
use crate::admin::storage_api::bucket::versioning_sys::BucketVersioningSys;
use crate::admin::storage_api::contract::admin::StorageAdminApi;
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
use crate::admin::storage_api::data_usage::{apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached};
use crate::admin::storage_api::data_usage::load_admin_data_usage_with_memory_overlay;
use crate::admin::storage_api::metadata_sys;
use crate::auth::get_condition_values;
use crate::server::ADMIN_PREFIX;
@@ -275,8 +275,7 @@ impl Operation for AccountInfoHandler {
// process-local absolute counters to cluster-wide bucket totals.
// This path never triggers a live full-version listing
// (rustfs/backlog#1306); freshness is owned by the scanner.
let mut data_usage_info = map_data_usage_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
apply_bucket_usage_memory_overlay(&mut data_usage_info).await;
let data_usage_info = map_data_usage_result(load_admin_data_usage_with_memory_overlay(store.clone()).await)?;
account_info.usage_snapshot_converged = data_usage_info.usage_snapshot_converged;
for bucket in buckets.iter() {
+6 -4
View File
@@ -939,10 +939,6 @@ pub(crate) static ERR_TIER_RESERVED_NAME: AdminErrorRef = AdminErrorRef(|| &ecst
pub(crate) mod data_usage {
use std::sync::Arc;
pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) {
crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
}
pub(crate) async fn load_data_usage_from_backend(
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
@@ -954,6 +950,12 @@ pub(crate) mod data_usage {
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_from_backend_cached(store).await
}
pub(crate) async fn load_admin_data_usage_with_memory_overlay(
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_with_memory_overlay(store).await
}
}
pub(crate) mod access {
+122 -15
View File
@@ -21,9 +21,7 @@ use super::storage_api::admin_usecase::capacity::{
};
use super::storage_api::admin_usecase::contract::StorageAdminApi;
use super::storage_api::admin_usecase::contract::bucket::{BucketOperations as _, BucketOptions};
use super::storage_api::admin_usecase::data_usage::{
apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached,
};
use super::storage_api::admin_usecase::data_usage::load_admin_data_usage_with_memory_overlay;
use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
use crate::app::runtime_sources::{
AppContext, current_app_context, current_endpoints_handle, current_object_store_handle_for_context,
@@ -258,7 +256,9 @@ impl DefaultAdminUsecase {
/// namespace still contains, carries no usable information and is dropped
/// exactly as before.
fn narrow_data_usage_snapshot_to_measured_buckets(info: &mut DataUsageInfo, buckets: impl IntoIterator<Item = String>) {
if !info.is_complete_bucket_usage_snapshot() && !info.is_valid_partial_snapshot() {
let snapshot_measured =
info.usage_snapshot_admin_projection || info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot();
if !snapshot_measured {
*info = DataUsageInfo::default();
return;
}
@@ -268,20 +268,30 @@ impl DefaultAdminUsecase {
info.buckets_usage.retain(|bucket, _| current_buckets.contains(bucket));
info.bucket_sizes.retain(|bucket, _| current_buckets.contains(bucket));
if info.buckets_usage.is_empty() && !current_buckets.is_empty() {
if info.buckets_usage.is_empty() && !current_buckets.is_empty() && !info.usage_snapshot_admin_projection {
*info = DataUsageInfo::default();
return;
}
let covers_namespace = current_buckets.len() == info.buckets_usage.len();
info.buckets_count = info.buckets_usage.len() as u64;
if !covers_namespace {
// Some bucket in the namespace was never measured, so the totals
// below are a floor rather than the converged truth.
if !covers_namespace || !info.is_complete_bucket_usage_snapshot() {
// Uncovered buckets or unreconciled mutations keep the displayed
// totals from representing converged usage.
info.usage_snapshot_converged = Some(false);
}
}
/// Keep already identity-bound display values without accepting a display
/// projection as a persistable or quota-authoritative snapshot.
fn present_identity_bound_data_usage_snapshot(
mut info: DataUsageInfo,
buckets: impl IntoIterator<Item = String>,
) -> DataUsageInfo {
Self::narrow_data_usage_snapshot_to_measured_buckets(&mut info, buckets);
info
}
async fn rebalance_status_snapshot(store: &ECStore) -> AdminUsecaseResult<Option<RebalanceMeta>> {
store.rebalance_status_meta_snapshot().await.map_err(|err| {
error!("refresh rebalance metadata for pool status failed: {:?}", err);
@@ -323,8 +333,7 @@ impl DefaultAdminUsecase {
/// This path never triggers a live full-version listing
/// (rustfs/backlog#1306); freshness is owned by the scanner.
pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
let mut info = Self::map_data_usage_load_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
apply_bucket_usage_memory_overlay(&mut info).await;
let info = Self::map_data_usage_load_result(load_admin_data_usage_with_memory_overlay(store.clone()).await)?;
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
@@ -333,11 +342,7 @@ impl DefaultAdminUsecase {
})
.await
.map_err(|_| Self::app_error(S3ErrorCode::InternalError, "list_bucket failed"))?;
// Serve what the scan measured; buckets it never reached stay absent
// (absent means unknown on the wire, unlike a present zero) and the
// snapshot is flagged non-converged. See rustfs#5806 for why blanking
// the whole document instead was worse than the rule it enforced.
Self::narrow_data_usage_snapshot_to_measured_buckets(&mut info, buckets.into_iter().map(|bucket| bucket.name));
let mut info = Self::present_identity_bound_data_usage_snapshot(info, buckets.into_iter().map(|bucket| bucket.name));
let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
@@ -962,6 +967,108 @@ mod tests {
assert_eq!(info.buckets_count, 0);
}
#[test]
fn identity_bound_partial_display_keeps_valid_buckets_and_unknown_state() {
use std::collections::HashMap;
let mut projected = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
usage_snapshot_admin_projection: true,
usage_snapshot_partial: true,
usage_snapshot_converged: Some(false),
buckets_count: 1,
objects_total_count: 7,
objects_total_size: 70,
buckets_usage: HashMap::from([(
"cold".to_string(),
rustfs_data_usage::BucketUsageInfo {
size: 70,
objects_count: 7,
..Default::default()
},
)]),
bucket_sizes: HashMap::from([("cold".to_string(), 70)]),
..Default::default()
};
projected = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(
projected,
["cold".to_string(), "recreated".to_string()],
);
assert_eq!(projected.buckets_usage["cold"].objects_count, 7);
assert!(!projected.buckets_usage.contains_key("recreated"));
assert_eq!(projected.objects_total_count, 7);
assert_eq!(projected.usage_snapshot_converged, Some(false));
assert!(!projected.is_complete_bucket_usage_snapshot());
assert!(!projected.is_valid_partial_snapshot());
projected.buckets_usage.clear();
projected.bucket_sizes.clear();
projected.calculate_totals();
projected.buckets_count = 0;
let unknown = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(projected, ["recreated".to_string()]);
assert!(unknown.buckets_usage.is_empty());
assert!(unknown.usage_snapshot_admin_projection);
assert_eq!(unknown.usage_snapshot_converged, Some(false));
assert!(!unknown.is_complete_bucket_usage_snapshot());
assert_eq!(unknown.last_update, Some(std::time::SystemTime::UNIX_EPOCH));
}
/// A write into a bucket without a verified baseline leaves process-local
/// counts the overlay cannot reconcile, so it clears
/// `usage_snapshot_complete`. That must flag the response non-converged,
/// not erase every bucket the persisted scan measured: under continuous
/// writes the scanner fence never clears and the blackout never ends.
#[tokio::test]
async fn unverified_bucket_writes_do_not_blank_measured_usage() {
let measured_bucket = "admin-usage-measured";
let unverified_bucket = "admin-usage-unverified";
let mut persisted = DataUsageInfo {
last_update: Some(std::time::SystemTime::now() - std::time::Duration::from_secs(60)),
usage_snapshot_complete: true,
buckets_count: 1,
..Default::default()
};
persisted.buckets_usage.insert(
measured_bucket.to_string(),
rustfs_data_usage::BucketUsageInfo {
objects_count: 4_000,
versions_count: 4_000,
size: 8_192_000,
..Default::default()
},
);
persisted.bucket_sizes.insert(measured_bucket.to_string(), 8_192_000);
persisted.calculate_totals();
assert!(persisted.is_complete_bucket_usage_snapshot());
crate::app::storage_api::test::data_usage::record_bucket_object_write_memory(unverified_bucket, None, 1_024).await;
// Exercise the production overlay, followed by the same identity-bound
// presentation function used by the admin query.
crate::app::storage_api::test::data_usage::apply_bucket_usage_memory_overlay(&mut persisted).await;
persisted.usage_snapshot_admin_projection = true;
persisted.usage_snapshot_converged = Some(false);
let info = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(
persisted,
[measured_bucket.to_string(), unverified_bucket.to_string()],
);
assert_eq!(
info.buckets_usage
.get(measured_bucket)
.map(|usage| (usage.objects_count, usage.size)),
Some((4_000, 8_192_000)),
"usage the scanner measured must survive an unrelated unverified bucket"
);
assert!(
!info.buckets_usage.contains_key(unverified_bucket),
"a bucket without a verified baseline stays absent (unknown)"
);
assert_eq!(info.usage_snapshot_converged, Some(false));
assert!(!info.is_complete_bucket_usage_snapshot());
assert_eq!(info.objects_total_count, 4_000);
}
#[tokio::test]
async fn execute_query_storage_info_returns_internal_error_when_store_uninitialized() {
let usecase = DefaultAdminUsecase::without_context();
@@ -104,6 +104,17 @@ async fn persist_scanner_snapshot(ecstore: Arc<ECStore>, info: DataUsageInfo) {
rustfs_scanner::scanner::store_data_usage_in_backend(CancellationToken::new(), ecstore, receiver).await;
}
async fn bind_current_bucket_incarnations(ecstore: &ECStore, info: &mut DataUsageInfo) {
for (bucket, usage) in &mut info.buckets_usage {
let incarnation = ecstore
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("load the current incarnation for the covered snapshot");
assert!(!incarnation.is_nil(), "covered snapshot must bind a non-nil bucket identity");
usage.bucket_incarnation = Some(incarnation.to_string());
}
}
#[tokio::test]
#[serial]
async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
@@ -170,6 +181,7 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
seeded_info.bucket_sizes.insert(overlay_bucket.clone(), 0);
seeded_info.buckets_count = u64::try_from(seeded_info.buckets_usage.len()).expect("test bucket count should fit into u64");
seeded_info.calculate_totals();
bind_current_bucket_incarnations(&ecstore, &mut seeded_info).await;
persist_scanner_snapshot(ecstore.clone(), seeded_info).await;
record_bucket_object_write_memory(&overlay_bucket, None, 512).await;
@@ -337,6 +349,7 @@ async fn data_usage_endpoint_treats_legacy_partial_stats_as_unknown_until_covere
completed.bucket_sizes.insert(small_bucket.clone(), 57_344);
completed.buckets_count = u64::try_from(completed.buckets_usage.len()).expect("test bucket count should fit into u64");
completed.calculate_totals();
bind_current_bucket_incarnations(&ecstore, &mut completed).await;
store_data_usage_in_backend(completed, ecstore.clone())
.await
.expect("persist complete covered snapshot");
+3 -2
View File
@@ -104,14 +104,15 @@ pub(crate) mod data_usage {
pub(crate) use crate::storage::storage_api::ecstore_data_usage::quota_object_size;
#[cfg(test)]
pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) {
crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
}
pub(crate) async fn load_admin_data_usage_from_backend_cached(
pub(crate) async fn load_admin_data_usage_with_memory_overlay(
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_from_backend_cached(store).await
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_with_memory_overlay(store).await
}
pub(crate) async fn load_bucket_usage_from_backend_cached(
+4 -4
View File
@@ -502,8 +502,8 @@ pub(crate) mod ecstore_config {
pub(crate) mod ecstore_data_usage {
pub(crate) use rustfs_ecstore::api::data_usage::{
apply_bucket_usage_memory_overlay, get_bucket_usage_memory, init_compression_total_memory_from_backend,
load_admin_data_usage_from_backend_cached, load_bucket_usage_from_backend_cached, load_data_usage_from_backend,
get_bucket_usage_memory, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached,
load_admin_data_usage_with_memory_overlay, load_bucket_usage_from_backend_cached, load_data_usage_from_backend,
lookup_degraded_bucket_usage_baseline, quota_object_size, record_bucket_delete_marker_memory,
record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend,
@@ -511,8 +511,8 @@ pub(crate) mod ecstore_data_usage {
// Test-only observables for the rustfs/backlog#1306 revert detector.
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::data_usage::{
compute_bucket_usage, live_bucket_usage_computations, load_data_usage_from_backend_cached,
seed_bucket_usage_memory_for_test, store_data_usage_in_backend,
apply_bucket_usage_memory_overlay, compute_bucket_usage, live_bucket_usage_computations,
load_data_usage_from_backend_cached, seed_bucket_usage_memory_for_test, store_data_usage_in_backend,
};
}