gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 00:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): require authoritative usage snapshots (#5333)

Browse Source
This commit is contained in:
Henry Guo
2026-07-28 07:29:57 +08:00
committed by GitHub
parent 5a768d3a44
commit d5df66ac4f
25 changed files with 1829 additions and 374 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/data-usage/src/data_usage.rs
+54 -5
View File
@@ -32,12 +32,12 @@ use std::{
/// save forever and freeze admin usage stats; callers must bypass the skip instead.
pub const USAGE_LAST_UPDATE_FUTURE_TOLERANCE: Duration = Duration::from_secs(5 * 60);
/// Authoritative cluster-wide usage snapshot written by coordinated scanners.
/// Cluster-wide usage snapshot written by coordinated scanners.
///
/// This object name is intentionally distinct from the legacy unfenced
/// snapshot. Older binaries can continue writing the legacy object during a
/// rolling upgrade without overwriting a snapshot produced by the current
/// scanner protocol.
/// `usage_snapshot_complete` is an additive JSON field: older readers ignore
/// it, while current readers treat snapshots from older writers as unknown.
/// Keeping the existing object name preserves rolling-upgrade and rollback
/// compatibility without allowing an ambiguous snapshot to become authoritative.
pub const DATA_USAGE_OBJECT_NAME: &str = ".usage.v2.json";
/// Usage snapshot written by scanner implementations predating distributed
@@ -190,6 +190,12 @@ pub struct DataUsageInfo {
pub buckets_count: u64,
/// Buckets usage info provides following information across all buckets
pub buckets_usage: HashMap<String, BucketUsageInfo>,
/// Whether this snapshot covers the complete bucket namespace.
///
/// Legacy snapshots default to `false`. A complete snapshot contains an
/// explicit entry for every bucket, including confirmed-empty buckets.
#[serde(default)]
pub usage_snapshot_complete: bool,
/// Deprecated kept here for backward compatibility reasons
pub bucket_sizes: HashMap<String, u64>,
/// Per-disk snapshot information when available
@@ -681,6 +687,12 @@ pub struct DataUsageCacheInfo {
pub skip_healing: bool,
#[serde(default)]
pub failed_objects: HashMap<String, u64>,
/// Whether this per-set cache was produced by a completed scanner pass.
///
/// Older cache writers omit this field and therefore deserialize as
/// incomplete instead of exposing partial set totals as confirmed zeros.
#[serde(default)]
pub snapshot_complete: bool,
}
/// Data usage cache
@@ -1000,6 +1012,7 @@ impl DataUsageCache {
objects_total_size: flat.size as u64,
buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX),
buckets_usage,
usage_snapshot_complete: self.info.snapshot_complete,
..Default::default()
}
}
@@ -1078,6 +1091,13 @@ impl DataUsageInfo {
Self::default()
}
/// Whether this snapshot authoritatively covers every reported bucket.
pub fn is_complete_bucket_usage_snapshot(&self) -> bool {
self.usage_snapshot_complete
&& self.last_update.is_some()
&& u64::try_from(self.buckets_usage.len()).ok() == Some(self.buckets_count)
}
/// Add object metadata to data usage statistics
pub fn add_object(&mut self, object_path: &str, meta_object: &rustfs_filemeta::MetaObject) {
// This method is kept for backward compatibility
@@ -1442,6 +1462,35 @@ pub struct CompressionTotalInfo {
mod tests {
use super::*;
#[derive(Deserialize)]
struct LegacyUsageReader {
buckets_count: u64,
}
#[test]
fn completeness_marker_is_additive_for_legacy_named_readers() {
let current = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH),
usage_snapshot_complete: true,
..Default::default()
};
let encoded = rmp_serde::to_vec_named(&current).expect("encode current data usage snapshot");
let legacy: LegacyUsageReader = rmp_serde::from_slice(&encoded).expect("legacy reader should ignore additive fields");
assert_eq!(legacy.buckets_count, 0);
assert!(current.is_complete_bucket_usage_snapshot());
}
#[test]
fn completeness_marker_requires_a_snapshot_timestamp() {
let untimestamped = DataUsageInfo {
usage_snapshot_complete: true,
..Default::default()
};
assert!(!untimestamped.is_complete_bucket_usage_snapshot());
}
#[test]
fn test_usage_last_update_future_tolerance_boundary() {
let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
crates/ecstore/src/api/mod.rs
+4 -4
View File
@@ -287,10 +287,10 @@ pub mod config {
pub mod data_usage {
pub use crate::data_usage::{
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
init_compression_total_memory_from_backend, live_bucket_usage_computations, load_compression_total_from_memory,
load_data_usage_from_backend, load_data_usage_from_backend_cached, record_bucket_delete_marker_memory,
record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
init_compression_total_memory_from_backend, invalidate_data_usage_snapshot_cache, live_bucket_usage_computations,
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
store_data_usage_in_backend,
crates/ecstore/src/bucket/quota/checker.rs
+28 -2
View File
@@ -167,7 +167,7 @@ impl QuotaChecker {
}
let quota = self.get_quota_config(bucket).await?;
let current_usage = self.get_real_time_usage(bucket).await.unwrap_or(0);
let current_usage = self.get_real_time_usage(bucket).await?;
Ok((quota, Some(current_usage)))
}
@@ -177,7 +177,11 @@ impl QuotaChecker {
}
pub async fn get_real_time_usage(&self, bucket: &str) -> Result<u64, QuotaError> {
Ok(get_bucket_usage_memory(bucket).await.unwrap_or(0))
get_bucket_usage_memory(bucket)
.await
.ok_or_else(|| QuotaError::UsageUnavailable {
bucket: bucket.to_string(),
})
}
}
@@ -185,6 +189,8 @@ impl QuotaChecker {
mod tests {
use super::*;
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
use serial_test::serial;
use uuid::Uuid;
/// Regression (PR #5307 / s3-tests `test_100_continue_error_retry`): a
/// bucket with no persisted metadata has no quota, so the admission check
@@ -205,6 +211,26 @@ mod tests {
assert_eq!(result.quota_limit, None);
}
#[tokio::test]
#[serial]
async fn quota_usage_rejects_an_unknown_mutation_baseline() {
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
let checker = QuotaChecker::new(sys);
let bucket = format!("quota-unknown-{}", Uuid::new_v4().simple());
crate::data_usage::record_bucket_object_write_memory(&bucket, None, 42).await;
let result = checker.get_real_time_usage(&bucket).await;
crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
.await
.expect("test usage cache cleanup should succeed");
assert!(
matches!(result, Err(QuotaError::UsageUnavailable { bucket: failed_bucket }) if failed_bucket == bucket),
"quota decisions must fail closed without an authoritative usage baseline"
);
}
#[tokio::test]
async fn test_quota_check_no_limit() {
let result = QuotaCheckResult {
crates/ecstore/src/bucket/quota/mod.rs
+3 -1
View File
@@ -110,6 +110,8 @@ pub enum QuotaError {
QuotaExceeded { current: u64, limit: u64, operation: u64 },
#[error("Quota configuration not found for bucket: {bucket}")]
ConfigNotFound { bucket: String },
#[error("Authoritative data usage is unavailable for bucket: {bucket}")]
UsageUnavailable { bucket: String },
#[error("Invalid quota configuration: {reason}")]
InvalidConfig { reason: String },
#[error("Storage error: {0}")]
@@ -155,7 +157,7 @@ impl QuotaErrorResponse {
request_id: request_id.to_string(),
host_id: host_id.to_string(),
},
QuotaError::StorageError(_) => Self {
QuotaError::UsageUnavailable { .. } | QuotaError::StorageError(_) => Self {
code: QUOTA_INTERNAL_ERROR_CODE.to_string(),
message: quota_error.to_string(),
resource: QUOTA_API_PATH.to_string(),
crates/ecstore/src/data_usage/mod.rs
+550 -80
View File
@@ -48,7 +48,7 @@ use std::{
time::{Duration, SystemTime},
};
use tokio::fs;
use tokio::sync::RwLock;
use tokio::sync::{Mutex as TokioMutex, RwLock};
use tracing::{debug, error, info, instrument};
// Data usage storage constants
@@ -63,12 +63,17 @@ const DATA_USAGE_REMOVE_CAS_RETRIES: usize = 3;
#[derive(Debug, Clone)]
struct CachedBucketUsage {
usage: BucketUsageInfo,
authoritative: bool,
refreshed_at: SystemTime,
usage_updated_at: SystemTime,
// Set by request-path mutations until a scanner snapshot catches up to the same core counts.
dirty: bool,
// Set when a newer scanner snapshot was observed but did not include the dirty counts yet.
stale_snapshot_pending: bool,
// First complete scanner generation observed after an unknown-baseline
// mutation. A strictly later generation is required before the mutation
// evidence can be discarded.
pending_scanner_position: Option<(u64, u64)>,
}
type UsageMemoryCache = Arc<RwLock<HashMap<String, CachedBucketUsage>>>;
@@ -78,6 +83,7 @@ type LiveBucketUsageCache = moka::future::Cache<String, BucketUsageInfo>;
static USAGE_MEMORY_CACHE: OnceLock<UsageMemoryCache> = OnceLock::new();
static USAGE_CACHE_UPDATING: OnceLock<CacheUpdating> = OnceLock::new();
static LIVE_BUCKET_USAGE_CACHE: OnceLock<LiveBucketUsageCache> = OnceLock::new();
static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
/// Cached copy of the last persisted data usage snapshot, served to admin
/// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
@@ -110,8 +116,13 @@ fn cache_data_usage_snapshot_result(
cache: &mut Option<CachedDataUsageSnapshot>,
result: Result<DataUsageInfo, Error>,
loaded_at: tokio::time::Instant,
) -> Result<DataUsageInfo, Error> {
match result {
refresh_generation: u64,
) -> Option<Result<DataUsageInfo, Error>> {
if data_usage_snapshot_generation() != refresh_generation {
return None;
}
Some(match result {
Ok(info) => {
*cache = Some(CachedDataUsageSnapshot {
info: Some(info.clone()),
@@ -123,12 +134,14 @@ fn cache_data_usage_snapshot_result(
*cache = Some(CachedDataUsageSnapshot { info: None, loaded_at });
Err(e)
}
}
})
}
type DataUsageSnapshotCache = Arc<RwLock<Option<CachedDataUsageSnapshot>>>;
static DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
static DATA_USAGE_SNAPSHOT_REFRESH: OnceLock<Arc<TokioMutex<()>>> = OnceLock::new();
static DATA_USAGE_SNAPSHOT_GENERATION: AtomicU64 = AtomicU64::new(0);
// Always-on revert detector for rustfs/backlog#1306: one relaxed increment per
// full-bucket version listing is negligible and lets tests prove that admin
@@ -183,6 +196,15 @@ fn data_usage_snapshot_cache() -> &'static DataUsageSnapshotCache {
DATA_USAGE_SNAPSHOT_CACHE.get_or_init(|| Arc::new(RwLock::new(None)))
}
fn data_usage_snapshot_generation() -> u64 {
DATA_USAGE_SNAPSHOT_GENERATION.load(Ordering::Acquire)
}
fn clear_data_usage_snapshot_cache(cache: &mut Option<CachedDataUsageSnapshot>) {
DATA_USAGE_SNAPSHOT_GENERATION.fetch_add(1, Ordering::AcqRel);
*cache = None;
}
fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
LIVE_BUCKET_USAGE_CACHE.get_or_init(|| {
moka::future::Cache::builder()
@@ -191,6 +213,10 @@ fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
})
}
fn usage_memory_generation() -> u64 {
USAGE_MEMORY_GENERATION.load(Ordering::Acquire)
}
// Data usage storage paths
lazy_static::lazy_static! {
pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}",
@@ -245,9 +271,21 @@ fn stale_data_usage_persist_reason(incoming: &DataUsageInfo, existing: &DataUsag
enum UsageSnapshotSource {
Primary,
Backup,
LegacyPrimary,
LegacyBackup,
Missing,
}
impl UsageSnapshotSource {
fn is_authoritative(self) -> bool {
matches!(self, Self::Primary | Self::Backup)
}
fn is_backup(self) -> bool {
matches!(self, Self::Backup | Self::LegacyBackup)
}
}
fn stale_data_usage_persist_reason_for_source(
incoming: &DataUsageInfo,
existing: &DataUsageInfo,
@@ -255,7 +293,7 @@ fn stale_data_usage_persist_reason_for_source(
now: SystemTime,
) -> Option<&'static str> {
let reason = stale_data_usage_persist_reason(incoming, existing, now);
if source == UsageSnapshotSource::Backup && incoming.last_update == existing.last_update {
if source.is_backup() && incoming.last_update == existing.last_update {
None
} else {
reason
@@ -267,6 +305,7 @@ fn stale_data_usage_persist_reason_for_source(
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
// Prevent older data from overwriting newer persisted stats
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
&& source.is_authoritative()
&& let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
{
info!(
@@ -292,7 +331,7 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
// next request instead of waiting out the remaining TTL. The next cached
// read reloads through `load_data_usage_from_backend`, keeping its
// backward-compatibility post-processing.
*data_usage_snapshot_cache().write().await = None;
invalidate_data_usage_snapshot_cache().await;
Ok(())
}
@@ -343,12 +382,13 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error> {
ensure_bucket_namespace_guard(guard, bucket, "data usage cache cleanup")?;
let _ = USAGE_MEMORY_GENERATION.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)));
live_bucket_usage_cache().invalidate(bucket).await;
clear_bucket_usage_memory(bucket, guard).await?;
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache cleanup")?;
*snapshot_cache = None;
clear_data_usage_snapshot_cache(&mut snapshot_cache);
Ok(())
}
@@ -363,7 +403,7 @@ where
let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
*snapshot_cache = None;
clear_data_usage_snapshot_cache(&mut snapshot_cache);
result
}
@@ -386,7 +426,9 @@ where
.filter(|etag| !etag.is_empty())
.map(str::to_owned)
.ok_or_else(|| Error::other("data usage snapshot has no ETag"))?;
let data_usage_info = normalize_loaded_data_usage(parse_usage_snapshot(&reader.read_all().await?)?).await;
let mut data_usage_info = parse_usage_snapshot(&reader.read_all().await?)?;
populate_backward_compatible_usage_maps(&mut data_usage_info);
validate_complete_usage_snapshot(&mut data_usage_info);
Ok(Some((data_usage_info, revision)))
}
@@ -468,12 +510,15 @@ async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<DataUs
where
S: EcstoreObjectIO + ?Sized,
{
for object in [
DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
for (object, authoritative) in [
(DATA_USAGE_OBJ_BACKUP_PATH.as_str(), true),
(LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), false),
(LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(), false),
] {
if let Some((data_usage_info, _)) = load_data_usage_for_bucket_removal(store, object).await? {
if let Some((mut data_usage_info, _)) = load_data_usage_for_bucket_removal(store, object).await? {
if !authoritative {
data_usage_info.usage_snapshot_complete = false;
}
return Ok(data_usage_info);
}
}
@@ -692,13 +737,21 @@ async fn load_data_usage_snapshot_from_store<S: EcstoreObjectIO>(
}
let legacy_primary = read_config_preserve_empty(store.clone(), &LEGACY_DATA_USAGE_OBJ_NAME_PATH).await;
resolve_loaded_snapshot_pair_with_source(
let legacy = resolve_loaded_snapshot_pair_with_source(
legacy_primary,
async move { read_config_preserve_empty(store, &LEGACY_DATA_USAGE_OBJ_BACKUP_PATH).await },
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
)
.await
.await?;
Ok((
legacy.0,
match legacy.1 {
UsageSnapshotSource::Primary => UsageSnapshotSource::LegacyPrimary,
UsageSnapshotSource::Backup => UsageSnapshotSource::LegacyBackup,
source => source,
},
))
}
async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo, UsageSnapshotSource), Error> {
@@ -708,13 +761,27 @@ async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo,
/// Load data usage info from backend storage
#[instrument(skip(store))]
pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
Ok(normalize_loaded_data_usage(load_data_usage_snapshot(store).await?.0).await)
let (data_usage_info, source) = load_data_usage_snapshot(store).await?;
Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await)
}
async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo) -> DataUsageInfo {
info!("Loaded data usage info from backend with {} buckets", data_usage_info.buckets_count);
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
if !data_usage_info.is_complete_bucket_usage_snapshot() {
data_usage_info.usage_snapshot_complete = false;
data_usage_info.buckets_usage.clear();
data_usage_info.bucket_sizes.clear();
data_usage_info.buckets_count = 0;
data_usage_info.calculate_totals();
}
}
// Handle backward compatibility
fn validate_complete_usage_snapshot(data_usage_info: &mut DataUsageInfo) {
if data_usage_info.usage_snapshot_complete && !data_usage_info.is_complete_bucket_usage_snapshot() {
data_usage_info.usage_snapshot_complete = false;
}
}
fn populate_backward_compatible_usage_maps(data_usage_info: &mut DataUsageInfo) {
if data_usage_info.buckets_usage.is_empty() {
data_usage_info.buckets_usage = data_usage_info
.bucket_sizes
@@ -738,6 +805,17 @@ async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo) -> Data
.map(|(bucket, bui)| (bucket.clone(), bui.size))
.collect();
}
}
async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo, authoritative_format: bool) -> DataUsageInfo {
info!("Loaded data usage info from backend with {} buckets", data_usage_info.buckets_count);
if !authoritative_format {
data_usage_info.usage_snapshot_complete = false;
}
populate_backward_compatible_usage_maps(&mut data_usage_info);
validate_complete_usage_snapshot(&mut data_usage_info);
discard_incomplete_bucket_usage(&mut data_usage_info);
// Handle replication info
for (bucket, bui) in &data_usage_info.buckets_usage {
@@ -774,22 +852,43 @@ async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo) -> Data
pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
{
let cache = data_usage_snapshot_cache().read().await;
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
loop {
{
let cache = data_usage_snapshot_cache().read().await;
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
return result;
}
}
// Serialize refreshes without holding the cache lock across backend I/O.
let refresh_guard = DATA_USAGE_SNAPSHOT_REFRESH
.get_or_init(|| Arc::new(TokioMutex::new(())))
.lock()
.await;
{
let cache = data_usage_snapshot_cache().read().await;
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
return result;
}
}
let refresh_generation = data_usage_snapshot_generation();
let result = load_data_usage_from_backend(store.clone()).await;
let loaded_at = tokio::time::Instant::now();
let mut cache = data_usage_snapshot_cache().write().await;
if let Some(result) = cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation) {
return result;
}
drop(cache);
drop(refresh_guard);
}
}
// Re-check under the write lock so concurrent expirations trigger a single
// backend read instead of a stampede.
/// Invalidate the process-local persisted usage snapshot cache after a durable
/// scanner write.
pub async fn invalidate_data_usage_snapshot_cache() {
let mut cache = data_usage_snapshot_cache().write().await;
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
return result;
}
let result = load_data_usage_from_backend(store).await;
cache_data_usage_snapshot_result(&mut cache, result, tokio::time::Instant::now())
clear_data_usage_snapshot_cache(&mut cache);
}
/// Aggregate usage information from local disk snapshots.
@@ -1114,8 +1213,12 @@ where
}
fn apply_live_bucket_usage_to_response(data_usage_info: &mut DataUsageInfo, bucket: &str, usage: &BucketUsageInfo) {
let inserted_bucket = !data_usage_info.buckets_usage.contains_key(bucket);
data_usage_info.bucket_sizes.insert(bucket.to_string(), usage.size);
data_usage_info.buckets_usage.insert(bucket.to_string(), usage.clone());
if inserted_bucket {
data_usage_info.usage_snapshot_complete = false;
}
set_buckets_count_from_usage(data_usage_info);
data_usage_info.calculate_totals();
}
@@ -1180,13 +1283,15 @@ async fn ensure_bucket_usage_cached(bucket: &str) {
update_usage_cache_if_needed().await;
}
fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTime) -> CachedBucketUsage {
fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTime, authoritative: bool) -> CachedBucketUsage {
CachedBucketUsage {
usage,
authoritative,
refreshed_at: SystemTime::now(),
usage_updated_at: updated_at,
dirty: false,
stale_snapshot_pending: false,
pending_scanner_position: None,
}
}
@@ -1194,10 +1299,12 @@ fn cached_bucket_usage_now(usage: BucketUsageInfo) -> CachedBucketUsage {
let now = SystemTime::now();
CachedBucketUsage {
usage,
authoritative: false,
refreshed_at: now,
usage_updated_at: now,
dirty: false,
stale_snapshot_pending: false,
pending_scanner_position: None,
}
}
@@ -1212,6 +1319,30 @@ fn bucket_usage_counts_match(left: &BucketUsageInfo, right: &BucketUsageInfo) ->
&& left.delete_markers_count == right.delete_markers_count
}
fn preserve_unknown_dirty_usage(
existing: &CachedBucketUsage,
snapshot_position: Option<(u64, u64)>,
snapshot_update: SystemTime,
) -> Option<CachedBucketUsage> {
if existing.authoritative || !existing.dirty {
return None;
}
if existing
.pending_scanner_position
.zip(snapshot_position)
.is_some_and(|(previous, current)| current.0 > previous.0 || (current.0 == previous.0 && current.1 > previous.1))
{
return None;
}
let mut preserved = existing.clone();
preserved.stale_snapshot_pending = true;
if preserved.pending_scanner_position.is_none() && snapshot_update >= existing.usage_updated_at {
preserved.pending_scanner_position = snapshot_position;
}
Some(preserved)
}
#[cfg(test)]
async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: BucketUsageInfo, refresh_started_at: SystemTime) {
let mut cache = memory_cache().write().await;
@@ -1221,7 +1352,7 @@ async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: Buc
return;
}
cache.insert(bucket.to_string(), cached_bucket_usage_from_backend(usage, refresh_started_at));
cache.insert(bucket.to_string(), cached_bucket_usage_from_backend(usage, refresh_started_at, true));
}
/// Fast in-memory update for immediate quota and admin usage consistency.
@@ -1271,6 +1402,7 @@ async fn record_bucket_object_write_memory_inner(
entry.usage_updated_at = now;
entry.dirty = true;
entry.stale_snapshot_pending = false;
entry.pending_scanner_position = None;
}
/// Degraded in-memory update for an object write whose previous current size
@@ -1299,6 +1431,7 @@ pub async fn record_bucket_object_write_unknown_previous_memory(bucket: &str, ne
entry.usage_updated_at = now;
entry.dirty = true;
entry.stale_snapshot_pending = false;
entry.pending_scanner_position = None;
}
/// Fast in-memory increment for immediate quota consistency.
@@ -1326,6 +1459,7 @@ pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64,
entry.usage_updated_at = now;
entry.dirty = true;
entry.stale_snapshot_pending = false;
entry.pending_scanner_position = None;
}
/// Fast in-memory update for successful delete marker creation.
@@ -1344,6 +1478,7 @@ pub async fn record_bucket_delete_marker_memory(bucket: &str) {
entry.usage_updated_at = now;
entry.dirty = true;
entry.stale_snapshot_pending = false;
entry.pending_scanner_position = None;
}
/// Fast in-memory decrement for immediate quota consistency
@@ -1354,8 +1489,8 @@ pub async fn decrement_bucket_usage_memory(bucket: &str, size_decrement: u64) {
/// Get bucket usage from the authoritative cache for this topology.
async fn get_persisted_bucket_usage(bucket: &str) -> Option<u64> {
let store = runtime_sources::object_store_handle()?;
let data_usage_info = load_data_usage_from_backend_cached(store).await.ok()?;
data_usage_info.buckets_usage.get(bucket).map(|usage| usage.size)
let info = load_data_usage_from_backend_cached(store).await.ok()?;
info.buckets_usage.get(bucket).map(|usage| usage.size)
}
pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
@@ -1371,7 +1506,10 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
update_usage_cache_if_needed().await;
let cache = memory_cache().read().await;
cache.get(bucket).map(|cached| cached.usage.size)
cache
.get(bucket)
.filter(|cached| cached.authoritative)
.map(|cached| cached.usage.size)
}
async fn update_usage_cache_if_needed() {
@@ -1401,11 +1539,12 @@ async fn update_usage_cache_if_needed() {
drop(updating);
let updating_clone = (*cache_updating()).clone();
let refresh_generation = usage_memory_generation();
tokio::spawn(async move {
if let Some(store) = runtime_sources::object_store_handle()
&& let Ok(data_usage_info) = load_data_usage_from_backend(store).await
{
replace_bucket_usage_memory_from_info(&data_usage_info).await;
replace_bucket_usage_memory_from_info_if_generation(&data_usage_info, Some(refresh_generation)).await;
}
let mut updating = updating_clone.write().await;
*updating = false;
@@ -1426,10 +1565,11 @@ async fn update_usage_cache_if_needed() {
*updating = true;
drop(updating);
let refresh_generation = usage_memory_generation();
if let Some(store) = runtime_sources::object_store_handle()
&& let Ok(data_usage_info) = load_data_usage_from_backend(store).await
{
replace_bucket_usage_memory_from_info(&data_usage_info).await;
replace_bucket_usage_memory_from_info_if_generation(&data_usage_info, Some(refresh_generation)).await;
}
let mut updating = cache_updating().write().await;
@@ -1437,40 +1577,57 @@ async fn update_usage_cache_if_needed() {
}
pub async fn replace_bucket_usage_memory_from_info(data_usage_info: &DataUsageInfo) {
let usage_updated_at = data_usage_info_updated_at(data_usage_info);
let mut cache = memory_cache().write().await;
let mut next_cache = HashMap::new();
replace_bucket_usage_memory_from_info_if_generation(data_usage_info, None).await;
}
for (bucket, bucket_usage) in data_usage_info.buckets_usage.iter() {
if let Some(existing) = cache.get(bucket) {
if existing.usage_updated_at > usage_updated_at {
next_cache.insert(bucket.clone(), existing.clone());
continue;
}
if existing.dirty && !bucket_usage_counts_match(&existing.usage, bucket_usage) {
// A scanner snapshot can be saved after newer writes but still miss them if it listed the bucket earlier.
let mut preserved = existing.clone();
preserved.stale_snapshot_pending = true;
next_cache.insert(bucket.clone(), preserved);
continue;
}
}
next_cache.insert(bucket.clone(), cached_bucket_usage_from_backend(bucket_usage.clone(), usage_updated_at));
async fn replace_bucket_usage_memory_from_info_if_generation(data_usage_info: &DataUsageInfo, expected_generation: Option<u64>) {
if !data_usage_info.is_complete_bucket_usage_snapshot() {
return;
}
for (bucket, existing) in cache.iter() {
if !data_usage_info.buckets_usage.contains_key(bucket) {
if existing.usage_updated_at > usage_updated_at {
next_cache.insert(bucket.clone(), existing.clone());
continue;
}
let usage_updated_at = data_usage_info_updated_at(data_usage_info);
let snapshot_position = data_usage_info.scanner_epoch.zip(data_usage_info.scanner_cycle);
let mut next_cache = HashMap::with_capacity(data_usage_info.buckets_usage.len());
for (bucket, bucket_usage) in data_usage_info.buckets_usage.iter() {
next_cache.insert(
bucket.clone(),
cached_bucket_usage_from_backend(bucket_usage.clone(), usage_updated_at, true),
);
}
if existing.dirty {
let mut preserved = existing.clone();
preserved.stale_snapshot_pending = true;
next_cache.insert(bucket.clone(), preserved);
let mut cache = memory_cache().write().await;
if expected_generation.is_some_and(|expected| usage_memory_generation() != expected) {
return;
}
for (bucket, existing) in cache.iter() {
match next_cache.entry(bucket.clone()) {
Entry::Occupied(mut candidate) => {
if let Some(preserved) = preserve_unknown_dirty_usage(existing, snapshot_position, usage_updated_at) {
candidate.insert(preserved);
continue;
}
if existing.authoritative && existing.usage_updated_at > usage_updated_at {
candidate.insert(existing.clone());
continue;
}
if existing.authoritative && existing.dirty && !bucket_usage_counts_match(&existing.usage, &candidate.get().usage)
{
// A scanner snapshot can be saved after newer writes but still miss them if it listed the bucket earlier.
let mut preserved = existing.clone();
preserved.stale_snapshot_pending = true;
candidate.insert(preserved);
}
}
Entry::Vacant(candidate) => {
if let Some(preserved) = preserve_unknown_dirty_usage(existing, snapshot_position, usage_updated_at) {
candidate.insert(preserved);
} else if existing.authoritative && existing.usage_updated_at > usage_updated_at {
candidate.insert(existing.clone());
} else if existing.authoritative && existing.dirty {
let mut preserved = existing.clone();
preserved.stale_snapshot_pending = true;
candidate.insert(preserved);
}
}
}
}
@@ -1490,18 +1647,29 @@ async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mu
let persisted_update = data_usage_info.last_update;
let mut changed = false;
let mut added_bucket = false;
for (bucket, cached) in cache.iter() {
if !cached.authoritative {
if cached.dirty {
data_usage_info.usage_snapshot_complete = false;
}
continue;
}
if !cached.stale_snapshot_pending && persisted_update.is_some_and(|persisted| cached.usage_updated_at <= persisted) {
continue;
}
added_bucket |= !data_usage_info.buckets_usage.contains_key(bucket);
data_usage_info.buckets_usage.insert(bucket.clone(), cached.usage.clone());
data_usage_info.bucket_sizes.insert(bucket.clone(), cached.usage.size);
changed = true;
}
if changed {
if added_bucket {
data_usage_info.usage_snapshot_complete = false;
}
data_usage_info.buckets_count = data_usage_info.buckets_usage.len() as u64;
data_usage_info.calculate_totals();
}
@@ -2061,6 +2229,7 @@ mod tests {
..Default::default()
},
);
info.usage_snapshot_complete = true;
info.bucket_sizes.insert(bucket.to_string(), size);
info.buckets_count = info.buckets_usage.len() as u64;
info.calculate_totals();
@@ -2184,12 +2353,140 @@ mod tests {
assert!(!err.to_string().contains("secret-marker"));
}
#[tokio::test]
async fn legacy_snapshot_bucket_usage_is_unknown_without_completeness_marker() {
let mut legacy = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
legacy.usage_snapshot_complete = true;
legacy.buckets_usage.insert("large".to_string(), BucketUsageInfo::default());
legacy.bucket_sizes.insert("large".to_string(), 0);
legacy.buckets_count = 2;
let normalized = normalize_loaded_data_usage(legacy, false).await;
assert_eq!(normalized.buckets_count, 0);
assert!(normalized.buckets_usage.is_empty());
assert!(normalized.bucket_sizes.is_empty());
assert_eq!(normalized.objects_total_count, 0);
assert_eq!(normalized.objects_total_size, 0);
assert!(!normalized.usage_snapshot_complete);
}
#[test]
fn legacy_snapshot_json_defaults_completeness_to_unknown() {
let info = data_usage_info_for_test("legacy", 1, 42, SystemTime::UNIX_EPOCH);
let mut value = serde_json::to_value(info).expect("serialize data usage fixture");
let object = value.as_object_mut().expect("data usage fixture should be a JSON object");
object.remove("usage_snapshot_complete");
let decoded: DataUsageInfo = serde_json::from_value(value).expect("deserialize legacy data usage fixture");
assert!(!decoded.usage_snapshot_complete);
}
#[test]
fn complete_snapshot_json_is_additive_for_legacy_readers() {
#[derive(serde::Deserialize)]
struct LegacyUsageReader {
buckets_count: u64,
}
let info = data_usage_info_for_test("current", 1, 42, SystemTime::UNIX_EPOCH);
let encoded = serde_json::to_vec(&info).expect("serialize current data usage fixture");
let legacy: LegacyUsageReader =
serde_json::from_slice(&encoded).expect("legacy JSON reader should ignore additive fields");
assert_eq!(legacy.buckets_count, 1);
}
#[tokio::test]
async fn complete_snapshot_preserves_confirmed_empty_bucket() {
let mut info = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
info.buckets_count = 2;
let normalized = normalize_loaded_data_usage(info, true).await;
assert_eq!(normalized.buckets_count, 2);
assert!(normalized.usage_snapshot_complete);
assert_eq!(
normalized
.buckets_usage
.get("control")
.map(|usage| (usage.objects_count, usage.size)),
Some((10, 10_285))
);
assert_eq!(
normalized
.buckets_usage
.get("empty")
.map(|usage| (usage.objects_count, usage.size)),
Some((0, 0))
);
}
#[tokio::test]
async fn malformed_complete_snapshot_is_rejected_as_a_whole() {
let mut info = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
info.buckets_usage.insert(
"partial".to_string(),
BucketUsageInfo {
objects_count: 1_502,
versions_count: 1_502,
size: 196_870_144,
..Default::default()
},
);
info.bucket_sizes.insert("partial".to_string(), 196_870_144);
info.buckets_count = 1;
let normalized = normalize_loaded_data_usage(info, true).await;
assert_eq!(normalized.buckets_count, 0);
assert!(!normalized.buckets_usage.contains_key("control"));
assert!(!normalized.buckets_usage.contains_key("partial"));
assert!(!normalized.bucket_sizes.contains_key("partial"));
assert!(!normalized.usage_snapshot_complete);
}
#[tokio::test]
async fn completeness_marker_does_not_fabricate_a_missing_bucket() {
let mut info = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
info.buckets_count = 2;
assert!(!data_usage_contains_bucket(&info, "missing"));
let normalized = normalize_loaded_data_usage(info, true).await;
assert!(!normalized.usage_snapshot_complete);
assert!(normalized.buckets_usage.is_empty());
assert!(!normalized.buckets_usage.contains_key("missing"));
}
#[tokio::test]
async fn complete_empty_snapshot_remains_authoritative() {
let normalized = normalize_loaded_data_usage(
DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH),
usage_snapshot_complete: true,
..Default::default()
},
true,
)
.await;
assert!(normalized.usage_snapshot_complete);
assert_eq!(normalized.buckets_count, 0);
assert!(normalized.buckets_usage.is_empty());
}
#[test]
#[serial]
fn cached_snapshot_failure_is_reused_until_ttl_expires() {
let loaded_at = tokio::time::Instant::now();
let mut cache = None;
let refresh_generation = data_usage_snapshot_generation();
let first = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at);
let first = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at, refresh_generation)
.expect("an uninterrupted refresh should populate the cache");
assert!(matches!(first, Err(Error::ErasureReadQuorum)));
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
@@ -2200,13 +2497,16 @@ mod tests {
}
#[test]
#[serial]
fn cached_snapshot_success_is_reused_until_ttl_expires() {
let loaded_at = tokio::time::Instant::now();
let expected = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH);
let mut cache = None;
let refresh_generation = data_usage_snapshot_generation();
let first =
cache_data_usage_snapshot_result(&mut cache, Ok(expected), loaded_at).expect("successful load must be returned");
let first = cache_data_usage_snapshot_result(&mut cache, Ok(expected), loaded_at, refresh_generation)
.expect("an uninterrupted refresh should populate the cache")
.expect("successful load must be returned");
assert_snapshot_bucket(&first, "bucket");
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
@@ -2215,6 +2515,31 @@ mod tests {
assert_snapshot_bucket(&cached, "bucket");
}
#[test]
#[serial]
fn cache_invalidation_during_refresh_prevents_stale_snapshot_resurrection() {
let loaded_at = tokio::time::Instant::now();
let refresh_generation = data_usage_snapshot_generation();
let mut cache = Some(CachedDataUsageSnapshot {
info: Some(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
loaded_at,
});
clear_data_usage_snapshot_cache(&mut cache);
let stale_result = cache_data_usage_snapshot_result(
&mut cache,
Ok(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
loaded_at,
refresh_generation,
);
assert!(stale_result.is_none());
assert!(
cache.is_none(),
"an in-flight load must not repopulate a cache invalidated after it started"
);
}
#[tokio::test]
async fn resolve_snapshot_primary_ok_is_used_without_backup_read() {
let backup_read = Arc::new(std::sync::atomic::AtomicBool::new(false));
@@ -2312,7 +2637,7 @@ mod tests {
}
#[tokio::test]
async fn authoritative_usage_loader_migrates_only_when_v2_is_absent() {
async fn authoritative_usage_loader_prefers_v2_over_legacy_input() {
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
legacy_object: Some((snapshot_bytes("legacy-bucket"), 1)),
@@ -2320,18 +2645,20 @@ mod tests {
}),
});
let (legacy, _) = load_data_usage_snapshot_from_store(store.clone())
let (legacy, source) = load_data_usage_snapshot_from_store(store.clone())
.await
.expect("a legacy snapshot should seed an upgrade when v2 is absent");
.expect("a legacy snapshot should seed an upgrade when newer formats are absent");
assert_snapshot_bucket(&legacy, "legacy-bucket");
assert_eq!(source, UsageSnapshotSource::LegacyPrimary);
store.state.lock().await.object = Some((snapshot_bytes("v2-bucket"), 1));
let (authoritative, _) = load_data_usage_snapshot_from_store(store.clone())
store.state.lock().await.object = Some((snapshot_bytes("v2-bucket"), 2));
let (authoritative, source) = load_data_usage_snapshot_from_store(store.clone())
.await
.expect("a v2 snapshot should be authoritative");
.expect("a v2 snapshot should supersede the legacy migration input");
assert_snapshot_bucket(&authoritative, "v2-bucket");
assert_eq!(source, UsageSnapshotSource::Primary);
store.state.lock().await.object = Some((b"corrupt-v2".to_vec(), 2));
store.state.lock().await.object = Some((b"corrupt-v2".to_vec(), 3));
let err = load_data_usage_snapshot_from_store(store)
.await
.expect_err("corrupt v2 state must not fall back to a legacy writer");
@@ -2686,6 +3013,140 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn memory_overlay_does_not_publish_delta_from_unknown_baseline() {
clear_usage_memory_cache_for_test().await;
let bucket = "unknown-baseline";
memory_cache().write().await.insert(
bucket.to_string(),
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
);
record_bucket_object_write_memory(bucket, None, 42).await;
let mut response = DataUsageInfo::default();
apply_bucket_usage_memory_overlay(&mut response).await;
assert!(!response.buckets_usage.contains_key(bucket));
assert_eq!(get_bucket_usage_memory(bucket).await, None);
}
#[tokio::test]
#[serial]
async fn unknown_dirty_usage_requires_a_followup_scanner_generation() {
clear_usage_memory_cache_for_test().await;
let bucket = "unknown-baseline";
memory_cache().write().await.insert(
bucket.to_string(),
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
);
record_bucket_object_write_memory(bucket, None, 42).await;
let mutation_update = memory_cache()
.read()
.await
.get(bucket)
.expect("unknown usage mutation should remain cached")
.usage_updated_at;
let mut first_snapshot = data_usage_info_for_test(bucket, 10, 420, mutation_update + Duration::from_nanos(1));
first_snapshot.scanner_epoch = Some(7);
first_snapshot.scanner_cycle = Some(10);
replace_bucket_usage_memory_from_info(&first_snapshot).await;
let mut first_response = first_snapshot.clone();
apply_bucket_usage_memory_overlay_if_authoritative(&mut first_response, true).await;
assert!(
!first_response.usage_snapshot_complete,
"the first scanner generation after an unknown mutation cannot prove that it observed the mutation"
);
assert_eq!(
first_response
.buckets_usage
.get(bucket)
.map(|usage| (usage.objects_count, usage.size)),
Some((10, 420)),
"unknown memory deltas must not be overlaid as absolute usage"
);
replace_bucket_usage_memory_from_info(&first_snapshot).await;
let mut repeated_response = first_snapshot.clone();
apply_bucket_usage_memory_overlay_if_authoritative(&mut repeated_response, true).await;
assert!(
!repeated_response.usage_snapshot_complete,
"reloading the same scanner generation must not clear unknown mutation evidence"
);
let mut successor = data_usage_info_for_test(bucket, 11, 462, mutation_update + Duration::from_nanos(2));
successor.scanner_epoch = Some(7);
successor.scanner_cycle = Some(11);
replace_bucket_usage_memory_from_info(&successor).await;
let mut successor_response = successor;
apply_bucket_usage_memory_overlay_if_authoritative(&mut successor_response, true).await;
assert!(successor_response.is_complete_bucket_usage_snapshot());
assert_eq!(
successor_response
.buckets_usage
.get(bucket)
.map(|usage| (usage.objects_count, usage.size)),
Some((11, 462))
);
}
#[tokio::test]
#[serial]
async fn a_new_unknown_mutation_restarts_the_scanner_generation_fence() {
clear_usage_memory_cache_for_test().await;
let bucket = "unknown-baseline";
memory_cache().write().await.insert(
bucket.to_string(),
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
);
record_bucket_object_write_memory(bucket, None, 42).await;
let first_mutation_update = memory_cache()
.read()
.await
.get(bucket)
.expect("unknown usage mutation should remain cached")
.usage_updated_at;
let mut first_snapshot = data_usage_info_for_test(bucket, 10, 420, first_mutation_update + Duration::from_nanos(1));
first_snapshot.scanner_epoch = Some(7);
first_snapshot.scanner_cycle = Some(10);
replace_bucket_usage_memory_from_info(&first_snapshot).await;
record_bucket_object_write_memory(bucket, None, 42).await;
let second_mutation_update = memory_cache()
.read()
.await
.get(bucket)
.expect("second unknown usage mutation should remain cached")
.usage_updated_at;
let mut next_snapshot = data_usage_info_for_test(bucket, 11, 462, second_mutation_update + Duration::from_nanos(1));
next_snapshot.scanner_epoch = Some(7);
next_snapshot.scanner_cycle = Some(11);
replace_bucket_usage_memory_from_info(&next_snapshot).await;
let mut next_response = next_snapshot.clone();
apply_bucket_usage_memory_overlay_if_authoritative(&mut next_response, true).await;
assert!(
!next_response.usage_snapshot_complete,
"a mutation after the first observation must require another scanner generation"
);
let mut final_snapshot = data_usage_info_for_test(bucket, 12, 504, second_mutation_update + Duration::from_nanos(2));
final_snapshot.scanner_epoch = Some(7);
final_snapshot.scanner_cycle = Some(12);
replace_bucket_usage_memory_from_info(&final_snapshot).await;
let mut final_response = final_snapshot;
apply_bucket_usage_memory_overlay_if_authoritative(&mut final_response, true).await;
assert!(final_response.is_complete_bucket_usage_snapshot());
}
#[test]
fn remove_bucket_usage_from_info_drops_bucket_and_recomputes_totals() {
let last_update = SystemTime::now();
@@ -3109,7 +3570,7 @@ mod tests {
"fresh successor snapshot cache must not be erased after namespace lock loss"
);
drop(snapshot_cache);
*data_usage_snapshot_cache().write().await = None;
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test(start_paused = true)]
@@ -3168,7 +3629,7 @@ mod tests {
"fresh successor snapshot cache must not be erased by final invalidation after lock loss"
);
drop(snapshot_cache);
*data_usage_snapshot_cache().write().await = None;
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test]
@@ -3255,6 +3716,7 @@ mod tests {
},
);
legacy.bucket_sizes.insert("bucket-b".to_string(), 126);
legacy.usage_snapshot_complete = false;
legacy.buckets_count = 2;
legacy.calculate_totals();
let store = Arc::new(UsageCasStore {
@@ -3288,6 +3750,13 @@ mod tests {
Some((3, 126))
);
}
for (data, _) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() {
let saved = serde_json::from_slice::<DataUsageInfo>(data).expect("migrated usage snapshot should decode");
assert!(
!saved.usage_snapshot_complete,
"legacy usage must not be promoted to an authoritative snapshot"
);
}
}
#[tokio::test]
@@ -3428,6 +3897,7 @@ mod tests {
},
);
backup.bucket_sizes.insert("bucket-b".to_string(), 126);
backup.buckets_count = 2;
backup.calculate_totals();
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
crates/ecstore/src/diagnostics/admin_server_info.rs
+96 -11
View File
@@ -57,14 +57,14 @@ fn apply_data_usage_result(
usage: &mut rustfs_madmin::Usage,
) {
match result {
Ok(info) => {
Ok(info) if info.is_complete_bucket_usage_snapshot() => {
buckets.count = info.buckets_count;
objects.count = info.objects_total_count;
versions.count = info.versions_total_count;
delete_markers.count = info.delete_markers_total_count;
usage.size = info.objects_total_size;
}
Err(_) => {
Ok(_) | Err(_) => {
buckets.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
objects.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
versions.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
@@ -649,18 +649,15 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
if erasure_set.id == 0 {
erasure_set.id = d.set_index;
if let Ok(cache) = load_data_usage_cache(
match load_data_usage_cache(
&store.pools[d.pool_index as usize].disk_set[d.set_index as usize].clone(),
DATA_USAGE_CACHE_NAME,
)
.await
{
let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::<String>::new());
erasure_set.objects_count = data_usage_info.objects_total_count;
erasure_set.versions_count = data_usage_info.versions_total_count;
erasure_set.delete_markers_count = data_usage_info.delete_markers_total_count;
erasure_set.usage = data_usage_info.objects_total_size;
};
Ok(cache) => apply_erasure_set_usage(&cache, erasure_set),
Err(_) => erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
}
}
erasure_set.raw_capacity += d.total_space;
@@ -672,6 +669,20 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
Ok(pools_info)
}
fn apply_erasure_set_usage(cache: &rustfs_data_usage::DataUsageCache, erasure_set: &mut ErasureSetInfo) {
let data_usage_info = cache.dui(DATA_USAGE_ROOT, &[]);
if !data_usage_info.is_complete_bucket_usage_snapshot() {
erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
return;
}
erasure_set.objects_count = data_usage_info.objects_total_count;
erasure_set.versions_count = data_usage_info.versions_total_count;
erasure_set.delete_markers_count = data_usage_info.delete_markers_total_count;
erasure_set.usage = data_usage_info.objects_total_size;
erasure_set.usage_error = None;
}
#[allow(clippy::const_is_empty)]
pub fn get_commit_id() -> String {
let ver = if !build::TAG.is_empty() {
@@ -697,8 +708,9 @@ mod tests {
use rustfs_madmin::{Disk, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, ServerProperties};
use super::{
DATA_USAGE_UNAVAILABLE_ERROR, apply_data_usage_result, get_local_server_property, get_online_offline_disks_stats,
get_server_info, reconcile_servers_with_endpoint_topology, server_topology_completeness_report,
DATA_USAGE_ROOT, DATA_USAGE_UNAVAILABLE_ERROR, apply_data_usage_result, apply_erasure_set_usage,
get_local_server_property, get_online_offline_disks_stats, get_server_info, reconcile_servers_with_endpoint_topology,
server_topology_completeness_report,
};
fn disk_with_state(endpoint: &str, state: &str) -> Disk {
@@ -900,11 +912,19 @@ mod tests {
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
let mut usage = rustfs_madmin::Usage::default();
let info = rustfs_data_usage::DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
buckets_count: 2,
objects_total_count: 3,
versions_total_count: 4,
delete_markers_total_count: 5,
objects_total_size: 6,
buckets_usage: [
("bucket-a".to_string(), rustfs_data_usage::BucketUsageInfo::default()),
("bucket-b".to_string(), rustfs_data_usage::BucketUsageInfo::default()),
]
.into_iter()
.collect(),
usage_snapshot_complete: true,
..Default::default()
};
@@ -917,6 +937,71 @@ mod tests {
assert_eq!(usage.size, 6);
}
#[test]
fn incomplete_data_usage_is_unavailable_in_server_info() {
let mut buckets = rustfs_madmin::Buckets::default();
let mut objects = rustfs_madmin::Objects::default();
let mut versions = rustfs_madmin::Versions::default();
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
let mut usage = rustfs_madmin::Usage::default();
let info = rustfs_data_usage::DataUsageInfo {
buckets_count: 1,
objects_total_count: 100,
objects_total_size: 1024,
..Default::default()
};
apply_data_usage_result(Ok(info), &mut buckets, &mut objects, &mut versions, &mut delete_markers, &mut usage);
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(objects.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(versions.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(delete_markers.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(usage.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
}
#[test]
fn incomplete_erasure_set_cache_is_not_reported_as_zero() {
let mut cache = rustfs_data_usage::DataUsageCache::default();
cache.info.name = DATA_USAGE_ROOT.to_string();
cache.replace(DATA_USAGE_ROOT, "", rustfs_data_usage::DataUsageEntry::default());
let mut set = rustfs_madmin::ErasureSetInfo::default();
apply_erasure_set_usage(&cache, &mut set);
assert_eq!(set.usage_error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(set.objects_count, 0);
assert_eq!(set.usage, 0);
}
#[test]
fn complete_erasure_set_cache_is_reported() {
let mut cache = rustfs_data_usage::DataUsageCache::default();
cache.info.name = DATA_USAGE_ROOT.to_string();
cache.info.last_update = Some(std::time::SystemTime::UNIX_EPOCH);
cache.info.snapshot_complete = true;
cache.replace(
DATA_USAGE_ROOT,
"",
rustfs_data_usage::DataUsageEntry {
size: 512,
objects: 3,
versions: 4,
delete_markers: 1,
..Default::default()
},
);
let mut set = rustfs_madmin::ErasureSetInfo::default();
apply_erasure_set_usage(&cache, &mut set);
assert_eq!(set.objects_count, 3);
assert_eq!(set.versions_count, 4);
assert_eq!(set.delete_markers_count, 1);
assert_eq!(set.usage, 512);
assert!(set.usage_error.is_none());
}
#[serial]
#[tokio::test]
async fn server_info_includes_global_deployment_id() {
crates/ecstore/src/store/bucket.rs
+24 -13
View File
@@ -326,6 +326,13 @@ impl ECStore {
)
.await?;
if confirmed_missing && !is_meta_bucketname(bucket) {
// A scanner may have sampled the first fence before the bucket
// became visible. Fence again after metadata initialization so
// that snapshot cannot publish as complete.
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
Ok(())
}
@@ -545,6 +552,9 @@ impl ECStore {
"physical bucket deletion succeeded but metadata cleanup remains pending"
);
}
// A scanner may have sampled the first fence before the physical
// namespace disappeared. The completion fence invalidates that scan.
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
Ok(())
}
}
@@ -839,8 +849,8 @@ mod tests {
.expect("bucket should be created");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_make.saturating_add(1),
"successful bucket creation should advance scanner namespace activity"
generation_before_make.saturating_add(2),
"successful bucket creation should fence scanner namespace activity before and after creation"
);
let generation_before_put = ecstore.scanner_namespace_mutation_generation();
@@ -1057,8 +1067,8 @@ mod tests {
.expect("MarkDelete should remove an empty bucket");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_delete.saturating_add(1),
"successful bucket deletion should advance scanner namespace activity"
generation_before_delete.saturating_add(2),
"successful bucket deletion should fence scanner namespace activity before and after deletion"
);
assert!(
@@ -1179,8 +1189,8 @@ mod tests {
.expect("Purge should force-delete bucket data");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_delete.saturating_add(1),
"successful bucket purge should advance scanner namespace activity"
generation_before_delete.saturating_add(2),
"successful bucket purge should fence scanner namespace activity before and after deletion"
);
assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
@@ -1249,7 +1259,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn bucket_delete_finishes_usage_cleanup_before_same_name_recreation() {
async fn bucket_recreation_does_not_publish_unverified_usage() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple());
ecstore
@@ -1271,6 +1281,7 @@ mod tests {
..Default::default()
},
);
snapshot.usage_snapshot_complete = true;
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
@@ -1296,13 +1307,11 @@ mod tests {
.await
.expect("recreated bucket usage base should load");
crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await;
assert_eq!(
recreated
.buckets_usage
.get(&bucket)
.map(|usage| (usage.objects_count, usage.versions_count, usage.size)),
Some((1, 1, 84))
assert!(
!recreated.buckets_usage.contains_key(&bucket),
"a request-path delta without an authoritative baseline must remain unavailable"
);
assert_eq!(crate::data_usage::get_bucket_usage_memory(&bucket).await, None);
}
#[tokio::test]
@@ -1324,6 +1333,7 @@ mod tests {
..Default::default()
},
);
snapshot.usage_snapshot_complete = true;
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
@@ -1373,6 +1383,7 @@ mod tests {
..Default::default()
},
);
snapshot.usage_snapshot_complete = true;
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
crates/madmin/src/info_commands.rs
+36
View File
@@ -317,6 +317,8 @@ pub struct ErasureSetInfo {
pub versions_count: u64,
#[serde(rename = "deleteMarkersCount")]
pub delete_markers_count: u64,
#[serde(rename = "usageError", default, skip_serializing_if = "Option::is_none")]
pub usage_error: Option<String>,
#[serde(rename = "healDisks")]
pub heal_disks: i32,
}
@@ -1107,6 +1109,7 @@ mod tests {
assert_eq!(erasure_set.objects_count, 0);
assert_eq!(erasure_set.versions_count, 0);
assert_eq!(erasure_set.delete_markers_count, 0);
assert!(erasure_set.usage_error.is_none());
assert_eq!(erasure_set.heal_disks, 0);
}
@@ -1120,6 +1123,7 @@ mod tests {
objects_count: 10000,
versions_count: 15000,
delete_markers_count: 500,
usage_error: None,
heal_disks: 2,
};
@@ -1133,6 +1137,38 @@ mod tests {
assert_eq!(erasure_set.heal_disks, 2);
}
#[test]
fn erasure_set_usage_error_is_additive_on_the_wire() {
#[derive(Deserialize)]
struct LegacyErasureSetInfo {
id: i32,
usage: u64,
}
let legacy = r#"{"id":1,"rawUsage":2,"rawCapacity":3,"usage":4,"objectsCount":5,"versionsCount":6,"deleteMarkersCount":7,"healDisks":8}"#;
let decoded: ErasureSetInfo = serde_json::from_str(legacy).expect("legacy erasure set payload should decode");
assert!(decoded.usage_error.is_none());
let encoded = serde_json::to_value(decoded).expect("erasure set payload should encode");
assert!(
encoded.get("usageError").is_none(),
"an absent usage error must not change the legacy wire shape"
);
let current = ErasureSetInfo {
id: 9,
usage: 10,
usage_error: Some("data usage unavailable".to_string()),
..Default::default()
};
let encoded = serde_json::to_value(current).expect("current erasure set payload should encode");
let legacy: LegacyErasureSetInfo =
serde_json::from_value(encoded).expect("legacy erasure set reader should ignore additive fields");
assert_eq!(legacy.id, 9);
assert_eq!(legacy.usage, 10);
}
#[test]
fn test_backend_type_default() {
let backend_type = BackendType::default();
crates/obs/src/metrics/collectors/bucket.rs
+40 -22
View File
@@ -33,10 +33,10 @@ use std::borrow::Cow;
pub struct BucketStats {
/// Name of the bucket
pub name: String,
/// Total size of all objects in the bucket (bytes)
pub size_bytes: u64,
/// Number of objects in the bucket
pub objects_count: u64,
/// Total size of all objects in the bucket (bytes), when authoritative.
pub size_bytes: Option<u64>,
/// Number of objects in the bucket, when authoritative.
pub objects_count: Option<u64>,
/// Quota limit for the bucket (bytes), 0 if no quota
pub quota_bytes: u64,
}
@@ -54,15 +54,19 @@ pub fn collect_bucket_metrics(buckets: &[BucketStats]) -> Vec<PrometheusMetric>
for bucket in buckets {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone());
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, bucket.size_bytes as f64)
.with_label("bucket", bucket_label.clone()),
);
if let Some(size_bytes) = bucket.size_bytes {
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, size_bytes as f64)
.with_label("bucket", bucket_label.clone()),
);
}
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, bucket.objects_count as f64)
.with_label("bucket", bucket_label.clone()),
);
if let Some(objects_count) = bucket.objects_count {
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, objects_count as f64)
.with_label("bucket", bucket_label.clone()),
);
}
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_QUOTA_BYTES_MD, bucket.quota_bytes as f64)
@@ -83,14 +87,14 @@ mod tests {
let buckets = vec![
BucketStats {
name: "test-bucket".to_string(),
size_bytes: 1000,
objects_count: 100,
size_bytes: Some(1000),
objects_count: Some(100),
quota_bytes: 0,
},
BucketStats {
name: "another-bucket".to_string(),
size_bytes: 2000,
objects_count: 200,
size_bytes: Some(2000),
objects_count: Some(200),
quota_bytes: 0,
},
];
@@ -114,8 +118,8 @@ mod tests {
fn test_collect_bucket_metrics_with_quotas() {
let buckets = vec![BucketStats {
name: "quota-bucket".to_string(),
size_bytes: 500,
objects_count: 10,
size_bytes: Some(500),
objects_count: Some(10),
quota_bytes: 10000,
}];
@@ -146,8 +150,8 @@ mod tests {
fn test_collect_bucket_metrics_zero_quota_always_reported() {
let buckets = vec![BucketStats {
name: "no-quota-bucket".to_string(),
size_bytes: 100,
objects_count: 5,
size_bytes: Some(100),
objects_count: Some(5),
quota_bytes: 0,
}];
@@ -165,12 +169,26 @@ mod tests {
assert!(quota_metric.is_some());
}
#[test]
fn unknown_usage_only_reports_quota() {
let metrics = collect_bucket_metrics(&[BucketStats {
name: "unknown-usage".to_string(),
size_bytes: None,
objects_count: None,
quota_bytes: 4096,
}]);
assert_eq!(metrics.len(), 1);
assert_eq!(metrics[0].name, BUCKET_QUOTA_BYTES_MD.get_full_metric_name());
assert_eq!(metrics[0].value, 4096.0);
}
#[test]
fn test_bucket_stats_default() {
let stats = BucketStats::default();
assert!(stats.name.is_empty());
assert_eq!(stats.size_bytes, 0);
assert_eq!(stats.objects_count, 0);
assert_eq!(stats.size_bytes, None);
assert_eq!(stats.objects_count, None);
assert_eq!(stats.quota_bytes, 0);
}
}
crates/obs/src/metrics/collectors/cluster.rs
+36 -11
View File
@@ -43,9 +43,9 @@ pub struct ClusterStats {
/// Number of drives with no capacity observation
pub missing_capacity_drives: u64,
/// Total number of objects in the cluster
pub objects_count: u64,
pub objects_count: Option<u64>,
/// Total number of buckets in the cluster
pub buckets_count: u64,
pub buckets_count: Option<u64>,
}
/// Collects cluster-wide metrics from the provided cluster statistics.
@@ -53,16 +53,21 @@ pub struct ClusterStats {
/// Uses the metric descriptors from `metrics_type::cluster` module.
/// Returns a vector of Prometheus metrics for cluster statistics.
pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
vec![
let mut metrics = vec![
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD, stats.raw_capacity_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD, stats.usable_capacity_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USED_BYTES_MD, stats.used_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_FREE_BYTES_MD, stats.free_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_STALE_DRIVES_MD, stats.stale_capacity_drives as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_MISSING_DRIVES_MD, stats.missing_capacity_drives as f64),
PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, stats.objects_count as f64),
PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, stats.buckets_count as f64),
]
];
if let Some(objects_count) = stats.objects_count {
metrics.push(PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, objects_count as f64));
}
if let Some(buckets_count) = stats.buckets_count {
metrics.push(PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, buckets_count as f64));
}
metrics
}
#[cfg(test)]
@@ -79,8 +84,8 @@ mod tests {
free_bytes: 1300,
stale_capacity_drives: 1,
missing_capacity_drives: 0,
objects_count: 100,
buckets_count: 5,
objects_count: Some(100),
buckets_count: Some(5),
};
let metrics = collect_cluster_metrics(&stats);
@@ -116,7 +121,7 @@ mod tests {
let metrics = collect_cluster_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 8);
assert_eq!(metrics.len(), 6);
// All values should be zero
for metric in &metrics {
@@ -134,7 +139,27 @@ mod tests {
assert_eq!(stats.free_bytes, 0);
assert_eq!(stats.stale_capacity_drives, 0);
assert_eq!(stats.missing_capacity_drives, 0);
assert_eq!(stats.objects_count, 0);
assert_eq!(stats.buckets_count, 0);
assert_eq!(stats.objects_count, None);
assert_eq!(stats.buckets_count, None);
}
#[test]
fn unknown_cluster_cardinality_is_omitted() {
let metrics = collect_cluster_metrics(&ClusterStats {
objects_count: None,
buckets_count: None,
..Default::default()
});
assert!(
metrics
.iter()
.all(|metric| metric.name != CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name())
);
assert!(
metrics
.iter()
.all(|metric| metric.name != CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name())
);
}
}
crates/obs/src/metrics/scheduler.rs
+222 -17
View File
@@ -79,10 +79,13 @@ use crate::metrics::schema::audit::{AUDIT_FAILED_MESSAGES_MD, AUDIT_TARGET_QUEUE
use crate::metrics::schema::bucket_replication::{
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, TARGET_ARN_L,
};
use crate::metrics::schema::cluster::{CLUSTER_BUCKETS_TOTAL_MD, CLUSTER_OBJECTS_TOTAL_MD};
use crate::metrics::schema::cluster_usage::{
BUCKET_LABEL as USAGE_BUCKET_LABEL, RANGE_LABEL as USAGE_RANGE_LABEL, USAGE_BUCKET_DELETE_MARKERS_COUNT_MD,
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, USAGE_BUCKET_OBJECTS_TOTAL_MD,
USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, USAGE_BUCKET_TOTAL_BYTES_MD, USAGE_BUCKET_VERSIONS_COUNT_MD,
USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, USAGE_BUCKET_TOTAL_BYTES_MD, USAGE_BUCKET_VERSIONS_COUNT_MD, USAGE_BUCKETS_COUNT_MD,
USAGE_DELETE_MARKERS_COUNT_MD, USAGE_OBJECTS_COUNT_MD, USAGE_OBJECTS_DISTRIBUTION_MD, USAGE_SINCE_LAST_UPDATE_SECONDS_MD,
USAGE_TOTAL_BYTES_MD, USAGE_VERSIONS_COUNT_MD, USAGE_VERSIONS_DISTRIBUTION_MD,
};
use crate::metrics::schema::node_bucket::{BUCKET_OBJECTS_TOTAL_MD, BUCKET_QUOTA_BYTES_MD, BUCKET_USAGE_BYTES_MD};
use crate::metrics::schema::notification_target::{
@@ -659,6 +662,26 @@ fn bucket_live_keys(stats: &[crate::metrics::collectors::BucketStats]) -> HashSe
stats.iter().map(|stat| stat.name.clone()).collect()
}
fn bucket_observation_live_keys(stats: &[crate::metrics::collectors::BucketStats]) -> HashSet<BucketKey> {
stats
.iter()
.filter(|stat| stat.size_bytes.is_some() || stat.objects_count.is_some())
.map(|stat| stat.name.clone())
.collect()
}
fn bucket_observation_retire_keys(
previous_observations: &HashSet<BucketKey>,
current_buckets: &HashSet<BucketKey>,
current_observations: &HashSet<BucketKey>,
) -> Vec<BucketKey> {
previous_observations
.difference(current_observations)
.filter(|bucket| current_buckets.contains(*bucket))
.cloned()
.collect()
}
fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8>) -> Vec<PrometheusMetric> {
if zero_tombstones.is_empty() {
return Vec::new();
@@ -677,6 +700,50 @@ fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8
zero_metrics
}
#[derive(Default)]
struct BucketSeriesState {
has_seen_snapshot: bool,
live_keys: HashSet<BucketKey>,
observation_keys: HashSet<BucketKey>,
zero_tombstones: HashMap<BucketKey, u8>,
}
struct BucketSeriesUpdate {
metrics: Vec<PrometheusMetric>,
retire_observations: Vec<BucketKey>,
retire_buckets: Vec<BucketKey>,
}
impl BucketSeriesState {
fn observe(
&mut self,
stats: Option<&[crate::metrics::collectors::BucketStats]>,
tombstone_cycles: u8,
) -> Option<BucketSeriesUpdate> {
let stats = stats?;
let current_bucket_keys = bucket_live_keys(stats);
let current_observation_keys = bucket_observation_live_keys(stats);
let retire_observations =
bucket_observation_retire_keys(&self.observation_keys, &current_bucket_keys, &current_observation_keys);
self.observation_keys = current_observation_keys;
update_series_zero_tombstones(
&mut self.has_seen_snapshot,
&mut self.live_keys,
&mut self.zero_tombstones,
current_bucket_keys,
tombstone_cycles,
);
let mut metrics = collect_bucket_metrics(stats);
metrics.extend(collect_bucket_zero_tombstone_metrics(&self.zero_tombstones));
let retire_buckets = expire_series_zero_tombstones(&mut self.zero_tombstones);
Some(BucketSeriesUpdate {
metrics,
retire_observations,
retire_buckets,
})
}
}
fn retire_bucket_metric_series(bucket: &str) -> usize {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.to_string());
let labels = [("bucket", bucket_label.clone())];
@@ -685,6 +752,32 @@ fn retire_bucket_metric_series(bucket: &str) -> usize {
+ retire_metric_series(&BUCKET_QUOTA_BYTES_MD.get_full_metric_name(), &labels)
}
fn retire_bucket_observation_metric_series(bucket: &str) -> usize {
let labels = [("bucket", Cow::Owned(bucket.to_string()))];
retire_metric_series(&BUCKET_USAGE_BYTES_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels)
}
fn retire_cluster_usage_metric_series() -> usize {
let labels: [(&'static str, Cow<'static, str>); 0] = [];
[
USAGE_SINCE_LAST_UPDATE_SECONDS_MD.get_full_metric_name(),
USAGE_TOTAL_BYTES_MD.get_full_metric_name(),
USAGE_OBJECTS_COUNT_MD.get_full_metric_name(),
USAGE_VERSIONS_COUNT_MD.get_full_metric_name(),
USAGE_DELETE_MARKERS_COUNT_MD.get_full_metric_name(),
USAGE_BUCKETS_COUNT_MD.get_full_metric_name(),
]
.iter()
.map(|name| retire_metric_series(name, &labels))
.sum()
}
fn retire_cluster_usage_distribution_series(metric_name: String, range: &str) -> usize {
let labels = [(USAGE_RANGE_LABEL, Cow::Owned(range.to_string()))];
retire_metric_series(&metric_name, &labels)
}
fn bucket_usage_live_keys(stats: &[crate::metrics::collectors::BucketUsageStats]) -> HashSet<BucketKey> {
stats.iter().map(|stat| stat.bucket.clone()).collect()
}
@@ -973,11 +1066,23 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let token_clone = token.clone();
tokio::spawn(async move {
let mut interval = metrics_interval(cluster_interval, Duration::ZERO);
let mut objects_count_was_authoritative = false;
let mut buckets_count_was_authoritative = false;
loop {
tokio::select! {
_ = interval.tick() => {
run_metrics_collector_tick(health, MetricsCollectorTaskId::ClusterStats, "cluster_stats", async {
let (stats, cluster_health) = collect_cluster_and_health_stats().await;
if objects_count_was_authoritative && stats.objects_count.is_none() {
let labels: [(&'static str, Cow<'static, str>); 0] = [];
let _ = retire_metric_series(&CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels);
}
if buckets_count_was_authoritative && stats.buckets_count.is_none() {
let labels: [(&'static str, Cow<'static, str>); 0] = [];
let _ = retire_metric_series(&CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name(), &labels);
}
objects_count_was_authoritative = stats.objects_count.is_some();
buckets_count_was_authoritative = stats.buckets_count.is_some();
let mut metrics = collect_cluster_metrics(&stats);
metrics.extend(collect_cluster_health_metrics(&cluster_health));
report_metrics(&metrics);
@@ -1004,6 +1109,8 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let mut bucket_usage_object_size_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
let mut prev_bucket_usage_version_keys: HashSet<BucketRangeKey> = HashSet::new();
let mut bucket_usage_version_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
let mut prev_cluster_usage_object_size_keys: HashSet<String> = HashSet::new();
let mut prev_cluster_usage_version_keys: HashSet<String> = HashSet::new();
loop {
tokio::select! {
_ = interval.tick() => {
@@ -1028,6 +1135,30 @@ pub fn init_metrics_runtime(token: CancellationToken) {
}
if let Some((cluster_usage, bucket_usage)) = collect_cluster_usage_metric_stats().await {
let current_cluster_usage_object_size_keys = cluster_usage
.object_size_distribution
.iter()
.map(|(range, _)| range.clone())
.collect::<HashSet<_>>();
for range in prev_cluster_usage_object_size_keys.difference(&current_cluster_usage_object_size_keys) {
let _ = retire_cluster_usage_distribution_series(
USAGE_OBJECTS_DISTRIBUTION_MD.get_full_metric_name(),
range,
);
}
prev_cluster_usage_object_size_keys = current_cluster_usage_object_size_keys;
let current_cluster_usage_version_keys = cluster_usage
.versions_distribution
.iter()
.map(|(range, _)| range.clone())
.collect::<HashSet<_>>();
for range in prev_cluster_usage_version_keys.difference(&current_cluster_usage_version_keys) {
let _ = retire_cluster_usage_distribution_series(
USAGE_VERSIONS_DISTRIBUTION_MD.get_full_metric_name(),
range,
);
}
prev_cluster_usage_version_keys = current_cluster_usage_version_keys;
metrics.extend(collect_cluster_usage_metrics(&cluster_usage));
update_series_zero_tombstones(
&mut has_seen_bucket_usage_snapshot,
@@ -1073,6 +1204,41 @@ pub fn init_metrics_runtime(token: CancellationToken) {
&range,
);
}
} else if has_seen_bucket_usage_snapshot {
let _ = retire_cluster_usage_metric_series();
for range in prev_cluster_usage_object_size_keys.drain() {
let _ = retire_cluster_usage_distribution_series(
USAGE_OBJECTS_DISTRIBUTION_MD.get_full_metric_name(),
&range,
);
}
for range in prev_cluster_usage_version_keys.drain() {
let _ = retire_cluster_usage_distribution_series(
USAGE_VERSIONS_DISTRIBUTION_MD.get_full_metric_name(),
&range,
);
}
for bucket in prev_bucket_usage_keys.drain() {
let _ = retire_bucket_usage_metric_series(&bucket);
}
for (bucket, range) in prev_bucket_usage_object_size_keys.drain() {
let _ = retire_bucket_usage_distribution_series(
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD.get_full_metric_name(),
&bucket,
&range,
);
}
for (bucket, range) in prev_bucket_usage_version_keys.drain() {
let _ = retire_bucket_usage_distribution_series(
USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD.get_full_metric_name(),
&bucket,
&range,
);
}
bucket_usage_zero_tombstones.clear();
bucket_usage_object_size_zero_tombstones.clear();
bucket_usage_version_zero_tombstones.clear();
has_seen_bucket_usage_snapshot = false;
}
if !metrics.is_empty() {
@@ -1094,25 +1260,20 @@ pub fn init_metrics_runtime(token: CancellationToken) {
tokio::spawn(async move {
let mut interval = metrics_interval(bucket_interval, Duration::ZERO);
let tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
let mut has_seen_bucket_snapshot = false;
let mut prev_bucket_keys: HashSet<BucketKey> = HashSet::new();
let mut bucket_zero_tombstones: HashMap<BucketKey, u8> = HashMap::new();
let mut series_state = BucketSeriesState::default();
loop {
tokio::select! {
_ = interval.tick() => {
run_metrics_collector_tick(health, MetricsCollectorTaskId::BucketStats, "bucket_stats", async {
let stats = collect_bucket_stats().await;
update_series_zero_tombstones(
&mut has_seen_bucket_snapshot,
&mut prev_bucket_keys,
&mut bucket_zero_tombstones,
bucket_live_keys(&stats),
tombstone_cycles,
);
let mut metrics = collect_bucket_metrics(&stats);
metrics.extend(collect_bucket_zero_tombstone_metrics(&bucket_zero_tombstones));
report_metrics(&metrics);
for bucket in expire_series_zero_tombstones(&mut bucket_zero_tombstones) {
let Some(update) = series_state.observe(stats.as_deref(), tombstone_cycles) else {
return;
};
for bucket in update.retire_observations {
let _ = retire_bucket_observation_metric_series(&bucket);
}
report_metrics(&update.metrics);
for bucket in update.retire_buckets {
let _ = retire_bucket_metric_series(&bucket);
}
}).await;
@@ -1850,8 +2011,8 @@ mod tests {
let mut zero_tombstones = HashMap::new();
let live_stats = vec![crate::metrics::collectors::BucketStats {
name: "tmp".to_string(),
size_bytes: 1024,
objects_count: 8,
size_bytes: Some(1024),
objects_count: Some(8),
quota_bytes: 2048,
}];
@@ -1885,6 +2046,50 @@ mod tests {
assert!(zero_tombstones.is_empty());
}
#[test]
fn bucket_observation_retirement_distinguishes_unknown_usage_from_deletion() {
let previous = HashSet::from(["bucket".to_string()]);
let unknown_stats = vec![crate::metrics::collectors::BucketStats {
name: "bucket".to_string(),
size_bytes: None,
objects_count: None,
quota_bytes: 1024,
}];
let current_buckets = bucket_live_keys(&unknown_stats);
let current_observations = bucket_observation_live_keys(&unknown_stats);
assert_eq!(
bucket_observation_retire_keys(&previous, &current_buckets, &current_observations),
vec!["bucket".to_string()],
"an existing bucket with unknown usage must retire its previous usage observations"
);
assert!(
bucket_observation_retire_keys(&previous, &HashSet::new(), &HashSet::new()).is_empty(),
"a deleted bucket remains governed by the zero-tombstone lifecycle"
);
}
#[test]
fn unavailable_bucket_snapshot_preserves_metric_series_state() {
let mut state = BucketSeriesState::default();
let initial = [crate::metrics::collectors::BucketStats {
name: "bucket".to_string(),
size_bytes: Some(512),
objects_count: Some(2),
quota_bytes: 1024,
}];
assert!(state.observe(Some(&initial), 2).is_some());
let live_keys = state.live_keys.clone();
let observation_keys = state.observation_keys.clone();
let zero_tombstones = state.zero_tombstones.clone();
assert!(state.observe(None, 2).is_none());
assert_eq!(state.live_keys, live_keys);
assert_eq!(state.observation_keys, observation_keys);
assert_eq!(state.zero_tombstones, zero_tombstones);
}
#[test]
fn parse_system_metrics_interval_rounds_legacy_millis_up_to_one_second() {
temp_env::with_vars(
crates/obs/src/metrics/stats_collector.rs
+124 -29
View File
@@ -41,7 +41,11 @@ use rustfs_io_metrics::{
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_resource_and_system,
snapshot_process_resource_and_system_with,
};
use std::{collections::HashMap, sync::Arc, time::SystemTime};
use std::{
collections::{HashMap, HashSet},
sync::Arc,
time::SystemTime,
};
use sysinfo::{Networks, System};
use tracing::{instrument, warn};
@@ -52,8 +56,10 @@ const EVENT_METRICS_COLLECTOR_STATE: &str = "metrics_collector_state";
type ObsStorageInfo = <ObsStore as StorageAdminApi>::StorageInfo;
type ObsBackendInfo = <ObsStore as StorageAdminApi>::BackendInfo;
#[derive(Default)]
struct ObsDataUsageInfo {
last_update: Option<SystemTime>,
usage_snapshot_complete: bool,
buckets_count: u64,
objects_total_count: u64,
versions_total_count: u64,
@@ -62,6 +68,7 @@ struct ObsDataUsageInfo {
buckets_usage: HashMap<String, ObsBucketUsageInfo>,
}
#[derive(Default)]
struct ObsBucketUsageInfo {
size: u64,
objects_count: u64,
@@ -85,9 +92,11 @@ fn usize_to_u64_saturating(value: usize) -> u64 {
async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreResult<ObsDataUsageInfo> {
let data_usage = obs_load_data_usage_from_backend(store).await?;
let usage_snapshot_complete = data_usage.is_complete_bucket_usage_snapshot();
Ok(ObsDataUsageInfo {
last_update: data_usage.last_update,
usage_snapshot_complete,
buckets_count: data_usage.buckets_count,
objects_total_count: data_usage.objects_total_count,
versions_total_count: data_usage.versions_total_count,
@@ -113,6 +122,21 @@ async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreRes
})
}
fn bucket_usage_metric_values(data_usage: Option<&ObsDataUsageInfo>, bucket: &str) -> (Option<u64>, Option<u64>) {
data_usage
.filter(|usage| usage.usage_snapshot_complete)
.and_then(|usage| usage.buckets_usage.get(bucket))
.map(|usage| (Some(usage.size), Some(usage.objects_count)))
.unwrap_or((None, None))
}
fn data_usage_snapshot_covers_bucket_namespace(data_usage: &ObsDataUsageInfo, buckets: &HashSet<String>) -> bool {
data_usage.usage_snapshot_complete
&& u64::try_from(buckets.len()).ok() == Some(data_usage.buckets_count)
&& buckets.len() == data_usage.buckets_usage.len()
&& buckets.iter().all(|bucket| data_usage.buckets_usage.contains_key(bucket))
}
fn resolve_obs_object_store_handle() -> Option<Arc<ObsStore>> {
obs_resolve_object_store_handle()
}
@@ -365,25 +389,16 @@ pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthS
})
.count() as u64;
// Get bucket and object counts from data usage info.
let (buckets_count, objects_count) = match load_obs_data_usage_from_backend(store.clone()).await {
Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
Err(e) => {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "data_usage_load_failed", error = %e, "metrics collector state changed");
// Fall back to bucket list for buckets_count, objects_count stays 0.
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
.unwrap_or_else(|err| {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "bucket_list_failed", error = %err, "metrics collector state changed");
Vec::new()
});
(buckets.len() as u64, 0)
let data_usage = match load_obs_data_usage_from_backend(store).await {
Ok(data_usage) if data_usage.usage_snapshot_complete => Some(data_usage),
Ok(_) => None,
Err(error) => {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "data_usage_load_failed", error = %error, "metrics collector state changed");
None
}
};
let buckets_count = data_usage.as_ref().map(|usage| usage.buckets_count);
let objects_count = data_usage.as_ref().map(|usage| usage.objects_total_count);
let mut online = 0u64;
let mut offline = 0u64;
@@ -428,10 +443,12 @@ pub async fn collect_cluster_health_stats() -> ClusterHealthStats {
}
/// Collect bucket statistics from the storage layer.
pub async fn collect_bucket_stats() -> Vec<BucketStats> {
let Some(store) = resolve_obs_object_store_handle() else {
return Vec::new();
};
///
/// `None` means the bucket namespace could not be observed. Callers must keep
/// their prior metric-series state instead of treating that failure as an
/// authoritative empty namespace.
pub async fn collect_bucket_stats() -> Option<Vec<BucketStats>> {
let store = resolve_obs_object_store_handle()?;
// Load data usage info from backend to get bucket sizes and object counts
let data_usage = match load_obs_data_usage_from_backend(store.clone()).await {
@@ -453,7 +470,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
Ok(buckets) => buckets,
Err(e) => {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "bucket_stats", result = "bucket_list_failed", error = %e, "metrics collector state changed");
return Vec::new();
return None;
}
};
@@ -465,11 +482,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
}
// Get size and objects_count from data usage info
let (size_bytes, objects_count) = data_usage
.as_ref()
.and_then(|du| du.buckets_usage.get(&bucket.name))
.map(|bui| (bui.size, bui.objects_count))
.unwrap_or((0, 0));
let (size_bytes, objects_count) = bucket_usage_metric_values(data_usage.as_ref(), &bucket.name);
// Get quota from bucket metadata
let quota_bytes = obs_bucket_quota_limit_bytes(&bucket.name).await;
@@ -482,7 +495,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
});
}
stats
Some(stats)
}
/// Collect bucket replication bandwidth stats from the global monitor.
@@ -935,6 +948,29 @@ pub async fn collect_iam_stats() -> Option<IamStats> {
pub async fn collect_cluster_usage_metric_stats() -> Option<(ClusterUsageStats, Vec<BucketUsageStats>)> {
let store = resolve_obs_object_store_handle()?;
let data_usage = load_obs_data_usage_from_backend(store.clone()).await.ok()?;
let bucket_namespace = store
.list_bucket(&BucketOptions {
cached: true,
no_metadata: true,
..Default::default()
})
.await
.ok()?
.into_iter()
.filter(|bucket| !bucket.name.starts_with('.'))
.map(|bucket| bucket.name)
.collect::<HashSet<_>>();
collect_cluster_usage_metric_stats_from_data_usage(data_usage, &bucket_namespace).await
}
async fn collect_cluster_usage_metric_stats_from_data_usage(
data_usage: ObsDataUsageInfo,
bucket_namespace: &HashSet<String>,
) -> Option<(ClusterUsageStats, Vec<BucketUsageStats>)> {
if !data_usage_snapshot_covers_bucket_namespace(&data_usage, bucket_namespace) {
return None;
}
let mut buckets = Vec::with_capacity(data_usage.buckets_usage.len());
for (bucket_name, usage) in &data_usage.buckets_usage {
@@ -1187,6 +1223,65 @@ mod tests {
info
}
#[test]
fn bucket_usage_metrics_distinguish_unknown_from_confirmed_zero() {
assert_eq!(bucket_usage_metric_values(None, "bucket"), (None, None));
let mut data_usage = ObsDataUsageInfo {
usage_snapshot_complete: true,
..Default::default()
};
data_usage
.buckets_usage
.insert("bucket".to_string(), ObsBucketUsageInfo::default());
assert_eq!(bucket_usage_metric_values(Some(&data_usage), "bucket"), (Some(0), Some(0)));
assert_eq!(bucket_usage_metric_values(Some(&data_usage), "missing"), (None, None));
}
#[tokio::test]
async fn cluster_usage_metrics_skip_incomplete_snapshot() {
assert!(
collect_cluster_usage_metric_stats_from_data_usage(ObsDataUsageInfo::default(), &HashSet::new())
.await
.is_none()
);
}
#[tokio::test]
async fn cluster_usage_metrics_publish_complete_empty_snapshot() {
let (cluster, buckets) = collect_cluster_usage_metric_stats_from_data_usage(
ObsDataUsageInfo {
usage_snapshot_complete: true,
..Default::default()
},
&HashSet::new(),
)
.await
.expect("complete empty usage should remain publishable");
assert_eq!(cluster.buckets_count, 0);
assert_eq!(cluster.objects_count, 0);
assert_eq!(cluster.total_bytes, 0);
assert!(buckets.is_empty());
}
#[tokio::test]
async fn cluster_usage_metrics_skip_snapshot_for_a_different_bucket_namespace() {
let data_usage = ObsDataUsageInfo {
usage_snapshot_complete: true,
buckets_count: 1,
buckets_usage: HashMap::from([("stale-bucket".to_string(), ObsBucketUsageInfo::default())]),
..Default::default()
};
assert!(
collect_cluster_usage_metric_stats_from_data_usage(data_usage, &HashSet::from(["live-bucket".to_string()]))
.await
.is_none()
);
}
#[test]
fn cluster_config_stats_accept_homogeneous_backend_parities() {
let stats = cluster_config_stats_from_backend_parities(Some(1), Some(2))
crates/scanner/src/lib.rs
+8 -4
View File
@@ -35,10 +35,10 @@ use storage_api::owner::{
EcstoreTierConfig, EcstoreVersioningApi, HTTPPreconditions, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete,
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_data_usage_snapshot_cache,
ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
@@ -462,6 +462,10 @@ pub(crate) async fn replace_bucket_usage_memory_from_info(data_usage_info: &rust
ecstore_replace_bucket_usage_memory_from_info(data_usage_info).await;
}
pub(crate) async fn invalidate_data_usage_snapshot_cache() {
ecstore_invalidate_data_usage_snapshot_cache().await;
}
pub trait ScannerObjectIO:
ObjectIO<
Error = EcstoreError,
crates/scanner/src/scanner.rs
+135 -90
View File
@@ -61,8 +61,9 @@ use crate::storage_api::scan::{
};
use crate::{
ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _,
get_lifecycle_config, get_replication_config, read_config, replace_bucket_usage_memory_from_info, save_config,
save_config_shared_with_preconditions, save_config_with_preconditions, scanner_is_erasure_sd,
get_lifecycle_config, get_replication_config, invalidate_data_usage_snapshot_cache, read_config,
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions,
scanner_is_erasure_sd,
};
const LOG_COMPONENT_SCANNER: &str = "scanner";
@@ -985,7 +986,7 @@ fn initial_scanner_delay_for_startup(
}
fn data_usage_info_is_cold(info: &DataUsageInfo) -> bool {
info.last_update.is_none() || (info.buckets_usage.is_empty() && info.bucket_sizes.is_empty())
!info.is_complete_bucket_usage_snapshot()
}
async fn read_data_usage_config_for_startup(storeapi: &Arc<impl ScannerObjectIO>) -> Result<Option<Vec<u8>>, EcstoreError> {
@@ -1004,10 +1005,12 @@ async fn read_data_usage_config_for_startup(storeapi: &Arc<impl ScannerObjectIO>
}
}
match read_pair(storeapi, DATA_USAGE_OBJ_NAME_PATH.as_str()).await? {
Some(data) => Ok(Some(data)),
None => read_pair(storeapi, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()).await,
for path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] {
if let Some(data) = read_pair(storeapi, path).await? {
return Ok(Some(data));
}
}
Ok(None)
}
fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool {
@@ -3611,6 +3614,21 @@ async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
data_usage_info.scanner_epoch = Some(leader_epoch);
}
if !data_usage_info.is_complete_bucket_usage_snapshot() {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "reject_incomplete_snapshot",
"Scanner refused to persist an incomplete data usage snapshot"
);
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed);
outcome = DataUsagePersistOutcome::Failed;
continue;
}
let data = match serde_json::to_vec(&data_usage_info) {
Ok(data) => data,
Err(e) => {
@@ -3767,16 +3785,19 @@ async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
match save_outcome {
DataUsagePersistOutcome::Current => {
invalidate_data_usage_snapshot_cache().await;
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::SkippedStale);
outcome = DataUsagePersistOutcome::Current;
continue;
}
DataUsagePersistOutcome::AlreadyDurable => {
invalidate_data_usage_snapshot_cache().await;
replace_bucket_usage_memory_from_info(&data_usage_info).await;
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
outcome = DataUsagePersistOutcome::AlreadyDurable;
}
DataUsagePersistOutcome::PriorCycleDurable => {
invalidate_data_usage_snapshot_cache().await;
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
outcome = DataUsagePersistOutcome::PriorCycleDurable;
}
@@ -3786,6 +3807,7 @@ async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
continue;
}
DataUsagePersistOutcome::Saved => {
invalidate_data_usage_snapshot_cache().await;
replace_bucket_usage_memory_from_info(&data_usage_info).await;
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
outcome = DataUsagePersistOutcome::Saved;
@@ -4485,8 +4507,25 @@ mod tests {
assert_eq!(epoch, 11);
}
#[test]
fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() {
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::now()), 1);
legacy.usage_snapshot_complete = false;
assert!(data_usage_info_is_cold(&legacy));
assert!(!data_usage_info_is_cold(&complete_usage_with_bucket_count(
Some(std::time::SystemTime::now()),
1,
)));
assert!(!data_usage_info_is_cold(&DataUsageInfo {
last_update: Some(std::time::SystemTime::now()),
usage_snapshot_complete: true,
..Default::default()
}));
}
#[tokio::test]
async fn scanner_startup_migrates_legacy_usage_only_until_v2_exists() {
async fn scanner_startup_prefers_v2_over_legacy_usage() {
let store = Arc::new(MemoryConfigStore::default());
let legacy = DataUsageInfo {
scanner_epoch: Some(19),
@@ -4517,9 +4556,10 @@ mod tests {
);
let authoritative = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(11),
scanner_epoch: Some(23),
scanner_cycle: Some(51),
last_update: Some(std::time::SystemTime::now()),
usage_snapshot_complete: true,
..Default::default()
};
let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode");
@@ -4539,8 +4579,8 @@ mod tests {
.await
.expect("v2 usage floor should be authoritative"),
PersistedUsageFloor {
next_cycle: 12,
leader_epoch: 7,
next_cycle: 52,
leader_epoch: 23,
}
);
@@ -4594,7 +4634,7 @@ mod tests {
scanner_epoch: Some(1),
scanner_cycle: Some(cycle),
last_update: Some(std::time::SystemTime::now()),
..Default::default()
..complete_usage_with_bucket_count(None, 0)
})
.await
.expect("usage update should queue");
@@ -4959,16 +4999,8 @@ mod tests {
let (sender, receiver) = mpsc::channel(2);
let ctx = CancellationToken::new();
let newer = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
buckets_count: 2,
..Default::default()
};
let older = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
buckets_count: 1,
..Default::default()
};
let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
let older = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
sender.send(newer).await.expect("newer usage snapshot should enqueue");
sender.send(older).await.expect("older usage snapshot should enqueue");
@@ -4993,16 +5025,8 @@ mod tests {
let (sender, receiver) = mpsc::channel(1);
let ctx = CancellationToken::new();
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let newer = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
buckets_count: 2,
..Default::default()
};
let stale = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
buckets_count: 1,
..Default::default()
};
let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
let stale = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
store
.interleaving_puts
.lock()
@@ -5045,6 +5069,7 @@ mod tests {
initial.bucket_sizes.insert("bucket-a".to_string(), 84);
initial.buckets_count = 1;
initial.calculate_totals();
mark_usage_snapshot_complete(&mut initial);
let initial_data = serde_json::to_vec(&initial).expect("initial usage snapshot should encode");
store.objects.lock().await.insert(key.clone(), initial_data.clone());
store.revisions.lock().await.insert(key.clone(), 1);
@@ -5054,6 +5079,7 @@ mod tests {
deleted.bucket_sizes.clear();
deleted.buckets_count = 0;
deleted.calculate_totals();
mark_usage_snapshot_complete(&mut deleted);
store
.interleaving_puts
.lock()
@@ -5102,7 +5128,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
scanner_epoch: Some(8),
scanner_cycle: Some(1),
..Default::default()
..complete_usage_with_bucket_count(None, 0)
};
store.objects.lock().await.insert(
backup_key.clone(),
@@ -5129,6 +5155,7 @@ mod tests {
incoming.bucket_sizes.insert("bucket-a".to_string(), 84);
incoming.buckets_count = 1;
incoming.calculate_totals();
mark_usage_snapshot_complete(&mut incoming);
sender.send(incoming).await.expect("usage snapshot should enqueue");
}
drop(sender);
@@ -5162,7 +5189,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
scanner_epoch: Some(8),
scanner_cycle: Some(10),
..Default::default()
..complete_usage_with_bucket_count(None, 0)
};
let encoded = serde_json::to_vec(&durable).expect("usage snapshot should encode");
store.objects.lock().await.insert(main_key.clone(), encoded.clone());
@@ -5205,6 +5232,7 @@ mod tests {
incoming.bucket_sizes.insert("bucket-a".to_string(), 84);
incoming.buckets_count = 1;
incoming.calculate_totals();
mark_usage_snapshot_complete(&mut incoming);
let mut deleted = incoming.clone();
deleted.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(31));
@@ -5212,6 +5240,7 @@ mod tests {
deleted.bucket_sizes.clear();
deleted.buckets_count = 0;
deleted.calculate_totals();
mark_usage_snapshot_complete(&mut deleted);
store.replace_after_successful_puts.lock().await.insert(
main_key.clone(),
(1, serde_json::to_vec(&deleted).expect("deleted primary snapshot should encode")),
@@ -5251,21 +5280,9 @@ mod tests {
let (sender, receiver) = mpsc::channel(1);
let ctx = CancellationToken::new();
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let initial = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
buckets_count: 1,
..Default::default()
};
let stale_winner = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
buckets_count: 2,
..Default::default()
};
let current = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
buckets_count: 3,
..Default::default()
};
let initial = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 3);
let stale_winner = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 3);
let current = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), 3);
store
.objects
.lock()
@@ -5298,21 +5315,13 @@ mod tests {
}
#[tokio::test]
async fn test_store_data_usage_in_backend_rejects_untimestamped_stale_snapshot() {
async fn test_store_data_usage_in_backend_rejects_untimestamped_complete_snapshot() {
let store = Arc::new(MemoryConfigStore::default());
let (sender, receiver) = mpsc::channel(2);
let ctx = CancellationToken::new();
let timestamped = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
buckets_count: 2,
..Default::default()
};
let untimestamped = DataUsageInfo {
last_update: None,
buckets_count: 1,
..Default::default()
};
let timestamped = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
let untimestamped = complete_usage_with_bucket_count(None, 1);
sender
.send(timestamped)
@@ -5334,7 +5343,7 @@ mod tests {
assert_eq!(saved.buckets_count, 2);
assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)));
assert_eq!(outcome, DataUsagePersistOutcome::Current);
assert_eq!(outcome, DataUsagePersistOutcome::Failed);
}
#[tokio::test]
@@ -5346,8 +5355,7 @@ mod tests {
let snapshot = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
scanner_cycle: Some(12),
buckets_count: 2,
..Default::default()
..complete_usage_with_bucket_count(None, 2)
};
store
.objects
@@ -5377,8 +5385,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
scanner_epoch: Some(8),
scanner_cycle: Some(12),
buckets_count: 2,
..Default::default()
..complete_usage_with_bucket_count(None, 2)
};
store
.objects
@@ -5392,8 +5399,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
scanner_epoch: Some(8),
scanner_cycle: Some(12),
buckets_count: 3,
..Default::default()
..complete_usage_with_bucket_count(None, 3)
})
.await
.expect("changed retry snapshot should enqueue");
@@ -5426,8 +5432,7 @@ mod tests {
let existing = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)),
scanner_cycle: Some(12),
buckets_count: 2,
..Default::default()
..complete_usage_with_bucket_count(None, 2)
};
store
.objects
@@ -5440,8 +5445,7 @@ mod tests {
let older = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
scanner_cycle: Some(11),
buckets_count: 1,
..Default::default()
..complete_usage_with_bucket_count(None, 1)
};
older_sender.send(older).await.expect("older-cycle snapshot should enqueue");
drop(older_sender);
@@ -5454,8 +5458,7 @@ mod tests {
let newer = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
scanner_cycle: Some(13),
buckets_count: 3,
..Default::default()
..complete_usage_with_bucket_count(None, 3)
};
newer_sender
.send(newer.clone())
@@ -5490,8 +5493,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)),
scanner_epoch: Some(8),
scanner_cycle: Some(12),
buckets_count: 2,
..Default::default()
..complete_usage_with_bucket_count(None, 2)
};
store
.objects
@@ -5506,8 +5508,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
scanner_epoch: Some(7),
scanner_cycle: Some(99),
buckets_count: 1,
..Default::default()
..complete_usage_with_bucket_count(None, 1)
})
.await
.expect("old-epoch snapshot should enqueue");
@@ -5523,8 +5524,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
scanner_epoch: None,
scanner_cycle: Some(1),
buckets_count: 3,
..Default::default()
..complete_usage_with_bucket_count(None, 3)
})
.await
.expect("replacement-epoch snapshot should enqueue");
@@ -5554,8 +5554,7 @@ mod tests {
let existing = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
scanner_cycle: Some(12),
buckets_count: 2,
..Default::default()
..complete_usage_with_bucket_count(None, 2)
};
store
.objects
@@ -5569,8 +5568,7 @@ mod tests {
.send(DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
scanner_cycle: Some(12),
buckets_count: 3,
..Default::default()
..complete_usage_with_bucket_count(None, 3)
})
.await
.expect("conflicting same-cycle snapshot should enqueue");
@@ -5595,11 +5593,59 @@ mod tests {
);
}
fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo {
DataUsageInfo {
#[tokio::test]
async fn test_store_data_usage_in_backend_rejects_incomplete_snapshot() {
let store = Arc::new(MemoryConfigStore::default());
let (sender, receiver) = mpsc::channel(2);
let complete_update = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10);
sender
.send(complete_usage_with_bucket_count(Some(complete_update), 1))
.await
.expect("complete usage snapshot should enqueue");
sender
.send(DataUsageInfo {
last_update: Some(complete_update + Duration::from_secs(1)),
buckets_count: 1,
..Default::default()
})
.await
.expect("incomplete usage snapshot should enqueue");
drop(sender);
let outcome = store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await;
let objects = store.objects.lock().await;
let saved = objects
.get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()))
.expect("complete data usage snapshot should remain saved");
let saved = serde_json::from_slice::<DataUsageInfo>(saved).expect("saved usage snapshot should decode");
assert_eq!(saved.last_update, Some(complete_update));
assert!(saved.is_complete_bucket_usage_snapshot());
assert_eq!(outcome, DataUsagePersistOutcome::Failed);
}
fn mark_usage_snapshot_complete(info: &mut DataUsageInfo) {
info.usage_snapshot_complete = true;
}
fn complete_usage_with_bucket_count(last_update: Option<std::time::SystemTime>, buckets_count: u64) -> DataUsageInfo {
let mut info = DataUsageInfo {
last_update,
buckets_count,
usage_snapshot_complete: true,
..Default::default()
};
for index in 0..buckets_count {
let bucket = format!("bucket-{index}");
info.buckets_usage.insert(bucket.clone(), Default::default());
info.bucket_sizes.insert(bucket, 0);
}
info
}
fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo {
complete_usage_with_bucket_count(last_update, 0)
}
#[test]
@@ -5681,11 +5727,10 @@ mod tests {
for idx in 1_u64..=11 {
sender
.send(DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(idx)),
buckets_count: idx,
..Default::default()
})
.send(complete_usage_with_bucket_count(
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(idx)),
idx,
))
.await
.expect("usage snapshot should enqueue");
}
crates/scanner/src/scanner_io.rs
+47 -4
View File
@@ -846,7 +846,7 @@ fn apply_bucket_result_to_cache(cache: &mut DataUsageCache, result: DataUsageEnt
}
fn should_publish_completed_snapshot(completed_count: usize, total_count: usize, budget_elapsed: bool, cancelled: bool) -> bool {
total_count > 0 && completed_count == total_count && !budget_elapsed && !cancelled
completed_count == total_count && !budget_elapsed && !cancelled
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -1015,6 +1015,10 @@ fn completed_data_usage_info(
}
let merged_last_update = results.iter().filter_map(|result| result.info.last_update).max()?;
let bucket_sizes = buckets_usage
.iter()
.map(|(bucket, usage)| (bucket.clone(), usage.size))
.collect();
let data_usage_info = DataUsageInfo {
last_update: Some(merged_last_update),
scanner_cycle: Some(results.first()?.info.next_cycle),
@@ -1023,7 +1027,9 @@ fn completed_data_usage_info(
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
objects_total_size: u64::try_from(total.size).ok()?,
buckets_count: u64::try_from(all_buckets.len()).ok()?,
bucket_sizes,
buckets_usage,
usage_snapshot_complete: true,
..Default::default()
};
Some((data_usage_info, merged_last_update))
@@ -1042,7 +1048,10 @@ mod publish_gate_tests {
assert!(!should_publish_completed_snapshot(2, 3, false, false));
assert!(!should_publish_completed_snapshot(3, 3, true, false));
assert!(!should_publish_completed_snapshot(3, 3, false, true));
assert!(!should_publish_completed_snapshot(0, 0, false, false));
assert!(
should_publish_completed_snapshot(0, 0, false, false),
"a completed empty namespace is an authoritative zero snapshot"
);
}
fn incomplete_scope_cache(source: DataUsageCacheSource) -> DataUsageCache {
@@ -1121,11 +1130,13 @@ mod publish_gate_tests {
#[test]
fn completed_data_usage_info_requires_every_set_before_publish() {
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()];
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string(), "bucket-empty".to_string()];
let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
first_set.replace("bucket-b", DATA_USAGE_ROOT, DataUsageEntry::default());
first_set.replace("bucket-empty", DATA_USAGE_ROOT, DataUsageEntry::default());
let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0));
second_set.replace("bucket-a", DATA_USAGE_ROOT, DataUsageEntry::default());
second_set.replace("bucket-empty", DATA_USAGE_ROOT, DataUsageEntry::default());
assert!(
completed_data_usage_info_for_test(&[first_set.clone(), DataUsageCache::default()], &all_buckets, false, false)
@@ -1144,7 +1155,38 @@ mod publish_gate_tests {
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
assert_eq!(data_usage_info.scanner_cycle, Some(0));
assert_eq!(data_usage_info.objects_total_count, 3);
assert_eq!(data_usage_info.buckets_usage.len(), 2);
assert_eq!(data_usage_info.buckets_usage.len(), 3);
assert!(data_usage_info.usage_snapshot_complete);
assert_eq!(
data_usage_info
.buckets_usage
.get("bucket-empty")
.map(|usage| (usage.objects_count, usage.size)),
Some((0, 0))
);
}
#[test]
fn completed_data_usage_info_publishes_confirmed_empty_namespace() {
let mut completed_set = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
source: Some(DataUsageCacheSource::new(0, 0)),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
..Default::default()
},
..Default::default()
};
completed_set.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
let (data_usage_info, _) = completed_data_usage_info_for_test(&[completed_set], &[], false, false)
.expect("a completed empty namespace should produce an authoritative snapshot");
assert!(data_usage_info.is_complete_bucket_usage_snapshot());
assert_eq!(data_usage_info.buckets_count, 0);
assert!(data_usage_info.buckets_usage.is_empty());
}
#[test]
@@ -2000,6 +2042,7 @@ impl ScannerIOCycle for ECStore {
let empty_usage = DataUsageInfo {
last_update: Some(SystemTime::now()),
scanner_cycle: Some(want_cycle),
usage_snapshot_complete: true,
..Default::default()
};
send_data_usage_update(&updates, empty_usage).await?;
crates/scanner/src/storage_api.rs
+10 -6
View File
@@ -57,7 +57,10 @@ pub(crate) use rustfs_ecstore::api::config::init as ecstore_config_init;
pub(crate) use rustfs_ecstore::api::config::storageclass::{
RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD,
};
pub(crate) use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info;
pub(crate) use rustfs_ecstore::api::data_usage::{
invalidate_data_usage_snapshot_cache as ecstore_invalidate_data_usage_snapshot_cache,
replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
@@ -107,11 +110,12 @@ pub(crate) mod owner {
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi,
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr,
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure,
ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle,
ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config,
ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
crates/test-utils/src/data_usage_snapshot_tests.rs
+1
View File
@@ -34,6 +34,7 @@ fn snapshot(bucket: &str, last_update: SystemTime) -> DataUsageInfo {
..Default::default()
},
);
info.usage_snapshot_complete = true;
info.bucket_sizes.insert(bucket.to_string(), 42);
info.buckets_count = 1;
info.calculate_totals();