gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-28 17:18:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): require authoritative usage snapshots (#5333)

Browse Source
This commit is contained in:
Henry Guo
2026-07-28 07:29:57 +08:00
committed by GitHub
parent 5a768d3a44
commit d5df66ac4f
25 changed files with 1829 additions and 374 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/data-usage/src/data_usage.rs
+54 -5
View File
@@ -32,12 +32,12 @@ use std::{
/// save forever and freeze admin usage stats; callers must bypass the skip instead.
pub const USAGE_LAST_UPDATE_FUTURE_TOLERANCE: Duration = Duration::from_secs(5 * 60);
/// Authoritative cluster-wide usage snapshot written by coordinated scanners.
/// Cluster-wide usage snapshot written by coordinated scanners.
///
/// This object name is intentionally distinct from the legacy unfenced
/// snapshot. Older binaries can continue writing the legacy object during a
/// rolling upgrade without overwriting a snapshot produced by the current
/// scanner protocol.
/// `usage_snapshot_complete` is an additive JSON field: older readers ignore
/// it, while current readers treat snapshots from older writers as unknown.
/// Keeping the existing object name preserves rolling-upgrade and rollback
/// compatibility without allowing an ambiguous snapshot to become authoritative.
pub const DATA_USAGE_OBJECT_NAME: &str = ".usage.v2.json";
/// Usage snapshot written by scanner implementations predating distributed
@@ -190,6 +190,12 @@ pub struct DataUsageInfo {
pub buckets_count: u64,
/// Buckets usage info provides following information across all buckets
pub buckets_usage: HashMap<String, BucketUsageInfo>,
/// Whether this snapshot covers the complete bucket namespace.
///
/// Legacy snapshots default to `false`. A complete snapshot contains an
/// explicit entry for every bucket, including confirmed-empty buckets.
#[serde(default)]
pub usage_snapshot_complete: bool,
/// Deprecated kept here for backward compatibility reasons
pub bucket_sizes: HashMap<String, u64>,
/// Per-disk snapshot information when available
@@ -681,6 +687,12 @@ pub struct DataUsageCacheInfo {
pub skip_healing: bool,
#[serde(default)]
pub failed_objects: HashMap<String, u64>,
/// Whether this per-set cache was produced by a completed scanner pass.
///
/// Older cache writers omit this field and therefore deserialize as
/// incomplete instead of exposing partial set totals as confirmed zeros.
#[serde(default)]
pub snapshot_complete: bool,
}
/// Data usage cache
@@ -1000,6 +1012,7 @@ impl DataUsageCache {
objects_total_size: flat.size as u64,
buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX),
buckets_usage,
usage_snapshot_complete: self.info.snapshot_complete,
..Default::default()
}
}
@@ -1078,6 +1091,13 @@ impl DataUsageInfo {
Self::default()
}
/// Whether this snapshot authoritatively covers every reported bucket.
pub fn is_complete_bucket_usage_snapshot(&self) -> bool {
self.usage_snapshot_complete
&& self.last_update.is_some()
&& u64::try_from(self.buckets_usage.len()).ok() == Some(self.buckets_count)
}
/// Add object metadata to data usage statistics
pub fn add_object(&mut self, object_path: &str, meta_object: &rustfs_filemeta::MetaObject) {
// This method is kept for backward compatibility
@@ -1442,6 +1462,35 @@ pub struct CompressionTotalInfo {
mod tests {
use super::*;
#[derive(Deserialize)]
struct LegacyUsageReader {
buckets_count: u64,
}
#[test]
fn completeness_marker_is_additive_for_legacy_named_readers() {
let current = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH),
usage_snapshot_complete: true,
..Default::default()
};
let encoded = rmp_serde::to_vec_named(&current).expect("encode current data usage snapshot");
let legacy: LegacyUsageReader = rmp_serde::from_slice(&encoded).expect("legacy reader should ignore additive fields");
assert_eq!(legacy.buckets_count, 0);
assert!(current.is_complete_bucket_usage_snapshot());
}
#[test]
fn completeness_marker_requires_a_snapshot_timestamp() {
let untimestamped = DataUsageInfo {
usage_snapshot_complete: true,
..Default::default()
};
assert!(!untimestamped.is_complete_bucket_usage_snapshot());
}
#[test]
fn test_usage_last_update_future_tolerance_boundary() {
let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
crates/ecstore/src/api/mod.rs
+4 -4
View File
@@ -287,10 +287,10 @@ pub mod config {
pub mod data_usage {
pub use crate::data_usage::{
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
init_compression_total_memory_from_backend, live_bucket_usage_computations, load_compression_total_from_memory,
load_data_usage_from_backend, load_data_usage_from_backend_cached, record_bucket_delete_marker_memory,
record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
init_compression_total_memory_from_backend, invalidate_data_usage_snapshot_cache, live_bucket_usage_computations,
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
store_data_usage_in_backend,
crates/ecstore/src/bucket/quota/checker.rs
+28 -2
View File
@@ -167,7 +167,7 @@ impl QuotaChecker {
}
let quota = self.get_quota_config(bucket).await?;
let current_usage = self.get_real_time_usage(bucket).await.unwrap_or(0);
let current_usage = self.get_real_time_usage(bucket).await?;
Ok((quota, Some(current_usage)))
}
@@ -177,7 +177,11 @@ impl QuotaChecker {
}
pub async fn get_real_time_usage(&self, bucket: &str) -> Result<u64, QuotaError> {
Ok(get_bucket_usage_memory(bucket).await.unwrap_or(0))
get_bucket_usage_memory(bucket)
.await
.ok_or_else(|| QuotaError::UsageUnavailable {
bucket: bucket.to_string(),
})
}
}
@@ -185,6 +189,8 @@ impl QuotaChecker {
mod tests {
use super::*;
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
use serial_test::serial;
use uuid::Uuid;
/// Regression (PR #5307 / s3-tests `test_100_continue_error_retry`): a
/// bucket with no persisted metadata has no quota, so the admission check
@@ -205,6 +211,26 @@ mod tests {
assert_eq!(result.quota_limit, None);
}
#[tokio::test]
#[serial]
async fn quota_usage_rejects_an_unknown_mutation_baseline() {
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
let checker = QuotaChecker::new(sys);
let bucket = format!("quota-unknown-{}", Uuid::new_v4().simple());
crate::data_usage::record_bucket_object_write_memory(&bucket, None, 42).await;
let result = checker.get_real_time_usage(&bucket).await;
crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
.await
.expect("test usage cache cleanup should succeed");
assert!(
matches!(result, Err(QuotaError::UsageUnavailable { bucket: failed_bucket }) if failed_bucket == bucket),
"quota decisions must fail closed without an authoritative usage baseline"
);
}
#[tokio::test]
async fn test_quota_check_no_limit() {
let result = QuotaCheckResult {
crates/ecstore/src/bucket/quota/mod.rs
+3 -1
View File
@@ -110,6 +110,8 @@ pub enum QuotaError {
QuotaExceeded { current: u64, limit: u64, operation: u64 },
#[error("Quota configuration not found for bucket: {bucket}")]
ConfigNotFound { bucket: String },
#[error("Authoritative data usage is unavailable for bucket: {bucket}")]
UsageUnavailable { bucket: String },
#[error("Invalid quota configuration: {reason}")]
InvalidConfig { reason: String },
#[error("Storage error: {0}")]
@@ -155,7 +157,7 @@ impl QuotaErrorResponse {
request_id: request_id.to_string(),
host_id: host_id.to_string(),
},
QuotaError::StorageError(_) => Self {
QuotaError::UsageUnavailable { .. } | QuotaError::StorageError(_) => Self {
code: QUOTA_INTERNAL_ERROR_CODE.to_string(),
message: quota_error.to_string(),
resource: QUOTA_API_PATH.to_string(),
crates/ecstore/src/data_usage/mod.rs
+550 -80
View File
@@ -48,7 +48,7 @@ use std::{
time::{Duration, SystemTime},
};
use tokio::fs;
use tokio::sync::RwLock;
use tokio::sync::{Mutex as TokioMutex, RwLock};
use tracing::{debug, error, info, instrument};
// Data usage storage constants
@@ -63,12 +63,17 @@ const DATA_USAGE_REMOVE_CAS_RETRIES: usize = 3;
#[derive(Debug, Clone)]
struct CachedBucketUsage {
usage: BucketUsageInfo,
authoritative: bool,
refreshed_at: SystemTime,
usage_updated_at: SystemTime,
// Set by request-path mutations until a scanner snapshot catches up to the same core counts.
dirty: bool,
// Set when a newer scanner snapshot was observed but did not include the dirty counts yet.
stale_snapshot_pending: bool,
// First complete scanner generation observed after an unknown-baseline
// mutation. A strictly later generation is required before the mutation
// evidence can be discarded.
pending_scanner_position: Option<(u64, u64)>,
}
type UsageMemoryCache = Arc<RwLock<HashMap<String, CachedBucketUsage>>>;
@@ -78,6 +83,7 @@ type LiveBucketUsageCache = moka::future::Cache<String, BucketUsageInfo>;
static USAGE_MEMORY_CACHE: OnceLock<UsageMemoryCache> = OnceLock::new();
static USAGE_CACHE_UPDATING: OnceLock<CacheUpdating> = OnceLock::new();
static LIVE_BUCKET_USAGE_CACHE: OnceLock<LiveBucketUsageCache> = OnceLock::new();
static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
/// Cached copy of the last persisted data usage snapshot, served to admin
/// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
@@ -110,8 +116,13 @@ fn cache_data_usage_snapshot_result(
cache: &mut Option<CachedDataUsageSnapshot>,
result: Result<DataUsageInfo, Error>,
loaded_at: tokio::time::Instant,
) -> Result<DataUsageInfo, Error> {
match result {
refresh_generation: u64,
) -> Option<Result<DataUsageInfo, Error>> {
if data_usage_snapshot_generation() != refresh_generation {
return None;
}
Some(match result {
Ok(info) => {
*cache = Some(CachedDataUsageSnapshot {
info: Some(info.clone()),
@@ -123,12 +134,14 @@ fn cache_data_usage_snapshot_result(
*cache = Some(CachedDataUsageSnapshot { info: None, loaded_at });
Err(e)
}
}
})
}
type DataUsageSnapshotCache = Arc<RwLock<Option<CachedDataUsageSnapshot>>>;
static DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
static DATA_USAGE_SNAPSHOT_REFRESH: OnceLock<Arc<TokioMutex<()>>> = OnceLock::new();
static DATA_USAGE_SNAPSHOT_GENERATION: AtomicU64 = AtomicU64::new(0);
// Always-on revert detector for rustfs/backlog#1306: one relaxed increment per
// full-bucket version listing is negligible and lets tests prove that admin
@@ -183,6 +196,15 @@ fn data_usage_snapshot_cache() -> &'static DataUsageSnapshotCache {
DATA_USAGE_SNAPSHOT_CACHE.get_or_init(|| Arc::new(RwLock::new(None)))
}
fn data_usage_snapshot_generation() -> u64 {
DATA_USAGE_SNAPSHOT_GENERATION.load(Ordering::Acquire)
}
fn clear_data_usage_snapshot_cache(cache: &mut Option<CachedDataUsageSnapshot>) {
DATA_USAGE_SNAPSHOT_GENERATION.fetch_add(1, Ordering::AcqRel);
*cache = None;
}
fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
LIVE_BUCKET_USAGE_CACHE.get_or_init(|| {
moka::future::Cache::builder()
@@ -191,6 +213,10 @@ fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
})
}
fn usage_memory_generation() -> u64 {
USAGE_MEMORY_GENERATION.load(Ordering::Acquire)
}
// Data usage storage paths
lazy_static::lazy_static! {
pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}",
@@ -245,9 +271,21 @@ fn stale_data_usage_persist_reason(incoming: &DataUsageInfo, existing: &DataUsag
enum UsageSnapshotSource {
Primary,
Backup,
LegacyPrimary,
LegacyBackup,
Missing,
}
impl UsageSnapshotSource {
fn is_authoritative(self) -> bool {
matches!(self, Self::Primary | Self::Backup)
}
fn is_backup(self) -> bool {
matches!(self, Self::Backup | Self::LegacyBackup)
}
}
fn stale_data_usage_persist_reason_for_source(
incoming: &DataUsageInfo,
existing: &DataUsageInfo,
@@ -255,7 +293,7 @@ fn stale_data_usage_persist_reason_for_source(
now: SystemTime,
) -> Option<&'static str> {
let reason = stale_data_usage_persist_reason(incoming, existing, now);
if source == UsageSnapshotSource::Backup && incoming.last_update == existing.last_update {
if source.is_backup() && incoming.last_update == existing.last_update {
None
} else {
reason
@@ -267,6 +305,7 @@ fn stale_data_usage_persist_reason_for_source(
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
// Prevent older data from overwriting newer persisted stats
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
&& source.is_authoritative()
&& let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
{
info!(
@@ -292,7 +331,7 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
// next request instead of waiting out the remaining TTL. The next cached
// read reloads through `load_data_usage_from_backend`, keeping its
// backward-compatibility post-processing.
*data_usage_snapshot_cache().write().await = None;
invalidate_data_usage_snapshot_cache().await;
Ok(())
}
@@ -343,12 +382,13 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error> {
ensure_bucket_namespace_guard(guard, bucket, "data usage cache cleanup")?;
let _ = USAGE_MEMORY_GENERATION.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)));
live_bucket_usage_cache().invalidate(bucket).await;
clear_bucket_usage_memory(bucket, guard).await?;
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache cleanup")?;
*snapshot_cache = None;
clear_data_usage_snapshot_cache(&mut snapshot_cache);
Ok(())
}
@@ -363,7 +403,7 @@ where
let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
*snapshot_cache = None;
clear_data_usage_snapshot_cache(&mut snapshot_cache);
result
}
@@ -386,7 +426,9 @@ where
.filter(|etag| !etag.is_empty())
.map(str::to_owned)
.ok_or_else(|| Error::other("data usage snapshot has no ETag"))?;
let data_usage_info = normalize_loaded_data_usage(parse_usage_snapshot(&reader.read_all().await?)?).await;
let mut data_usage_info = parse_usage_snapshot(&reader.read_all().await?)?;
populate_backward_compatible_usage_maps(&mut data_usage_info);
validate_complete_usage_snapshot(&mut data_usage_info);
Ok(Some((data_usage_info, revision)))
}
@@ -468,12 +510,15 @@ async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<DataUs
where
S: EcstoreObjectIO + ?Sized,
{
for object in [
DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
for (object, authoritative) in [
(DATA_USAGE_OBJ_BACKUP_PATH.as_str(), true),
(LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), false),
(LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(), false),
] {
if let Some((data_usage_info, _)) = load_data_usage_for_bucket_removal(store, object).await? {
if let Some((mut data_usage_info, _)) = load_data_usage_for_bucket_removal(store, object).await? {
if !authoritative {
data_usage_info.usage_snapshot_complete = false;
}
return Ok(data_usage_info);
}
}
@@ -692,13 +737,21 @@ async fn load_data_usage_snapshot_from_store<S: EcstoreObjectIO>(
}
let legacy_primary = read_config_preserve_empty(store.clone(), &LEGACY_DATA_USAGE_OBJ_NAME_PATH).await;
resolve_loaded_snapshot_pair_with_source(
let legacy = resolve_loaded_snapshot_pair_with_source(
legacy_primary,
async move { read_config_preserve_empty(store, &LEGACY_DATA_USAGE_OBJ_BACKUP_PATH).await },
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
)
.await
.await?;
Ok((
legacy.0,
match legacy.1 {
UsageSnapshotSource::Primary => UsageSnapshotSource::LegacyPrimary,
UsageSnapshotSource::Backup => UsageSnapshotSource::LegacyBackup,
source => source,
},
))
}
async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo, UsageSnapshotSource), Error> {
@@ -708,13 +761,27 @@ async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo,
/// Load data usage info from backend storage
#[instrument(skip(store))]
pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
Ok(normalize_loaded_data_usage(load_data_usage_snapshot(store).await?.0).await)
let (data_usage_info, source) = load_data_usage_snapshot(store).await?;
Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await)
}
async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo) -> DataUsageInfo {
info!("Loaded data usage info from backend with {} buckets", data_usage_info.buckets_count);
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
if !data_usage_info.is_complete_bucket_usage_snapshot() {
data_usage_info.usage_snapshot_complete = false;
data_usage_info.buckets_usage.clear();
data_usage_info.bucket_sizes.clear();
data_usage_info.buckets_count = 0;
data_usage_info.calculate_totals();
}
}
// Handle backward compatibility
fn validate_complete_usage_snapshot(data_usage_info: &mut DataUsageInfo) {
if data_usage_info.usage_snapshot_complete && !data_usage_info.is_complete_bucket_usage_snapshot() {
data_usage_info.usage_snapshot_complete = false;
}
}
fn populate_backward_compatible_usage_maps(data_usage_info: &mut DataUsageInfo) {
if data_usage_info.buckets_usage.is_empty() {
data_usage_info.buckets_usage = data_usage_info
.bucket_sizes
@@ -738,6 +805,17 @@ async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo) -> Data
.map(|(bucket, bui)| (bucket.clone(), bui.size))
.collect();
}
}
async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo, authoritative_format: bool) -> DataUsageInfo {
info!("Loaded data usage info from backend with {} buckets", data_usage_info.buckets_count);
if !authoritative_format {
data_usage_info.usage_snapshot_complete = false;
}
populate_backward_compatible_usage_maps(&mut data_usage_info);
validate_complete_usage_snapshot(&mut data_usage_info);
discard_incomplete_bucket_usage(&mut data_usage_info);
// Handle replication info
for (bucket, bui) in &data_usage_info.buckets_usage {
@@ -774,22 +852,43 @@ async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo) -> Data
pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
{
let cache = data_usage_snapshot_cache().read().await;
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
loop {
{
let cache = data_usage_snapshot_cache().read().await;
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
return result;
}
}
// Serialize refreshes without holding the cache lock across backend I/O.
let refresh_guard = DATA_USAGE_SNAPSHOT_REFRESH
.get_or_init(|| Arc::new(TokioMutex::new(())))
.lock()
.await;
{
let cache = data_usage_snapshot_cache().read().await;
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
return result;
}
}
let refresh_generation = data_usage_snapshot_generation();
let result = load_data_usage_from_backend(store.clone()).await;
let loaded_at = tokio::time::Instant::now();
let mut cache = data_usage_snapshot_cache().write().await;
if let Some(result) = cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation) {
return result;
}
drop(cache);
drop(refresh_guard);
}
}
// Re-check under the write lock so concurrent expirations trigger a single
// backend read instead of a stampede.
/// Invalidate the process-local persisted usage snapshot cache after a durable
/// scanner write.
pub async fn invalidate_data_usage_snapshot_cache() {
let mut cache = data_usage_snapshot_cache().write().await;
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
return result;
}
let result = load_data_usage_from_backend(store).await;
cache_data_usage_snapshot_result(&mut cache, result, tokio::time::Instant::now())
clear_data_usage_snapshot_cache(&mut cache);
}
/// Aggregate usage information from local disk snapshots.
@@ -1114,8 +1213,12 @@ where
}
fn apply_live_bucket_usage_to_response(data_usage_info: &mut DataUsageInfo, bucket: &str, usage: &BucketUsageInfo) {
let inserted_bucket = !data_usage_info.buckets_usage.contains_key(bucket);
data_usage_info.bucket_sizes.insert(bucket.to_string(), usage.size);
data_usage_info.buckets_usage.insert(bucket.to_string(), usage.clone());
if inserted_bucket {
data_usage_info.usage_snapshot_complete = false;
}
set_buckets_count_from_usage(data_usage_info);
data_usage_info.calculate_totals();
}
@@ -1180,13 +1283,15 @@ async fn ensure_bucket_usage_cached(bucket: &str) {
update_usage_cache_if_needed().await;
}
fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTime) -> CachedBucketUsage {
fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTime, authoritative: bool) -> CachedBucketUsage {
CachedBucketUsage {
usage,
authoritative,
refreshed_at: SystemTime::now(),
usage_updated_at: updated_at,
dirty: false,
stale_snapshot_pending: false,
pending_scanner_position: None,
}
}
@@ -1194,10 +1299,12 @@ fn cached_bucket_usage_now(usage: BucketUsageInfo) -> CachedBucketUsage {
let now = SystemTime::now();
CachedBucketUsage {
usage,
authoritative: false,
refreshed_at: now,
usage_updated_at: now,
dirty: false,
stale_snapshot_pending: false,
pending_scanner_position: None,
}
}
@@ -1212,6 +1319,30 @@ fn bucket_usage_counts_match(left: &BucketUsageInfo, right: &BucketUsageInfo) ->
&& left.delete_markers_count == right.delete_markers_count
}
fn preserve_unknown_dirty_usage(
existing: &CachedBucketUsage,
snapshot_position: Option<(u64, u64)>,
snapshot_update: SystemTime,
) -> Option<CachedBucketUsage> {
if existing.authoritative || !existing.dirty {
return None;
}
if existing
.pending_scanner_position
.zip(snapshot_position)
.is_some_and(|(previous, current)| current.0 > previous.0 || (current.0 == previous.0 && current.1 > previous.1))
{
return None;
}
let mut preserved = existing.clone();
preserved.stale_snapshot_pending = true;
if preserved.pending_scanner_position.is_none() && snapshot_update >= existing.usage_updated_at {
preserved.pending_scanner_position = snapshot_position;
}
Some(preserved)
}
#[cfg(test)]
async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: BucketUsageInfo, refresh_started_at: SystemTime) {
let mut cache = memory_cache().write().await;
@@ -1221,7 +1352,7 @@ async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: Buc
return;
}
cache.insert(bucket.to_string(), cached_bucket_usage_from_backend(usage, refresh_started_at));
cache.insert(bucket.to_string(), cached_bucket_usage_from_backend(usage, refresh_started_at, true));
}
/// Fast in-memory update for immediate quota and admin usage consistency.
@@ -1271,6 +1402,7 @@ async fn record_bucket_object_write_memory_inner(
entry.usage_updated_at = now;
entry.dirty = true;
entry.stale_snapshot_pending = false;
entry.pending_scanner_position = None;
}
/// Degraded in-memory update for an object write whose previous current size
@@ -1299,6 +1431,7 @@ pub async fn record_bucket_object_write_unknown_previous_memory(bucket: &str, ne
entry.usage_updated_at = now;
entry.dirty = true;
entry.stale_snapshot_pending = false;
entry.pending_scanner_position = None;
}
/// Fast in-memory increment for immediate quota consistency.
@@ -1326,6 +1459,7 @@ pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64,
entry.usage_updated_at = now;
entry.dirty = true;
entry.stale_snapshot_pending = false;
entry.pending_scanner_position = None;
}
/// Fast in-memory update for successful delete marker creation.
@@ -1344,6 +1478,7 @@ pub async fn record_bucket_delete_marker_memory(bucket: &str) {
entry.usage_updated_at = now;
entry.dirty = true;
entry.stale_snapshot_pending = false;
entry.pending_scanner_position = None;
}
/// Fast in-memory decrement for immediate quota consistency
@@ -1354,8 +1489,8 @@ pub async fn decrement_bucket_usage_memory(bucket: &str, size_decrement: u64) {
/// Get bucket usage from the authoritative cache for this topology.
async fn get_persisted_bucket_usage(bucket: &str) -> Option<u64> {
let store = runtime_sources::object_store_handle()?;
let data_usage_info = load_data_usage_from_backend_cached(store).await.ok()?;
data_usage_info.buckets_usage.get(bucket).map(|usage| usage.size)
let info = load_data_usage_from_backend_cached(store).await.ok()?;
info.buckets_usage.get(bucket).map(|usage| usage.size)
}
pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
@@ -1371,7 +1506,10 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
update_usage_cache_if_needed().await;
let cache = memory_cache().read().await;
cache.get(bucket).map(|cached| cached.usage.size)
cache
.get(bucket)
.filter(|cached| cached.authoritative)
.map(|cached| cached.usage.size)
}
async fn update_usage_cache_if_needed() {
@@ -1401,11 +1539,12 @@ async fn update_usage_cache_if_needed() {
drop(updating);
let updating_clone = (*cache_updating()).clone();
let refresh_generation = usage_memory_generation();
tokio::spawn(async move {
if let Some(store) = runtime_sources::object_store_handle()
&& let Ok(data_usage_info) = load_data_usage_from_backend(store).await
{
replace_bucket_usage_memory_from_info(&data_usage_info).await;
replace_bucket_usage_memory_from_info_if_generation(&data_usage_info, Some(refresh_generation)).await;
}
let mut updating = updating_clone.write().await;
*updating = false;
@@ -1426,10 +1565,11 @@ async fn update_usage_cache_if_needed() {
*updating = true;
drop(updating);
let refresh_generation = usage_memory_generation();
if let Some(store) = runtime_sources::object_store_handle()
&& let Ok(data_usage_info) = load_data_usage_from_backend(store).await
{
replace_bucket_usage_memory_from_info(&data_usage_info).await;
replace_bucket_usage_memory_from_info_if_generation(&data_usage_info, Some(refresh_generation)).await;
}
let mut updating = cache_updating().write().await;
@@ -1437,40 +1577,57 @@ async fn update_usage_cache_if_needed() {
}
pub async fn replace_bucket_usage_memory_from_info(data_usage_info: &DataUsageInfo) {
let usage_updated_at = data_usage_info_updated_at(data_usage_info);
let mut cache = memory_cache().write().await;
let mut next_cache = HashMap::new();
replace_bucket_usage_memory_from_info_if_generation(data_usage_info, None).await;
}
for (bucket, bucket_usage) in data_usage_info.buckets_usage.iter() {
if let Some(existing) = cache.get(bucket) {
if existing.usage_updated_at > usage_updated_at {
next_cache.insert(bucket.clone(), existing.clone());
continue;
}
if existing.dirty && !bucket_usage_counts_match(&existing.usage, bucket_usage) {
// A scanner snapshot can be saved after newer writes but still miss them if it listed the bucket earlier.
let mut preserved = existing.clone();
preserved.stale_snapshot_pending = true;
next_cache.insert(bucket.clone(), preserved);
continue;
}
}
next_cache.insert(bucket.clone(), cached_bucket_usage_from_backend(bucket_usage.clone(), usage_updated_at));
async fn replace_bucket_usage_memory_from_info_if_generation(data_usage_info: &DataUsageInfo, expected_generation: Option<u64>) {
if !data_usage_info.is_complete_bucket_usage_snapshot() {
return;
}
for (bucket, existing) in cache.iter() {
if !data_usage_info.buckets_usage.contains_key(bucket) {
if existing.usage_updated_at > usage_updated_at {
next_cache.insert(bucket.clone(), existing.clone());
continue;
}
let usage_updated_at = data_usage_info_updated_at(data_usage_info);
let snapshot_position = data_usage_info.scanner_epoch.zip(data_usage_info.scanner_cycle);
let mut next_cache = HashMap::with_capacity(data_usage_info.buckets_usage.len());
for (bucket, bucket_usage) in data_usage_info.buckets_usage.iter() {
next_cache.insert(
bucket.clone(),
cached_bucket_usage_from_backend(bucket_usage.clone(), usage_updated_at, true),
);
}
if existing.dirty {
let mut preserved = existing.clone();
preserved.stale_snapshot_pending = true;
next_cache.insert(bucket.clone(), preserved);
let mut cache = memory_cache().write().await;
if expected_generation.is_some_and(|expected| usage_memory_generation() != expected) {
return;
}
for (bucket, existing) in cache.iter() {
match next_cache.entry(bucket.clone()) {
Entry::Occupied(mut candidate) => {
if let Some(preserved) = preserve_unknown_dirty_usage(existing, snapshot_position, usage_updated_at) {
candidate.insert(preserved);
continue;
}
if existing.authoritative && existing.usage_updated_at > usage_updated_at {
candidate.insert(existing.clone());
continue;
}
if existing.authoritative && existing.dirty && !bucket_usage_counts_match(&existing.usage, &candidate.get().usage)
{
// A scanner snapshot can be saved after newer writes but still miss them if it listed the bucket earlier.
let mut preserved = existing.clone();
preserved.stale_snapshot_pending = true;
candidate.insert(preserved);
}
}
Entry::Vacant(candidate) => {
if let Some(preserved) = preserve_unknown_dirty_usage(existing, snapshot_position, usage_updated_at) {
candidate.insert(preserved);
} else if existing.authoritative && existing.usage_updated_at > usage_updated_at {
candidate.insert(existing.clone());
} else if existing.authoritative && existing.dirty {
let mut preserved = existing.clone();
preserved.stale_snapshot_pending = true;
candidate.insert(preserved);
}
}
}
}
@@ -1490,18 +1647,29 @@ async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mu
let persisted_update = data_usage_info.last_update;
let mut changed = false;
let mut added_bucket = false;
for (bucket, cached) in cache.iter() {
if !cached.authoritative {
if cached.dirty {
data_usage_info.usage_snapshot_complete = false;
}
continue;
}
if !cached.stale_snapshot_pending && persisted_update.is_some_and(|persisted| cached.usage_updated_at <= persisted) {
continue;
}
added_bucket |= !data_usage_info.buckets_usage.contains_key(bucket);
data_usage_info.buckets_usage.insert(bucket.clone(), cached.usage.clone());
data_usage_info.bucket_sizes.insert(bucket.clone(), cached.usage.size);
changed = true;
}
if changed {
if added_bucket {
data_usage_info.usage_snapshot_complete = false;
}
data_usage_info.buckets_count = data_usage_info.buckets_usage.len() as u64;
data_usage_info.calculate_totals();
}
@@ -2061,6 +2229,7 @@ mod tests {
..Default::default()
},
);
info.usage_snapshot_complete = true;
info.bucket_sizes.insert(bucket.to_string(), size);
info.buckets_count = info.buckets_usage.len() as u64;
info.calculate_totals();
@@ -2184,12 +2353,140 @@ mod tests {
assert!(!err.to_string().contains("secret-marker"));
}
#[tokio::test]
async fn legacy_snapshot_bucket_usage_is_unknown_without_completeness_marker() {
let mut legacy = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
legacy.usage_snapshot_complete = true;
legacy.buckets_usage.insert("large".to_string(), BucketUsageInfo::default());
legacy.bucket_sizes.insert("large".to_string(), 0);
legacy.buckets_count = 2;
let normalized = normalize_loaded_data_usage(legacy, false).await;
assert_eq!(normalized.buckets_count, 0);
assert!(normalized.buckets_usage.is_empty());
assert!(normalized.bucket_sizes.is_empty());
assert_eq!(normalized.objects_total_count, 0);
assert_eq!(normalized.objects_total_size, 0);
assert!(!normalized.usage_snapshot_complete);
}
#[test]
fn legacy_snapshot_json_defaults_completeness_to_unknown() {
let info = data_usage_info_for_test("legacy", 1, 42, SystemTime::UNIX_EPOCH);
let mut value = serde_json::to_value(info).expect("serialize data usage fixture");
let object = value.as_object_mut().expect("data usage fixture should be a JSON object");
object.remove("usage_snapshot_complete");
let decoded: DataUsageInfo = serde_json::from_value(value).expect("deserialize legacy data usage fixture");
assert!(!decoded.usage_snapshot_complete);
}
#[test]
fn complete_snapshot_json_is_additive_for_legacy_readers() {
#[derive(serde::Deserialize)]
struct LegacyUsageReader {
buckets_count: u64,
}
let info = data_usage_info_for_test("current", 1, 42, SystemTime::UNIX_EPOCH);
let encoded = serde_json::to_vec(&info).expect("serialize current data usage fixture");
let legacy: LegacyUsageReader =
serde_json::from_slice(&encoded).expect("legacy JSON reader should ignore additive fields");
assert_eq!(legacy.buckets_count, 1);
}
#[tokio::test]
async fn complete_snapshot_preserves_confirmed_empty_bucket() {
let mut info = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
info.buckets_count = 2;
let normalized = normalize_loaded_data_usage(info, true).await;
assert_eq!(normalized.buckets_count, 2);
assert!(normalized.usage_snapshot_complete);
assert_eq!(
normalized
.buckets_usage
.get("control")
.map(|usage| (usage.objects_count, usage.size)),
Some((10, 10_285))
);
assert_eq!(
normalized
.buckets_usage
.get("empty")
.map(|usage| (usage.objects_count, usage.size)),
Some((0, 0))
);
}
#[tokio::test]
async fn malformed_complete_snapshot_is_rejected_as_a_whole() {
let mut info = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
info.buckets_usage.insert(
"partial".to_string(),
BucketUsageInfo {
objects_count: 1_502,
versions_count: 1_502,
size: 196_870_144,
..Default::default()
},
);
info.bucket_sizes.insert("partial".to_string(), 196_870_144);
info.buckets_count = 1;
let normalized = normalize_loaded_data_usage(info, true).await;
assert_eq!(normalized.buckets_count, 0);
assert!(!normalized.buckets_usage.contains_key("control"));
assert!(!normalized.buckets_usage.contains_key("partial"));
assert!(!normalized.bucket_sizes.contains_key("partial"));
assert!(!normalized.usage_snapshot_complete);
}
#[tokio::test]
async fn completeness_marker_does_not_fabricate_a_missing_bucket() {
let mut info = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
info.buckets_count = 2;
assert!(!data_usage_contains_bucket(&info, "missing"));
let normalized = normalize_loaded_data_usage(info, true).await;
assert!(!normalized.usage_snapshot_complete);
assert!(normalized.buckets_usage.is_empty());
assert!(!normalized.buckets_usage.contains_key("missing"));
}
#[tokio::test]
async fn complete_empty_snapshot_remains_authoritative() {
let normalized = normalize_loaded_data_usage(
DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH),
usage_snapshot_complete: true,
..Default::default()
},
true,
)
.await;
assert!(normalized.usage_snapshot_complete);
assert_eq!(normalized.buckets_count, 0);
assert!(normalized.buckets_usage.is_empty());
}
#[test]
#[serial]
fn cached_snapshot_failure_is_reused_until_ttl_expires() {
let loaded_at = tokio::time::Instant::now();
let mut cache = None;
let refresh_generation = data_usage_snapshot_generation();
let first = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at);
let first = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at, refresh_generation)
.expect("an uninterrupted refresh should populate the cache");
assert!(matches!(first, Err(Error::ErasureReadQuorum)));
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
@@ -2200,13 +2497,16 @@ mod tests {
}
#[test]
#[serial]
fn cached_snapshot_success_is_reused_until_ttl_expires() {
let loaded_at = tokio::time::Instant::now();
let expected = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH);
let mut cache = None;
let refresh_generation = data_usage_snapshot_generation();
let first =
cache_data_usage_snapshot_result(&mut cache, Ok(expected), loaded_at).expect("successful load must be returned");
let first = cache_data_usage_snapshot_result(&mut cache, Ok(expected), loaded_at, refresh_generation)
.expect("an uninterrupted refresh should populate the cache")
.expect("successful load must be returned");
assert_snapshot_bucket(&first, "bucket");
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
@@ -2215,6 +2515,31 @@ mod tests {
assert_snapshot_bucket(&cached, "bucket");
}
#[test]
#[serial]
fn cache_invalidation_during_refresh_prevents_stale_snapshot_resurrection() {
let loaded_at = tokio::time::Instant::now();
let refresh_generation = data_usage_snapshot_generation();
let mut cache = Some(CachedDataUsageSnapshot {
info: Some(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
loaded_at,
});
clear_data_usage_snapshot_cache(&mut cache);
let stale_result = cache_data_usage_snapshot_result(
&mut cache,
Ok(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
loaded_at,
refresh_generation,
);
assert!(stale_result.is_none());
assert!(
cache.is_none(),
"an in-flight load must not repopulate a cache invalidated after it started"
);
}
#[tokio::test]
async fn resolve_snapshot_primary_ok_is_used_without_backup_read() {
let backup_read = Arc::new(std::sync::atomic::AtomicBool::new(false));
@@ -2312,7 +2637,7 @@ mod tests {
}
#[tokio::test]
async fn authoritative_usage_loader_migrates_only_when_v2_is_absent() {
async fn authoritative_usage_loader_prefers_v2_over_legacy_input() {
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
legacy_object: Some((snapshot_bytes("legacy-bucket"), 1)),
@@ -2320,18 +2645,20 @@ mod tests {
}),
});
let (legacy, _) = load_data_usage_snapshot_from_store(store.clone())
let (legacy, source) = load_data_usage_snapshot_from_store(store.clone())
.await
.expect("a legacy snapshot should seed an upgrade when v2 is absent");
.expect("a legacy snapshot should seed an upgrade when newer formats are absent");
assert_snapshot_bucket(&legacy, "legacy-bucket");
assert_eq!(source, UsageSnapshotSource::LegacyPrimary);
store.state.lock().await.object = Some((snapshot_bytes("v2-bucket"), 1));
let (authoritative, _) = load_data_usage_snapshot_from_store(store.clone())
store.state.lock().await.object = Some((snapshot_bytes("v2-bucket"), 2));
let (authoritative, source) = load_data_usage_snapshot_from_store(store.clone())
.await
.expect("a v2 snapshot should be authoritative");
.expect("a v2 snapshot should supersede the legacy migration input");
assert_snapshot_bucket(&authoritative, "v2-bucket");
assert_eq!(source, UsageSnapshotSource::Primary);
store.state.lock().await.object = Some((b"corrupt-v2".to_vec(), 2));
store.state.lock().await.object = Some((b"corrupt-v2".to_vec(), 3));
let err = load_data_usage_snapshot_from_store(store)
.await
.expect_err("corrupt v2 state must not fall back to a legacy writer");
@@ -2686,6 +3013,140 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn memory_overlay_does_not_publish_delta_from_unknown_baseline() {
clear_usage_memory_cache_for_test().await;
let bucket = "unknown-baseline";
memory_cache().write().await.insert(
bucket.to_string(),
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
);
record_bucket_object_write_memory(bucket, None, 42).await;
let mut response = DataUsageInfo::default();
apply_bucket_usage_memory_overlay(&mut response).await;
assert!(!response.buckets_usage.contains_key(bucket));
assert_eq!(get_bucket_usage_memory(bucket).await, None);
}
#[tokio::test]
#[serial]
async fn unknown_dirty_usage_requires_a_followup_scanner_generation() {
clear_usage_memory_cache_for_test().await;
let bucket = "unknown-baseline";
memory_cache().write().await.insert(
bucket.to_string(),
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
);
record_bucket_object_write_memory(bucket, None, 42).await;
let mutation_update = memory_cache()
.read()
.await
.get(bucket)
.expect("unknown usage mutation should remain cached")
.usage_updated_at;
let mut first_snapshot = data_usage_info_for_test(bucket, 10, 420, mutation_update + Duration::from_nanos(1));
first_snapshot.scanner_epoch = Some(7);
first_snapshot.scanner_cycle = Some(10);
replace_bucket_usage_memory_from_info(&first_snapshot).await;
let mut first_response = first_snapshot.clone();
apply_bucket_usage_memory_overlay_if_authoritative(&mut first_response, true).await;
assert!(
!first_response.usage_snapshot_complete,
"the first scanner generation after an unknown mutation cannot prove that it observed the mutation"
);
assert_eq!(
first_response
.buckets_usage
.get(bucket)
.map(|usage| (usage.objects_count, usage.size)),
Some((10, 420)),
"unknown memory deltas must not be overlaid as absolute usage"
);
replace_bucket_usage_memory_from_info(&first_snapshot).await;
let mut repeated_response = first_snapshot.clone();
apply_bucket_usage_memory_overlay_if_authoritative(&mut repeated_response, true).await;
assert!(
!repeated_response.usage_snapshot_complete,
"reloading the same scanner generation must not clear unknown mutation evidence"
);
let mut successor = data_usage_info_for_test(bucket, 11, 462, mutation_update + Duration::from_nanos(2));
successor.scanner_epoch = Some(7);
successor.scanner_cycle = Some(11);
replace_bucket_usage_memory_from_info(&successor).await;
let mut successor_response = successor;
apply_bucket_usage_memory_overlay_if_authoritative(&mut successor_response, true).await;
assert!(successor_response.is_complete_bucket_usage_snapshot());
assert_eq!(
successor_response
.buckets_usage
.get(bucket)
.map(|usage| (usage.objects_count, usage.size)),
Some((11, 462))
);
}
#[tokio::test]
#[serial]
async fn a_new_unknown_mutation_restarts_the_scanner_generation_fence() {
clear_usage_memory_cache_for_test().await;
let bucket = "unknown-baseline";
memory_cache().write().await.insert(
bucket.to_string(),
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
);
record_bucket_object_write_memory(bucket, None, 42).await;
let first_mutation_update = memory_cache()
.read()
.await
.get(bucket)
.expect("unknown usage mutation should remain cached")
.usage_updated_at;
let mut first_snapshot = data_usage_info_for_test(bucket, 10, 420, first_mutation_update + Duration::from_nanos(1));
first_snapshot.scanner_epoch = Some(7);
first_snapshot.scanner_cycle = Some(10);
replace_bucket_usage_memory_from_info(&first_snapshot).await;
record_bucket_object_write_memory(bucket, None, 42).await;
let second_mutation_update = memory_cache()
.read()
.await
.get(bucket)
.expect("second unknown usage mutation should remain cached")
.usage_updated_at;
let mut next_snapshot = data_usage_info_for_test(bucket, 11, 462, second_mutation_update + Duration::from_nanos(1));
next_snapshot.scanner_epoch = Some(7);
next_snapshot.scanner_cycle = Some(11);
replace_bucket_usage_memory_from_info(&next_snapshot).await;
let mut next_response = next_snapshot.clone();
apply_bucket_usage_memory_overlay_if_authoritative(&mut next_response, true).await;
assert!(
!next_response.usage_snapshot_complete,
"a mutation after the first observation must require another scanner generation"
);
let mut final_snapshot = data_usage_info_for_test(bucket, 12, 504, second_mutation_update + Duration::from_nanos(2));
final_snapshot.scanner_epoch = Some(7);
final_snapshot.scanner_cycle = Some(12);
replace_bucket_usage_memory_from_info(&final_snapshot).await;
let mut final_response = final_snapshot;
apply_bucket_usage_memory_overlay_if_authoritative(&mut final_response, true).await;
assert!(final_response.is_complete_bucket_usage_snapshot());
}
#[test]
fn remove_bucket_usage_from_info_drops_bucket_and_recomputes_totals() {
let last_update = SystemTime::now();
@@ -3109,7 +3570,7 @@ mod tests {
"fresh successor snapshot cache must not be erased after namespace lock loss"
);
drop(snapshot_cache);
*data_usage_snapshot_cache().write().await = None;
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test(start_paused = true)]
@@ -3168,7 +3629,7 @@ mod tests {
"fresh successor snapshot cache must not be erased by final invalidation after lock loss"
);
drop(snapshot_cache);
*data_usage_snapshot_cache().write().await = None;
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test]
@@ -3255,6 +3716,7 @@ mod tests {
},
);
legacy.bucket_sizes.insert("bucket-b".to_string(), 126);
legacy.usage_snapshot_complete = false;
legacy.buckets_count = 2;
legacy.calculate_totals();
let store = Arc::new(UsageCasStore {
@@ -3288,6 +3750,13 @@ mod tests {
Some((3, 126))
);
}
for (data, _) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() {
let saved = serde_json::from_slice::<DataUsageInfo>(data).expect("migrated usage snapshot should decode");
assert!(
!saved.usage_snapshot_complete,
"legacy usage must not be promoted to an authoritative snapshot"
);
}
}
#[tokio::test]
@@ -3428,6 +3897,7 @@ mod tests {
},
);
backup.bucket_sizes.insert("bucket-b".to_string(), 126);
backup.buckets_count = 2;
backup.calculate_totals();
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
crates/ecstore/src/diagnostics/admin_server_info.rs
+96 -11
View File
@@ -57,14 +57,14 @@ fn apply_data_usage_result(
usage: &mut rustfs_madmin::Usage,
) {
match result {
Ok(info) => {
Ok(info) if info.is_complete_bucket_usage_snapshot() => {
buckets.count = info.buckets_count;
objects.count = info.objects_total_count;
versions.count = info.versions_total_count;
delete_markers.count = info.delete_markers_total_count;
usage.size = info.objects_total_size;
}
Err(_) => {
Ok(_) | Err(_) => {
buckets.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
objects.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
versions.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
@@ -649,18 +649,15 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
if erasure_set.id == 0 {
erasure_set.id = d.set_index;
if let Ok(cache) = load_data_usage_cache(
match load_data_usage_cache(
&store.pools[d.pool_index as usize].disk_set[d.set_index as usize].clone(),
DATA_USAGE_CACHE_NAME,
)
.await
{
let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::<String>::new());
erasure_set.objects_count = data_usage_info.objects_total_count;
erasure_set.versions_count = data_usage_info.versions_total_count;
erasure_set.delete_markers_count = data_usage_info.delete_markers_total_count;
erasure_set.usage = data_usage_info.objects_total_size;
};
Ok(cache) => apply_erasure_set_usage(&cache, erasure_set),
Err(_) => erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
}
}
erasure_set.raw_capacity += d.total_space;
@@ -672,6 +669,20 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
Ok(pools_info)
}
fn apply_erasure_set_usage(cache: &rustfs_data_usage::DataUsageCache, erasure_set: &mut ErasureSetInfo) {
let data_usage_info = cache.dui(DATA_USAGE_ROOT, &[]);
if !data_usage_info.is_complete_bucket_usage_snapshot() {
erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
return;
}
erasure_set.objects_count = data_usage_info.objects_total_count;
erasure_set.versions_count = data_usage_info.versions_total_count;
erasure_set.delete_markers_count = data_usage_info.delete_markers_total_count;
erasure_set.usage = data_usage_info.objects_total_size;
erasure_set.usage_error = None;
}
#[allow(clippy::const_is_empty)]
pub fn get_commit_id() -> String {
let ver = if !build::TAG.is_empty() {
@@ -697,8 +708,9 @@ mod tests {
use rustfs_madmin::{Disk, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, ServerProperties};
use super::{
DATA_USAGE_UNAVAILABLE_ERROR, apply_data_usage_result, get_local_server_property, get_online_offline_disks_stats,
get_server_info, reconcile_servers_with_endpoint_topology, server_topology_completeness_report,
DATA_USAGE_ROOT, DATA_USAGE_UNAVAILABLE_ERROR, apply_data_usage_result, apply_erasure_set_usage,
get_local_server_property, get_online_offline_disks_stats, get_server_info, reconcile_servers_with_endpoint_topology,
server_topology_completeness_report,
};
fn disk_with_state(endpoint: &str, state: &str) -> Disk {
@@ -900,11 +912,19 @@ mod tests {
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
let mut usage = rustfs_madmin::Usage::default();
let info = rustfs_data_usage::DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
buckets_count: 2,
objects_total_count: 3,
versions_total_count: 4,
delete_markers_total_count: 5,
objects_total_size: 6,
buckets_usage: [
("bucket-a".to_string(), rustfs_data_usage::BucketUsageInfo::default()),
("bucket-b".to_string(), rustfs_data_usage::BucketUsageInfo::default()),
]
.into_iter()
.collect(),
usage_snapshot_complete: true,
..Default::default()
};
@@ -917,6 +937,71 @@ mod tests {
assert_eq!(usage.size, 6);
}
#[test]
fn incomplete_data_usage_is_unavailable_in_server_info() {
let mut buckets = rustfs_madmin::Buckets::default();
let mut objects = rustfs_madmin::Objects::default();
let mut versions = rustfs_madmin::Versions::default();
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
let mut usage = rustfs_madmin::Usage::default();
let info = rustfs_data_usage::DataUsageInfo {
buckets_count: 1,
objects_total_count: 100,
objects_total_size: 1024,
..Default::default()
};
apply_data_usage_result(Ok(info), &mut buckets, &mut objects, &mut versions, &mut delete_markers, &mut usage);
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(objects.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(versions.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(delete_markers.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(usage.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
}
#[test]
fn incomplete_erasure_set_cache_is_not_reported_as_zero() {
let mut cache = rustfs_data_usage::DataUsageCache::default();
cache.info.name = DATA_USAGE_ROOT.to_string();
cache.replace(DATA_USAGE_ROOT, "", rustfs_data_usage::DataUsageEntry::default());
let mut set = rustfs_madmin::ErasureSetInfo::default();
apply_erasure_set_usage(&cache, &mut set);
assert_eq!(set.usage_error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(set.objects_count, 0);
assert_eq!(set.usage, 0);
}
#[test]
fn complete_erasure_set_cache_is_reported() {
let mut cache = rustfs_data_usage::DataUsageCache::default();
cache.info.name = DATA_USAGE_ROOT.to_string();
cache.info.last_update = Some(std::time::SystemTime::UNIX_EPOCH);
cache.info.snapshot_complete = true;
cache.replace(
DATA_USAGE_ROOT,
"",
rustfs_data_usage::DataUsageEntry {
size: 512,
objects: 3,
versions: 4,
delete_markers: 1,
..Default::default()
},
);
let mut set = rustfs_madmin::ErasureSetInfo::default();
apply_erasure_set_usage(&cache, &mut set);
assert_eq!(set.objects_count, 3);
assert_eq!(set.versions_count, 4);
assert_eq!(set.delete_markers_count, 1);
assert_eq!(set.usage, 512);
assert!(set.usage_error.is_none());
}
#[serial]
#[tokio::test]
async fn server_info_includes_global_deployment_id() {
crates/ecstore/src/store/bucket.rs
+24 -13
View File
@@ -326,6 +326,13 @@ impl ECStore {
)
.await?;
if confirmed_missing && !is_meta_bucketname(bucket) {
// A scanner may have sampled the first fence before the bucket
// became visible. Fence again after metadata initialization so
// that snapshot cannot publish as complete.
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
Ok(())
}
@@ -545,6 +552,9 @@ impl ECStore {
"physical bucket deletion succeeded but metadata cleanup remains pending"
);
}
// A scanner may have sampled the first fence before the physical
// namespace disappeared. The completion fence invalidates that scan.
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
Ok(())
}
}
@@ -839,8 +849,8 @@ mod tests {
.expect("bucket should be created");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_make.saturating_add(1),
"successful bucket creation should advance scanner namespace activity"
generation_before_make.saturating_add(2),
"successful bucket creation should fence scanner namespace activity before and after creation"
);
let generation_before_put = ecstore.scanner_namespace_mutation_generation();
@@ -1057,8 +1067,8 @@ mod tests {
.expect("MarkDelete should remove an empty bucket");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_delete.saturating_add(1),
"successful bucket deletion should advance scanner namespace activity"
generation_before_delete.saturating_add(2),
"successful bucket deletion should fence scanner namespace activity before and after deletion"
);
assert!(
@@ -1179,8 +1189,8 @@ mod tests {
.expect("Purge should force-delete bucket data");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_delete.saturating_add(1),
"successful bucket purge should advance scanner namespace activity"
generation_before_delete.saturating_add(2),
"successful bucket purge should fence scanner namespace activity before and after deletion"
);
assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
@@ -1249,7 +1259,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn bucket_delete_finishes_usage_cleanup_before_same_name_recreation() {
async fn bucket_recreation_does_not_publish_unverified_usage() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple());
ecstore
@@ -1271,6 +1281,7 @@ mod tests {
..Default::default()
},
);
snapshot.usage_snapshot_complete = true;
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
@@ -1296,13 +1307,11 @@ mod tests {
.await
.expect("recreated bucket usage base should load");
crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await;
assert_eq!(
recreated
.buckets_usage
.get(&bucket)
.map(|usage| (usage.objects_count, usage.versions_count, usage.size)),
Some((1, 1, 84))
assert!(
!recreated.buckets_usage.contains_key(&bucket),
"a request-path delta without an authoritative baseline must remain unavailable"
);
assert_eq!(crate::data_usage::get_bucket_usage_memory(&bucket).await, None);
}
#[tokio::test]
@@ -1324,6 +1333,7 @@ mod tests {
..Default::default()
},
);
snapshot.usage_snapshot_complete = true;
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
@@ -1373,6 +1383,7 @@ mod tests {
..Default::default()
},
);
snapshot.usage_snapshot_complete = true;
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
crates/madmin/src/info_commands.rs
+36
View File
@@ -317,6 +317,8 @@ pub struct ErasureSetInfo {
pub versions_count: u64,
#[serde(rename = "deleteMarkersCount")]
pub delete_markers_count: u64,
#[serde(rename = "usageError", default, skip_serializing_if = "Option::is_none")]
pub usage_error: Option<String>,
#[serde(rename = "healDisks")]
pub heal_disks: i32,
}
@@ -1107,6 +1109,7 @@ mod tests {
assert_eq!(erasure_set.objects_count, 0);
assert_eq!(erasure_set.versions_count, 0);
assert_eq!(erasure_set.delete_markers_count, 0);
assert!(erasure_set.usage_error.is_none());
assert_eq!(erasure_set.heal_disks, 0);
}
@@ -1120,6 +1123,7 @@ mod tests {
objects_count: 10000,
versions_count: 15000,
delete_markers_count: 500,
usage_error: None,
heal_disks: 2,
};
@@ -1133,6 +1137,38 @@ mod tests {
assert_eq!(erasure_set.heal_disks, 2);
}
#[test]
fn erasure_set_usage_error_is_additive_on_the_wire() {
#[derive(Deserialize)]
struct LegacyErasureSetInfo {
id: i32,
usage: u64,
}
let legacy = r#"{"id":1,"rawUsage":2,"rawCapacity":3,"usage":4,"objectsCount":5,"versionsCount":6,"deleteMarkersCount":7,"healDisks":8}"#;
let decoded: ErasureSetInfo = serde_json::from_str(legacy).expect("legacy erasure set payload should decode");
assert!(decoded.usage_error.is_none());
let encoded = serde_json::to_value(decoded).expect("erasure set payload should encode");
assert!(
encoded.get("usageError").is_none(),
"an absent usage error must not change the legacy wire shape"
);
let current = ErasureSetInfo {
id: 9,
usage: 10,
usage_error: Some("data usage unavailable".to_string()),
..Default::default()
};
let encoded = serde_json::to_value(current).expect("current erasure set payload should encode");
let legacy: LegacyErasureSetInfo =
serde_json::from_value(encoded).expect("legacy erasure set reader should ignore additive fields");
assert_eq!(legacy.id, 9);
assert_eq!(legacy.usage, 10);
}
#[test]
fn test_backend_type_default() {
let backend_type = BackendType::default();
crates/obs/src/metrics/collectors/bucket.rs
+40 -22
View File
@@ -33,10 +33,10 @@ use std::borrow::Cow;
pub struct BucketStats {
/// Name of the bucket
pub name: String,
/// Total size of all objects in the bucket (bytes)
pub size_bytes: u64,
/// Number of objects in the bucket
pub objects_count: u64,
/// Total size of all objects in the bucket (bytes), when authoritative.
pub size_bytes: Option<u64>,
/// Number of objects in the bucket, when authoritative.
pub objects_count: Option<u64>,
/// Quota limit for the bucket (bytes), 0 if no quota
pub quota_bytes: u64,
}
@@ -54,15 +54,19 @@ pub fn collect_bucket_metrics(buckets: &[BucketStats]) -> Vec<PrometheusMetric>
for bucket in buckets {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone());
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, bucket.size_bytes as f64)
.with_label("bucket", bucket_label.clone()),
);
if let Some(size_bytes) = bucket.size_bytes {
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, size_bytes as f64)
.with_label("bucket", bucket_label.clone()),
);
}
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, bucket.objects_count as f64)
.with_label("bucket", bucket_label.clone()),
);
if let Some(objects_count) = bucket.objects_count {
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, objects_count as f64)
.with_label("bucket", bucket_label.clone()),
);
}
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_QUOTA_BYTES_MD, bucket.quota_bytes as f64)
@@ -83,14 +87,14 @@ mod tests {
let buckets = vec![
BucketStats {
name: "test-bucket".to_string(),
size_bytes: 1000,
objects_count: 100,
size_bytes: Some(1000),
objects_count: Some(100),
quota_bytes: 0,
},
BucketStats {
name: "another-bucket".to_string(),
size_bytes: 2000,
objects_count: 200,
size_bytes: Some(2000),
objects_count: Some(200),
quota_bytes: 0,
},
];
@@ -114,8 +118,8 @@ mod tests {
fn test_collect_bucket_metrics_with_quotas() {
let buckets = vec![BucketStats {
name: "quota-bucket".to_string(),
size_bytes: 500,
objects_count: 10,
size_bytes: Some(500),
objects_count: Some(10),
quota_bytes: 10000,
}];
@@ -146,8 +150,8 @@ mod tests {
fn test_collect_bucket_metrics_zero_quota_always_reported() {
let buckets = vec![BucketStats {
name: "no-quota-bucket".to_string(),
size_bytes: 100,
objects_count: 5,
size_bytes: Some(100),
objects_count: Some(5),
quota_bytes: 0,
}];
@@ -165,12 +169,26 @@ mod tests {
assert!(quota_metric.is_some());
}
#[test]
fn unknown_usage_only_reports_quota() {
let metrics = collect_bucket_metrics(&[BucketStats {
name: "unknown-usage".to_string(),
size_bytes: None,
objects_count: None,
quota_bytes: 4096,
}]);
assert_eq!(metrics.len(), 1);
assert_eq!(metrics[0].name, BUCKET_QUOTA_BYTES_MD.get_full_metric_name());
assert_eq!(metrics[0].value, 4096.0);
}
#[test]
fn test_bucket_stats_default() {
let stats = BucketStats::default();
assert!(stats.name.is_empty());
assert_eq!(stats.size_bytes, 0);
assert_eq!(stats.objects_count, 0);
assert_eq!(stats.size_bytes, None);
assert_eq!(stats.objects_count, None);
assert_eq!(stats.quota_bytes, 0);
}
}
crates/obs/src/metrics/collectors/cluster.rs
+36 -11
View File
@@ -43,9 +43,9 @@ pub struct ClusterStats {
/// Number of drives with no capacity observation
pub missing_capacity_drives: u64,
/// Total number of objects in the cluster
pub objects_count: u64,
pub objects_count: Option<u64>,
/// Total number of buckets in the cluster
pub buckets_count: u64,
pub buckets_count: Option<u64>,
}
/// Collects cluster-wide metrics from the provided cluster statistics.
@@ -53,16 +53,21 @@ pub struct ClusterStats {
/// Uses the metric descriptors from `metrics_type::cluster` module.
/// Returns a vector of Prometheus metrics for cluster statistics.
pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
vec![
let mut metrics = vec![
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD, stats.raw_capacity_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD, stats.usable_capacity_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USED_BYTES_MD, stats.used_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_FREE_BYTES_MD, stats.free_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_STALE_DRIVES_MD, stats.stale_capacity_drives as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_MISSING_DRIVES_MD, stats.missing_capacity_drives as f64),
PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, stats.objects_count as f64),
PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, stats.buckets_count as f64),
]
];
if let Some(objects_count) = stats.objects_count {
metrics.push(PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, objects_count as f64));
}
if let Some(buckets_count) = stats.buckets_count {
metrics.push(PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, buckets_count as f64));
}
metrics
}
#[cfg(test)]
@@ -79,8 +84,8 @@ mod tests {
free_bytes: 1300,
stale_capacity_drives: 1,
missing_capacity_drives: 0,
objects_count: 100,
buckets_count: 5,
objects_count: Some(100),
buckets_count: Some(5),
};
let metrics = collect_cluster_metrics(&stats);
@@ -116,7 +121,7 @@ mod tests {
let metrics = collect_cluster_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 8);
assert_eq!(metrics.len(), 6);
// All values should be zero
for metric in &metrics {
@@ -134,7 +139,27 @@ mod tests {
assert_eq!(stats.free_bytes, 0);
assert_eq!(stats.stale_capacity_drives, 0);
assert_eq!(stats.missing_capacity_drives, 0);
assert_eq!(stats.objects_count, 0);
assert_eq!(stats.buckets_count, 0);
assert_eq!(stats.objects_count, None);
assert_eq!(stats.buckets_count, None);
}
#[test]
fn unknown_cluster_cardinality_is_omitted() {
let metrics = collect_cluster_metrics(&ClusterStats {
objects_count: None,
buckets_count: None,
..Default::default()
});
assert!(
metrics
.iter()
.all(|metric| metric.name != CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name())
);
assert!(
metrics
.iter()
.all(|metric| metric.name != CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name())
);
}
}
crates/obs/src/metrics/scheduler.rs
+222 -17
View File
@@ -79,10 +79,13 @@ use crate::metrics::schema::audit::{AUDIT_FAILED_MESSAGES_MD, AUDIT_TARGET_QUEUE
use crate::metrics::schema::bucket_replication::{
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, TARGET_ARN_L,
};
use crate::metrics::schema::cluster::{CLUSTER_BUCKETS_TOTAL_MD, CLUSTER_OBJECTS_TOTAL_MD};
use crate::metrics::schema::cluster_usage::{
BUCKET_LABEL as USAGE_BUCKET_LABEL, RANGE_LABEL as USAGE_RANGE_LABEL, USAGE_BUCKET_DELETE_MARKERS_COUNT_MD,
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, USAGE_BUCKET_OBJECTS_TOTAL_MD,
USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, USAGE_BUCKET_TOTAL_BYTES_MD, USAGE_BUCKET_VERSIONS_COUNT_MD,
USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, USAGE_BUCKET_TOTAL_BYTES_MD, USAGE_BUCKET_VERSIONS_COUNT_MD, USAGE_BUCKETS_COUNT_MD,
USAGE_DELETE_MARKERS_COUNT_MD, USAGE_OBJECTS_COUNT_MD, USAGE_OBJECTS_DISTRIBUTION_MD, USAGE_SINCE_LAST_UPDATE_SECONDS_MD,
USAGE_TOTAL_BYTES_MD, USAGE_VERSIONS_COUNT_MD, USAGE_VERSIONS_DISTRIBUTION_MD,
};
use crate::metrics::schema::node_bucket::{BUCKET_OBJECTS_TOTAL_MD, BUCKET_QUOTA_BYTES_MD, BUCKET_USAGE_BYTES_MD};
use crate::metrics::schema::notification_target::{
@@ -659,6 +662,26 @@ fn bucket_live_keys(stats: &[crate::metrics::collectors::BucketStats]) -> HashSe
stats.iter().map(|stat| stat.name.clone()).collect()
}
fn bucket_observation_live_keys(stats: &[crate::metrics::collectors::BucketStats]) -> HashSet<BucketKey> {
stats
.iter()
.filter(|stat| stat.size_bytes.is_some() || stat.objects_count.is_some())
.map(|stat| stat.name.clone())
.collect()
}
fn bucket_observation_retire_keys(
previous_observations: &HashSet<BucketKey>,
current_buckets: &HashSet<BucketKey>,
current_observations: &HashSet<BucketKey>,
) -> Vec<BucketKey> {
previous_observations
.difference(current_observations)
.filter(|bucket| current_buckets.contains(*bucket))
.cloned()
.collect()
}
fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8>) -> Vec<PrometheusMetric> {
if zero_tombstones.is_empty() {
return Vec::new();
@@ -677,6 +700,50 @@ fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8
zero_metrics
}
#[derive(Default)]
struct BucketSeriesState {
has_seen_snapshot: bool,
live_keys: HashSet<BucketKey>,
observation_keys: HashSet<BucketKey>,
zero_tombstones: HashMap<BucketKey, u8>,
}
struct BucketSeriesUpdate {
metrics: Vec<PrometheusMetric>,
retire_observations: Vec<BucketKey>,
retire_buckets: Vec<BucketKey>,
}
impl BucketSeriesState {
fn observe(
&mut self,
stats: Option<&[crate::metrics::collectors::BucketStats]>,
tombstone_cycles: u8,
) -> Option<BucketSeriesUpdate> {
let stats = stats?;
let current_bucket_keys = bucket_live_keys(stats);
let current_observation_keys = bucket_observation_live_keys(stats);
let retire_observations =
bucket_observation_retire_keys(&self.observation_keys, &current_bucket_keys, &current_observation_keys);
self.observation_keys = current_observation_keys;
update_series_zero_tombstones(
&mut self.has_seen_snapshot,
&mut self.live_keys,
&mut self.zero_tombstones,
current_bucket_keys,
tombstone_cycles,
);
let mut metrics = collect_bucket_metrics(stats);
metrics.extend(collect_bucket_zero_tombstone_metrics(&self.zero_tombstones));
let retire_buckets = expire_series_zero_tombstones(&mut self.zero_tombstones);
Some(BucketSeriesUpdate {
metrics,
retire_observations,
retire_buckets,
})
}
}
fn retire_bucket_metric_series(bucket: &str) -> usize {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.to_string());
let labels = [("bucket", bucket_label.clone())];
@@ -685,6 +752,32 @@ fn retire_bucket_metric_series(bucket: &str) -> usize {
+ retire_metric_series(&BUCKET_QUOTA_BYTES_MD.get_full_metric_name(), &labels)
}
fn retire_bucket_observation_metric_series(bucket: &str) -> usize {
let labels = [("bucket", Cow::Owned(bucket.to_string()))];
retire_metric_series(&BUCKET_USAGE_BYTES_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels)
}
fn retire_cluster_usage_metric_series() -> usize {
let labels: [(&'static str, Cow<'static, str>); 0] = [];
[
USAGE_SINCE_LAST_UPDATE_SECONDS_MD.get_full_metric_name(),
USAGE_TOTAL_BYTES_MD.get_full_metric_name(),
USAGE_OBJECTS_COUNT_MD.get_full_metric_name(),
USAGE_VERSIONS_COUNT_MD.get_full_metric_name(),
USAGE_DELETE_MARKERS_COUNT_MD.get_full_metric_name(),
USAGE_BUCKETS_COUNT_MD.get_full_metric_name(),
]
.iter()
.map(|name| retire_metric_series(name, &labels))
.sum()
}
fn retire_cluster_usage_distribution_series(metric_name: String, range: &str) -> usize {
let labels = [(USAGE_RANGE_LABEL, Cow::Owned(range.to_string()))];
retire_metric_series(&metric_name, &labels)
}
fn bucket_usage_live_keys(stats: &[crate::metrics::collectors::BucketUsageStats]) -> HashSet<BucketKey> {
stats.iter().map(|stat| stat.bucket.clone()).collect()
}
@@ -973,11 +1066,23 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let token_clone = token.clone();
tokio::spawn(async move {
let mut interval = metrics_interval(cluster_interval, Duration::ZERO);
let mut objects_count_was_authoritative = false;
let mut buckets_count_was_authoritative = false;
loop {
tokio::select! {
_ = interval.tick() => {
run_metrics_collector_tick(health, MetricsCollectorTaskId::ClusterStats, "cluster_stats", async {
let (stats, cluster_health) = collect_cluster_and_health_stats().await;
if objects_count_was_authoritative && stats.objects_count.is_none() {
let labels: [(&'static str, Cow<'static, str>); 0] = [];
let _ = retire_metric_series(&CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels);
}
if buckets_count_was_authoritative && stats.buckets_count.is_none() {
let labels: [(&'static str, Cow<'static, str>); 0] = [];
let _ = retire_metric_series(&CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name(), &labels);
}
objects_count_was_authoritative = stats.objects_count.is_some();
buckets_count_was_authoritative = stats.buckets_count.is_some();
let mut metrics = collect_cluster_metrics(&stats);
metrics.extend(collect_cluster_health_metrics(&cluster_health));
report_metrics(&metrics);
@@ -1004,6 +1109,8 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let mut bucket_usage_object_size_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
let mut prev_bucket_usage_version_keys: HashSet<BucketRangeKey> = HashSet::new();
let mut bucket_usage_version_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
let mut prev_cluster_usage_object_size_keys: HashSet<String> = HashSet::new();
let mut prev_cluster_usage_version_keys: HashSet<String> = HashSet::new();
loop {
tokio::select! {
_ = interval.tick() => {
@@ -1028,6 +1135,30 @@ pub fn init_metrics_runtime(token: CancellationToken) {
}
if let Some((cluster_usage, bucket_usage)) = collect_cluster_usage_metric_stats().await {
let current_cluster_usage_object_size_keys = cluster_usage
.object_size_distribution
.iter()
.map(|(range, _)| range.clone())
.collect::<HashSet<_>>();
for range in prev_cluster_usage_object_size_keys.difference(&current_cluster_usage_object_size_keys) {
let _ = retire_cluster_usage_distribution_series(
USAGE_OBJECTS_DISTRIBUTION_MD.get_full_metric_name(),
range,
);
}
prev_cluster_usage_object_size_keys = current_cluster_usage_object_size_keys;
let current_cluster_usage_version_keys = cluster_usage
.versions_distribution
.iter()
.map(|(range, _)| range.clone())
.collect::<HashSet<_>>();
for range in prev_cluster_usage_version_keys.difference(&current_cluster_usage_version_keys) {
let _ = retire_cluster_usage_distribution_series(
USAGE_VERSIONS_DISTRIBUTION_MD.get_full_metric_name(),
range,
);
}
prev_cluster_usage_version_keys = current_cluster_usage_version_keys;
metrics.extend(collect_cluster_usage_metrics(&cluster_usage));
update_series_zero_tombstones(
&mut has_seen_bucket_usage_snapshot,
@@ -1073,6 +1204,41 @@ pub fn init_metrics_runtime(token: CancellationToken) {
&range,
);
}
} else if has_seen_bucket_usage_snapshot {
let _ = retire_cluster_usage_metric_series();
for range in prev_cluster_usage_object_size_keys.drain() {
let _ = retire_cluster_usage_distribution_series(
USAGE_OBJECTS_DISTRIBUTION_MD.get_full_metric_name(),
&range,
);
}
for range in prev_cluster_usage_version_keys.drain() {
let _ = retire_cluster_usage_distribution_series(
USAGE_VERSIONS_DISTRIBUTION_MD.get_full_metric_name(),
&range,
);
}
for bucket in prev_bucket_usage_keys.drain() {
let _ = retire_bucket_usage_metric_series(&bucket);
}
for (bucket, range) in prev_bucket_usage_object_size_keys.drain() {
let _ = retire_bucket_usage_distribution_series(
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD.get_full_metric_name(),
&bucket,
&range,
);
}
for (bucket, range) in prev_bucket_usage_version_keys.drain() {
let _ = retire_bucket_usage_distribution_series(
USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD.get_full_metric_name(),
&bucket,
&range,
);
}
bucket_usage_zero_tombstones.clear();
bucket_usage_object_size_zero_tombstones.clear();
bucket_usage_version_zero_tombstones.clear();
has_seen_bucket_usage_snapshot = false;
}
if !metrics.is_empty() {
@@ -1094,25 +1260,20 @@ pub fn init_metrics_runtime(token: CancellationToken) {
tokio::spawn(async move {
let mut interval = metrics_interval(bucket_interval, Duration::ZERO);
let tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
let mut has_seen_bucket_snapshot = false;
let mut prev_bucket_keys: HashSet<BucketKey> = HashSet::new();
let mut bucket_zero_tombstones: HashMap<BucketKey, u8> = HashMap::new();
let mut series_state = BucketSeriesState::default();
loop {
tokio::select! {
_ = interval.tick() => {
run_metrics_collector_tick(health, MetricsCollectorTaskId::BucketStats, "bucket_stats", async {
let stats = collect_bucket_stats().await;
update_series_zero_tombstones(
&mut has_seen_bucket_snapshot,
&mut prev_bucket_keys,
&mut bucket_zero_tombstones,
bucket_live_keys(&stats),
tombstone_cycles,
);
let mut metrics = collect_bucket_metrics(&stats);
metrics.extend(collect_bucket_zero_tombstone_metrics(&bucket_zero_tombstones));
report_metrics(&metrics);
for bucket in expire_series_zero_tombstones(&mut bucket_zero_tombstones) {
let Some(update) = series_state.observe(stats.as_deref(), tombstone_cycles) else {
return;
};
for bucket in update.retire_observations {
let _ = retire_bucket_observation_metric_series(&bucket);
}
report_metrics(&update.metrics);
for bucket in update.retire_buckets {
let _ = retire_bucket_metric_series(&bucket);
}
}).await;
@@ -1850,8 +2011,8 @@ mod tests {
let mut zero_tombstones = HashMap::new();
let live_stats = vec![crate::metrics::collectors::BucketStats {
name: "tmp".to_string(),
size_bytes: 1024,
objects_count: 8,
size_bytes: Some(1024),
objects_count: Some(8),
quota_bytes: 2048,
}];
@@ -1885,6 +2046,50 @@ mod tests {
assert!(zero_tombstones.is_empty());
}
#[test]
fn bucket_observation_retirement_distinguishes_unknown_usage_from_deletion() {
let previous = HashSet::from(["bucket".to_string()]);
let unknown_stats = vec![crate::metrics::collectors::BucketStats {
name: "bucket".to_string(),
size_bytes: None,
objects_count: None,
quota_bytes: 1024,
}];
let current_buckets = bucket_live_keys(&unknown_stats);
let current_observations = bucket_observation_live_keys(&unknown_stats);
assert_eq!(
bucket_observation_retire_keys(&previous, &current_buckets, &current_observations),
vec!["bucket".to_string()],
"an existing bucket with unknown usage must retire its previous usage observations"
);
assert!(
bucket_observation_retire_keys(&previous, &HashSet::new(), &HashSet::new()).is_empty(),
"a deleted bucket remains governed by the zero-tombstone lifecycle"
);
}
#[test]
fn unavailable_bucket_snapshot_preserves_metric_series_state() {
let mut state = BucketSeriesState::default();
let initial = [crate::metrics::collectors::BucketStats {
name: "bucket".to_string(),
size_bytes: Some(512),
objects_count: Some(2),
quota_bytes: 1024,
}];
assert!(state.observe(Some(&initial), 2).is_some());
let live_keys = state.live_keys.clone();
let observation_keys = state.observation_keys.clone();
let zero_tombstones = state.zero_tombstones.clone();
assert!(state.observe(None, 2).is_none());
assert_eq!(state.live_keys, live_keys);
assert_eq!(state.observation_keys, observation_keys);
assert_eq!(state.zero_tombstones, zero_tombstones);
}
#[test]
fn parse_system_metrics_interval_rounds_legacy_millis_up_to_one_second() {
temp_env::with_vars(
crates/obs/src/metrics/stats_collector.rs
+124 -29
View File
@@ -41,7 +41,11 @@ use rustfs_io_metrics::{
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_resource_and_system,
snapshot_process_resource_and_system_with,
};
use std::{collections::HashMap, sync::Arc, time::SystemTime};
use std::{
collections::{HashMap, HashSet},
sync::Arc,
time::SystemTime,
};
use sysinfo::{Networks, System};
use tracing::{instrument, warn};
@@ -52,8 +56,10 @@ const EVENT_METRICS_COLLECTOR_STATE: &str = "metrics_collector_state";
type ObsStorageInfo = <ObsStore as StorageAdminApi>::StorageInfo;
type ObsBackendInfo = <ObsStore as StorageAdminApi>::BackendInfo;
#[derive(Default)]
struct ObsDataUsageInfo {
last_update: Option<SystemTime>,
usage_snapshot_complete: bool,
buckets_count: u64,
objects_total_count: u64,
versions_total_count: u64,
@@ -62,6 +68,7 @@ struct ObsDataUsageInfo {
buckets_usage: HashMap<String, ObsBucketUsageInfo>,
}
#[derive(Default)]
struct ObsBucketUsageInfo {
size: u64,
objects_count: u64,
@@ -85,9 +92,11 @@ fn usize_to_u64_saturating(value: usize) -> u64 {
async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreResult<ObsDataUsageInfo> {
let data_usage = obs_load_data_usage_from_backend(store).await?;
let usage_snapshot_complete = data_usage.is_complete_bucket_usage_snapshot();
Ok(ObsDataUsageInfo {
last_update: data_usage.last_update,
usage_snapshot_complete,
buckets_count: data_usage.buckets_count,
objects_total_count: data_usage.objects_total_count,
versions_total_count: data_usage.versions_total_count,
@@ -113,6 +122,21 @@ async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreRes
})
}
fn bucket_usage_metric_values(data_usage: Option<&ObsDataUsageInfo>, bucket: &str) -> (Option<u64>, Option<u64>) {
data_usage
.filter(|usage| usage.usage_snapshot_complete)
.and_then(|usage| usage.buckets_usage.get(bucket))
.map(|usage| (Some(usage.size), Some(usage.objects_count)))
.unwrap_or((None, None))
}
fn data_usage_snapshot_covers_bucket_namespace(data_usage: &ObsDataUsageInfo, buckets: &HashSet<String>) -> bool {
data_usage.usage_snapshot_complete
&& u64::try_from(buckets.len()).ok() == Some(data_usage.buckets_count)
&& buckets.len() == data_usage.buckets_usage.len()
&& buckets.iter().all(|bucket| data_usage.buckets_usage.contains_key(bucket))
}
fn resolve_obs_object_store_handle() -> Option<Arc<ObsStore>> {
obs_resolve_object_store_handle()
}
@@ -365,25 +389,16 @@ pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthS
})
.count() as u64;
// Get bucket and object counts from data usage info.
let (buckets_count, objects_count) = match load_obs_data_usage_from_backend(store.clone()).await {
Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
Err(e) => {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "data_usage_load_failed", error = %e, "metrics collector state changed");
// Fall back to bucket list for buckets_count, objects_count stays 0.
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
.unwrap_or_else(|err| {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "bucket_list_failed", error = %err, "metrics collector state changed");
Vec::new()
});
(buckets.len() as u64, 0)
let data_usage = match load_obs_data_usage_from_backend(store).await {
Ok(data_usage) if data_usage.usage_snapshot_complete => Some(data_usage),
Ok(_) => None,
Err(error) => {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "data_usage_load_failed", error = %error, "metrics collector state changed");
None
}
};
let buckets_count = data_usage.as_ref().map(|usage| usage.buckets_count);
let objects_count = data_usage.as_ref().map(|usage| usage.objects_total_count);
let mut online = 0u64;
let mut offline = 0u64;
@@ -428,10 +443,12 @@ pub async fn collect_cluster_health_stats() -> ClusterHealthStats {
}
/// Collect bucket statistics from the storage layer.
pub async fn collect_bucket_stats() -> Vec<BucketStats> {
let Some(store) = resolve_obs_object_store_handle() else {
return Vec::new();
};
///
/// `None` means the bucket namespace could not be observed. Callers must keep
/// their prior metric-series state instead of treating that failure as an
/// authoritative empty namespace.
pub async fn collect_bucket_stats() -> Option<Vec<BucketStats>> {
let store = resolve_obs_object_store_handle()?;
// Load data usage info from backend to get bucket sizes and object counts
let data_usage = match load_obs_data_usage_from_backend(store.clone()).await {
@@ -453,7 +470,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
Ok(buckets) => buckets,
Err(e) => {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "bucket_stats", result = "bucket_list_failed", error = %e, "metrics collector state changed");
return Vec::new();
return None;
}
};
@@ -465,11 +482,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
}
// Get size and objects_count from data usage info
let (size_bytes, objects_count) = data_usage
.as_ref()
.and_then(|du| du.buckets_usage.get(&bucket.name))
.map(|bui| (bui.size, bui.objects_count))
.unwrap_or((0, 0));
let (size_bytes, objects_count) = bucket_usage_metric_values(data_usage.as_ref(), &bucket.name);
// Get quota from bucket metadata
let quota_bytes = obs_bucket_quota_limit_bytes(&bucket.name).await;
@@ -482,7 +495,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
});
}
stats
Some(stats)
}
/// Collect bucket replication bandwidth stats from the global monitor.
@@ -935,6 +948,29 @@ pub async fn collect_iam_stats() -> Option<IamStats> {
pub async fn collect_cluster_usage_metric_stats() -> Option<(ClusterUsageStats, Vec<BucketUsageStats>)> {
let store = resolve_obs_object_store_handle()?;
let data_usage = load_obs_data_usage_from_backend(store.clone()).await.ok()?;
let bucket_namespace = store
.list_bucket(&BucketOptions {
cached: true,
no_metadata: true,
..Default::default()
})
.await
.ok()?
.into_iter()
.filter(|bucket| !bucket.name.starts_with('.'))
.map(|bucket| bucket.name)
.collect::<HashSet<_>>();
collect_cluster_usage_metric_stats_from_data_usage(data_usage, &bucket_namespace).await
}
async fn collect_cluster_usage_metric_stats_from_data_usage(
data_usage: ObsDataUsageInfo,
bucket_namespace: &HashSet<String>,
) -> Option<(ClusterUsageStats, Vec<BucketUsageStats>)> {
if !data_usage_snapshot_covers_bucket_namespace(&data_usage, bucket_namespace) {
return None;
}
let mut buckets = Vec::with_capacity(data_usage.buckets_usage.len());
for (bucket_name, usage) in &data_usage.buckets_usage {
@@ -1187,6 +1223,65 @@ mod tests {
info
}
#[test]
fn bucket_usage_metrics_distinguish_unknown_from_confirmed_zero() {
assert_eq!(bucket_usage_metric_values(None, "bucket"), (None, None));
let mut data_usage = ObsDataUsageInfo {
usage_snapshot_complete: true,
..Default::default()
};
data_usage
.buckets_usage
.insert("bucket".to_string(), ObsBucketUsageInfo::default());
assert_eq!(bucket_usage_metric_values(Some(&data_usage), "bucket"), (Some(0), Some(0)));
assert_eq!(bucket_usage_metric_values(Some(&data_usage), "missing"), (None, None));
}
#[tokio::test]
async fn cluster_usage_metrics_skip_incomplete_snapshot() {
assert!(
collect_cluster_usage_metric_stats_from_data_usage(ObsDataUsageInfo::default(), &HashSet::new())
.await
.is_none()
);
}
#[tokio::test]
async fn cluster_usage_metrics_publish_complete_empty_snapshot() {
let (cluster, buckets) = collect_cluster_usage_metric_stats_from_data_usage(
ObsDataUsageInfo {
usage_snapshot_complete: true,
..Default::default()
},
&HashSet::new(),
)
.await
.expect("complete empty usage should remain publishable");
assert_eq!(cluster.buckets_count, 0);
assert_eq!(cluster.objects_count, 0);
assert_eq!(cluster.total_bytes, 0);
assert!(buckets.is_empty());
}
#[tokio::test]
async fn cluster_usage_metrics_skip_snapshot_for_a_different_bucket_namespace() {
let data_usage = ObsDataUsageInfo {
usage_snapshot_complete: true,
buckets_count: 1,
buckets_usage: HashMap::from([("stale-bucket".to_string(), ObsBucketUsageInfo::default())]),
..Default::default()
};
assert!(
collect_cluster_usage_metric_stats_from_data_usage(data_usage, &HashSet::from(["live-bucket".to_string()]))
.await
.is_none()
);
}
#[test]
fn cluster_config_stats_accept_homogeneous_backend_parities() {
let stats = cluster_config_stats_from_backend_parities(Some(1), Some(2))
crates/scanner/src/lib.rs
+8 -4
View File
@@ -35,10 +35,10 @@ use storage_api::owner::{
EcstoreTierConfig, EcstoreVersioningApi, HTTPPreconditions, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete,
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_data_usage_snapshot_cache,
ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
@@ -462,6 +462,10 @@ pub(crate) async fn replace_bucket_usage_memory_from_info(data_usage_info: &rust
ecstore_replace_bucket_usage_memory_from_info(data_usage_info).await;
}
pub(crate) async fn invalidate_data_usage_snapshot_cache() {
ecstore_invalidate_data_usage_snapshot_cache().await;
}
pub trait ScannerObjectIO:
ObjectIO<
Error = EcstoreError,
crates/scanner/src/scanner.rs
+135 -90
View File
@@ -61,8 +61,9 @@ use crate::storage_api::scan::{
};
use crate::{
ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _,
get_lifecycle_config, get_replication_config, read_config, replace_bucket_usage_memory_from_info, save_config,
save_config_shared_with_preconditions, save_config_with_preconditions, scanner_is_erasure_sd,
get_lifecycle_config, get_replication_config, invalidate_data_usage_snapshot_cache, read_config,
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions,
scanner_is_erasure_sd,
};
const LOG_COMPONENT_SCANNER: &str = "scanner";
@@ -985,7 +986,7 @@ fn initial_scanner_delay_for_startup(
}
fn data_usage_info_is_cold(info: &DataUsageInfo) -> bool {
info.last_update.is_none() || (info.buckets_usage.is_empty() && info.bucket_sizes.is_empty())
!info.is_complete_bucket_usage_snapshot()
}
async fn read_data_usage_config_for_startup(storeapi: &Arc<impl ScannerObjectIO>) -> Result<Option<Vec<u8>>, EcstoreError> {
@@ -1004,10 +1005,12 @@ async fn read_data_usage_config_for_startup(storeapi: &Arc<impl ScannerObjectIO>
}
}
match read_pair(storeapi, DATA_USAGE_OBJ_NAME_PATH.as_str()).await? {
Some(data) => Ok(Some(data)),
None => read_pair(storeapi, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()).await,
for path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] {
if let Some(data) = read_pair(storeapi, path).await? {
return Ok(Some(data));
}
}
Ok(None)
}
fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool {
@@ -3611,6 +3614,21 @@ async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
data_usage_info.scanner_epoch = Some(leader_epoch);
}
if !data_usage_info.is_complete_bucket_usage_snapshot() {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "reject_incomplete_snapshot",
"Scanner refused to persist an incomplete data usage snapshot"
);
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed);
outcome = DataUsagePersistOutcome::Failed;
continue;
}
let data = match serde_json::to_vec(&data_usage_info) {
Ok(data) => data,
Err(e) => {
@@ -3767,16 +3785,19 @@ async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
match save_outcome {
DataUsagePersistOutcome::Current => {
invalidate_data_usage_snapshot_cache().await;
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::SkippedStale);
outcome = DataUsagePersistOutcome::Current;
continue;
}
DataUsagePersistOutcome::AlreadyDurable => {
invalidate_data_usage_snapshot_cache().await;
replace_bucket_usage_memory_from_info(&data_usage_info).await;
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
outcome = DataUsagePersistOutcome::AlreadyDurable;
}
DataUsagePersistOutcome::PriorCycleDurable => {
invalidate_data_usage_snapshot_cache().await;
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
outcome = DataUsagePersistOutcome::PriorCycleDurable;
}
@@ -3786,6 +3807,7 @@ async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
continue;
}
DataUsagePersistOutcome::Saved => {
invalidate_data_usage_snapshot_cache().await;
replace_bucket_usage_memory_from_info(&data_usage_info).await;
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
outcome = DataUsagePersistOutcome::Saved;
@@ -4485,8 +4507,25 @@ mod tests {
assert_eq!(epoch, 11);
}
#[test]
fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() {
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::now()), 1);
legacy.usage_snapshot_complete = false;
assert!(data_usage_info_is_cold(&legacy));
assert!(!data_usage_info_is_cold(&complete_usage_with_bucket_count(
Some(std::time::SystemTime::now()),
1,
)));
assert!(!data_usage_info_is_cold(&DataUsageInfo {
last_update: Some(std::time::SystemTime::now()),
usage_snapshot_complete: true,
..Default::default()
}));
}
#[tokio::test]
async fn scanner_startup_migrates_legacy_usage_only_until_v2_exists() {
async fn scanner_startup_prefers_v2_over_legacy_usage() {
let store = Arc::new(MemoryConfigStore::default());
let legacy = DataUsageInfo {
scanner_epoch: Some(19),
@@ -4517,9 +4556,10 @@ mod tests {
);
let authoritative = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(11),
scanner_epoch: Some(23),
scanner_cycle: Some(51),
last_update: Some(std::time::SystemTime::now()),
usage_snapshot_complete: true,
..Default::default()
};
let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode");
@@ -4539,8 +4579,8 @@ mod tests {
.await
.expect("v2 usage floor should be authoritative"),
PersistedUsageFloor {
next_cycle: 12,
leader_epoch: 7,
next_cycle: 52,
leader_epoch: 23,
}
);
@@ -4594,7 +4634,7 @@ mod tests {
scanner_epoch: Some(1),
scanner_cycle: Some(cycle),
last_update: Some(std::time::SystemTime::now()),
..Default::default()
..complete_usage_with_bucket_count(None, 0)
})
.await
.expect("usage update should queue");
@@ -4959,16 +4999,8 @@ mod tests {
let (sender, receiver) = mpsc::channel(2);
let ctx = CancellationToken::new();
let newer = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
buckets_count: 2,
..Default::default()
};
let older = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
buckets_count: 1,
..Default::default()
};
let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
let older = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
sender.send(newer).await.expect("newer usage snapshot should enqueue");
sender.send(older).await.expect("older usage snapshot should enqueue");
@@ -4993,16 +5025,8 @@ mod tests {
let (sender, receiver) = mpsc::channel(1);
let ctx = CancellationToken::new();
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let newer = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
buckets_count: 2,
..Default::default()
};
let stale = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
buckets_count: 1,
..Default::default()
};
let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
let stale = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
store
.interleaving_puts
.lock()
@@ -5045,6 +5069,7 @@ mod tests {
initial.bucket_sizes.insert("bucket-a".to_string(), 84);
initial.buckets_count = 1;
initial.calculate_totals();
mark_usage_snapshot_complete(&mut initial);
let initial_data = serde_json::to_vec(&initial).expect("initial usage snapshot should encode");
store.objects.lock().await.insert(key.clone(), initial_data.clone());
store.revisions.lock().await.insert(key.clone(), 1);
@@ -5054,6 +5079,7 @@ mod tests {
deleted.bucket_sizes.clear();
deleted.buckets_count = 0;
deleted.calculate_totals();
mark_usage_snapshot_complete(&mut deleted);
store
.interleaving_puts
.lock()
@@ -5102,7 +5128,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
scanner_epoch: Some(8),
scanner_cycle: Some(1),
..Default::default()
..complete_usage_with_bucket_count(None, 0)
};
store.objects.lock().await.insert(
backup_key.clone(),
@@ -5129,6 +5155,7 @@ mod tests {
incoming.bucket_sizes.insert("bucket-a".to_string(), 84);
incoming.buckets_count = 1;
incoming.calculate_totals();
mark_usage_snapshot_complete(&mut incoming);
sender.send(incoming).await.expect("usage snapshot should enqueue");
}
drop(sender);
@@ -5162,7 +5189,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
scanner_epoch: Some(8),
scanner_cycle: Some(10),
..Default::default()
..complete_usage_with_bucket_count(None, 0)
};
let encoded = serde_json::to_vec(&durable).expect("usage snapshot should encode");
store.objects.lock().await.insert(main_key.clone(), encoded.clone());
@@ -5205,6 +5232,7 @@ mod tests {
incoming.bucket_sizes.insert("bucket-a".to_string(), 84);
incoming.buckets_count = 1;
incoming.calculate_totals();
mark_usage_snapshot_complete(&mut incoming);
let mut deleted = incoming.clone();
deleted.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(31));
@@ -5212,6 +5240,7 @@ mod tests {
deleted.bucket_sizes.clear();
deleted.buckets_count = 0;
deleted.calculate_totals();
mark_usage_snapshot_complete(&mut deleted);
store.replace_after_successful_puts.lock().await.insert(
main_key.clone(),
(1, serde_json::to_vec(&deleted).expect("deleted primary snapshot should encode")),
@@ -5251,21 +5280,9 @@ mod tests {
let (sender, receiver) = mpsc::channel(1);
let ctx = CancellationToken::new();
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let initial = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
buckets_count: 1,
..Default::default()
};
let stale_winner = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
buckets_count: 2,
..Default::default()
};
let current = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
buckets_count: 3,
..Default::default()
};
let initial = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 3);
let stale_winner = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 3);
let current = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), 3);
store
.objects
.lock()
@@ -5298,21 +5315,13 @@ mod tests {
}
#[tokio::test]
async fn test_store_data_usage_in_backend_rejects_untimestamped_stale_snapshot() {
async fn test_store_data_usage_in_backend_rejects_untimestamped_complete_snapshot() {
let store = Arc::new(MemoryConfigStore::default());
let (sender, receiver) = mpsc::channel(2);
let ctx = CancellationToken::new();
let timestamped = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
buckets_count: 2,
..Default::default()
};
let untimestamped = DataUsageInfo {
last_update: None,
buckets_count: 1,
..Default::default()
};
let timestamped = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
let untimestamped = complete_usage_with_bucket_count(None, 1);
sender
.send(timestamped)
@@ -5334,7 +5343,7 @@ mod tests {
assert_eq!(saved.buckets_count, 2);
assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)));
assert_eq!(outcome, DataUsagePersistOutcome::Current);
assert_eq!(outcome, DataUsagePersistOutcome::Failed);
}
#[tokio::test]
@@ -5346,8 +5355,7 @@ mod tests {
let snapshot = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
scanner_cycle: Some(12),
buckets_count: 2,
..Default::default()
..complete_usage_with_bucket_count(None, 2)
};
store
.objects
@@ -5377,8 +5385,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
scanner_epoch: Some(8),
scanner_cycle: Some(12),
buckets_count: 2,
..Default::default()
..complete_usage_with_bucket_count(None, 2)
};
store
.objects
@@ -5392,8 +5399,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
scanner_epoch: Some(8),
scanner_cycle: Some(12),
buckets_count: 3,
..Default::default()
..complete_usage_with_bucket_count(None, 3)
})
.await
.expect("changed retry snapshot should enqueue");
@@ -5426,8 +5432,7 @@ mod tests {
let existing = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)),
scanner_cycle: Some(12),
buckets_count: 2,
..Default::default()
..complete_usage_with_bucket_count(None, 2)
};
store
.objects
@@ -5440,8 +5445,7 @@ mod tests {
let older = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
scanner_cycle: Some(11),
buckets_count: 1,
..Default::default()
..complete_usage_with_bucket_count(None, 1)
};
older_sender.send(older).await.expect("older-cycle snapshot should enqueue");
drop(older_sender);
@@ -5454,8 +5458,7 @@ mod tests {
let newer = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
scanner_cycle: Some(13),
buckets_count: 3,
..Default::default()
..complete_usage_with_bucket_count(None, 3)
};
newer_sender
.send(newer.clone())
@@ -5490,8 +5493,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)),
scanner_epoch: Some(8),
scanner_cycle: Some(12),
buckets_count: 2,
..Default::default()
..complete_usage_with_bucket_count(None, 2)
};
store
.objects
@@ -5506,8 +5508,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
scanner_epoch: Some(7),
scanner_cycle: Some(99),
buckets_count: 1,
..Default::default()
..complete_usage_with_bucket_count(None, 1)
})
.await
.expect("old-epoch snapshot should enqueue");
@@ -5523,8 +5524,7 @@ mod tests {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
scanner_epoch: None,
scanner_cycle: Some(1),
buckets_count: 3,
..Default::default()
..complete_usage_with_bucket_count(None, 3)
})
.await
.expect("replacement-epoch snapshot should enqueue");
@@ -5554,8 +5554,7 @@ mod tests {
let existing = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
scanner_cycle: Some(12),
buckets_count: 2,
..Default::default()
..complete_usage_with_bucket_count(None, 2)
};
store
.objects
@@ -5569,8 +5568,7 @@ mod tests {
.send(DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
scanner_cycle: Some(12),
buckets_count: 3,
..Default::default()
..complete_usage_with_bucket_count(None, 3)
})
.await
.expect("conflicting same-cycle snapshot should enqueue");
@@ -5595,11 +5593,59 @@ mod tests {
);
}
fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo {
DataUsageInfo {
#[tokio::test]
async fn test_store_data_usage_in_backend_rejects_incomplete_snapshot() {
let store = Arc::new(MemoryConfigStore::default());
let (sender, receiver) = mpsc::channel(2);
let complete_update = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10);
sender
.send(complete_usage_with_bucket_count(Some(complete_update), 1))
.await
.expect("complete usage snapshot should enqueue");
sender
.send(DataUsageInfo {
last_update: Some(complete_update + Duration::from_secs(1)),
buckets_count: 1,
..Default::default()
})
.await
.expect("incomplete usage snapshot should enqueue");
drop(sender);
let outcome = store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await;
let objects = store.objects.lock().await;
let saved = objects
.get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()))
.expect("complete data usage snapshot should remain saved");
let saved = serde_json::from_slice::<DataUsageInfo>(saved).expect("saved usage snapshot should decode");
assert_eq!(saved.last_update, Some(complete_update));
assert!(saved.is_complete_bucket_usage_snapshot());
assert_eq!(outcome, DataUsagePersistOutcome::Failed);
}
fn mark_usage_snapshot_complete(info: &mut DataUsageInfo) {
info.usage_snapshot_complete = true;
}
fn complete_usage_with_bucket_count(last_update: Option<std::time::SystemTime>, buckets_count: u64) -> DataUsageInfo {
let mut info = DataUsageInfo {
last_update,
buckets_count,
usage_snapshot_complete: true,
..Default::default()
};
for index in 0..buckets_count {
let bucket = format!("bucket-{index}");
info.buckets_usage.insert(bucket.clone(), Default::default());
info.bucket_sizes.insert(bucket, 0);
}
info
}
fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo {
complete_usage_with_bucket_count(last_update, 0)
}
#[test]
@@ -5681,11 +5727,10 @@ mod tests {
for idx in 1_u64..=11 {
sender
.send(DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(idx)),
buckets_count: idx,
..Default::default()
})
.send(complete_usage_with_bucket_count(
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(idx)),
idx,
))
.await
.expect("usage snapshot should enqueue");
}
crates/scanner/src/scanner_io.rs
+47 -4
View File
@@ -846,7 +846,7 @@ fn apply_bucket_result_to_cache(cache: &mut DataUsageCache, result: DataUsageEnt
}
fn should_publish_completed_snapshot(completed_count: usize, total_count: usize, budget_elapsed: bool, cancelled: bool) -> bool {
total_count > 0 && completed_count == total_count && !budget_elapsed && !cancelled
completed_count == total_count && !budget_elapsed && !cancelled
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -1015,6 +1015,10 @@ fn completed_data_usage_info(
}
let merged_last_update = results.iter().filter_map(|result| result.info.last_update).max()?;
let bucket_sizes = buckets_usage
.iter()
.map(|(bucket, usage)| (bucket.clone(), usage.size))
.collect();
let data_usage_info = DataUsageInfo {
last_update: Some(merged_last_update),
scanner_cycle: Some(results.first()?.info.next_cycle),
@@ -1023,7 +1027,9 @@ fn completed_data_usage_info(
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
objects_total_size: u64::try_from(total.size).ok()?,
buckets_count: u64::try_from(all_buckets.len()).ok()?,
bucket_sizes,
buckets_usage,
usage_snapshot_complete: true,
..Default::default()
};
Some((data_usage_info, merged_last_update))
@@ -1042,7 +1048,10 @@ mod publish_gate_tests {
assert!(!should_publish_completed_snapshot(2, 3, false, false));
assert!(!should_publish_completed_snapshot(3, 3, true, false));
assert!(!should_publish_completed_snapshot(3, 3, false, true));
assert!(!should_publish_completed_snapshot(0, 0, false, false));
assert!(
should_publish_completed_snapshot(0, 0, false, false),
"a completed empty namespace is an authoritative zero snapshot"
);
}
fn incomplete_scope_cache(source: DataUsageCacheSource) -> DataUsageCache {
@@ -1121,11 +1130,13 @@ mod publish_gate_tests {
#[test]
fn completed_data_usage_info_requires_every_set_before_publish() {
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()];
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string(), "bucket-empty".to_string()];
let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
first_set.replace("bucket-b", DATA_USAGE_ROOT, DataUsageEntry::default());
first_set.replace("bucket-empty", DATA_USAGE_ROOT, DataUsageEntry::default());
let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0));
second_set.replace("bucket-a", DATA_USAGE_ROOT, DataUsageEntry::default());
second_set.replace("bucket-empty", DATA_USAGE_ROOT, DataUsageEntry::default());
assert!(
completed_data_usage_info_for_test(&[first_set.clone(), DataUsageCache::default()], &all_buckets, false, false)
@@ -1144,7 +1155,38 @@ mod publish_gate_tests {
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
assert_eq!(data_usage_info.scanner_cycle, Some(0));
assert_eq!(data_usage_info.objects_total_count, 3);
assert_eq!(data_usage_info.buckets_usage.len(), 2);
assert_eq!(data_usage_info.buckets_usage.len(), 3);
assert!(data_usage_info.usage_snapshot_complete);
assert_eq!(
data_usage_info
.buckets_usage
.get("bucket-empty")
.map(|usage| (usage.objects_count, usage.size)),
Some((0, 0))
);
}
#[test]
fn completed_data_usage_info_publishes_confirmed_empty_namespace() {
let mut completed_set = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
source: Some(DataUsageCacheSource::new(0, 0)),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
..Default::default()
},
..Default::default()
};
completed_set.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
let (data_usage_info, _) = completed_data_usage_info_for_test(&[completed_set], &[], false, false)
.expect("a completed empty namespace should produce an authoritative snapshot");
assert!(data_usage_info.is_complete_bucket_usage_snapshot());
assert_eq!(data_usage_info.buckets_count, 0);
assert!(data_usage_info.buckets_usage.is_empty());
}
#[test]
@@ -2000,6 +2042,7 @@ impl ScannerIOCycle for ECStore {
let empty_usage = DataUsageInfo {
last_update: Some(SystemTime::now()),
scanner_cycle: Some(want_cycle),
usage_snapshot_complete: true,
..Default::default()
};
send_data_usage_update(&updates, empty_usage).await?;
crates/scanner/src/storage_api.rs
+10 -6
View File
@@ -57,7 +57,10 @@ pub(crate) use rustfs_ecstore::api::config::init as ecstore_config_init;
pub(crate) use rustfs_ecstore::api::config::storageclass::{
RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD,
};
pub(crate) use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info;
pub(crate) use rustfs_ecstore::api::data_usage::{
invalidate_data_usage_snapshot_cache as ecstore_invalidate_data_usage_snapshot_cache,
replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
@@ -107,11 +110,12 @@ pub(crate) mod owner {
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi,
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr,
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure,
ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle,
ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config,
ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
crates/test-utils/src/data_usage_snapshot_tests.rs
+1
View File
@@ -34,6 +34,7 @@ fn snapshot(bucket: &str, last_update: SystemTime) -> DataUsageInfo {
..Default::default()
},
);
info.usage_snapshot_complete = true;
info.bucket_sizes.insert(bucket.to_string(), 42);
info.buckets_count = 1;
info.calculate_totals();
docs/architecture/compat-cleanup-register.md
+1 -1
View File
@@ -12,7 +12,7 @@ for later deletion.
## Open Items
- `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while the authoritative `.usage.v2.json` pair is absent and continue removing deleted buckets from legacy copies that still exist. Remove the fallback and legacy cleanup after every supported direct-upgrade source version writes `.usage.v2.json`.
- `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while `.usage.v2.json` is absent and continue removing deleted buckets from legacy copies that still exist. The additive usage_snapshot_complete field in `.usage.v2.json` must remain optional while mixed-version clusters are supported; a missing field means the snapshot is not authoritative. Remove the legacy object fallback and cleanup only after every supported direct-upgrade source writes `.usage.v2.json`.
- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Current protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while the distributed scanner publishes usage only after every peer returns authenticated protocol v5 state. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, and remove the protocol-v4 activity codec after every supported peer implements protocol v5; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
- `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
- `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
rustfs/src/admin/handlers/quota.rs
+52 -24
View File
@@ -27,7 +27,7 @@ use hyper::{Method, StatusCode};
use matchit::Params;
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use s3s::{Body, S3Error, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use serde_json;
use std::sync::Arc;
@@ -126,6 +126,16 @@ fn compat_bucket_quota_response(quota: &BucketQuota) -> CompatibleBucketQuotaRes
}
}
fn map_quota_query_error(error: QuotaError, bucket: &str, operation: &str) -> S3Error {
match error {
QuotaError::ConfigNotFound { .. } => s3_error!(NoSuchBucket, "bucket not found: {}", bucket),
QuotaError::UsageUnavailable { .. } => {
s3_error!(ServiceUnavailable, "authoritative bucket usage is not available yet")
}
error => s3_error!(InternalError, "failed to {}: {}", operation, error),
}
}
#[derive(Debug, Serialize)]
pub struct BucketQuotaResponse {
pub bucket: String,
@@ -376,27 +386,34 @@ impl Operation for GetBucketQuotaHandler {
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
let (quota, current_usage) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e {
QuotaError::ConfigNotFound { .. } => {
s3_error!(NoSuchBucket, "bucket not found: {}", bucket)
if is_compat_get_bucket_quota_path(req.uri.path()) {
if !quota_checker.bucket_exists(&bucket).await {
return Err(s3_error!(NoSuchBucket, "bucket not found: {}", bucket));
}
_ => s3_error!(InternalError, "failed to get quota: {}", e),
})?;
let quota = quota_checker
.get_quota_config(&bucket)
.await
.map_err(|e| s3_error!(InternalError, "failed to get quota: {}", e))?;
let json = serde_json::to_string(&compat_bucket_quota_response(&quota))
.map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
return Ok(S3Response::new((StatusCode::OK, Body::from(json))));
}
let json = if is_compat_get_bucket_quota_path(req.uri.path()) {
serde_json::to_string(&compat_bucket_quota_response(&quota))
.map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
} else {
let response = BucketQuotaResponse {
bucket,
quota: quota.quota,
size: current_usage.unwrap_or(0),
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
};
let (quota, current_usage) = quota_checker
.get_quota_stats(&bucket)
.await
.map_err(|error| map_quota_query_error(error, &bucket, "get quota"))?;
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
let response = BucketQuotaResponse {
bucket,
quota: quota.quota,
size: current_usage.unwrap_or(0),
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
};
let json =
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
}
}
@@ -550,12 +567,10 @@ impl Operation for GetBucketQuotaStatsHandler {
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
let (quota, current_usage_opt) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e {
QuotaError::ConfigNotFound { .. } => {
s3_error!(NoSuchBucket, "bucket not found: {}", bucket)
}
_ => s3_error!(InternalError, "failed to get quota stats: {}", e),
})?;
let (quota, current_usage_opt) = quota_checker
.get_quota_stats(&bucket)
.await
.map_err(|error| map_quota_query_error(error, &bucket, "get quota stats"))?;
let current_usage = current_usage_opt.unwrap_or(0);
let usage_percentage = quota.quota.and_then(|limit| {
@@ -659,7 +674,7 @@ impl Operation for CheckBucketQuotaHandler {
let result = quota_checker
.check_quota_with_usage_reporting(&bucket, operation, request.operation_size, true)
.await
.map_err(|e| s3_error!(InternalError, "failed to check quota: {}", e))?;
.map_err(|error| map_quota_query_error(error, &bucket, "check quota"))?;
let response = CheckQuotaResponse {
bucket,
@@ -715,6 +730,19 @@ mod tests {
assert!(json.contains("\"quotatype\":\"hard\""));
}
#[test]
fn unknown_authoritative_usage_is_retryable() {
let error = map_quota_query_error(
QuotaError::UsageUnavailable {
bucket: "test-bucket".to_string(),
},
"test-bucket",
"check quota",
);
assert_eq!(error.code(), &s3s::S3ErrorCode::ServiceUnavailable);
}
#[test]
fn test_quota_operation_parsing() {
let parse_operation = |operation: &str| match operation.to_uppercase().as_str() {
rustfs/src/app/admin_usecase.rs
+52
View File
@@ -19,6 +19,7 @@ use super::storage_api::admin_usecase::capacity::{
PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free,
};
use super::storage_api::admin_usecase::contract::StorageAdminApi;
use super::storage_api::admin_usecase::contract::bucket::{BucketOperations as _, BucketOptions};
use super::storage_api::admin_usecase::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend_cached};
use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
use crate::app::runtime_sources::{
@@ -33,6 +34,7 @@ use crate::server::{DependencyReadiness, collect_dependency_readiness_report as
use rustfs_data_usage::DataUsageInfo;
use rustfs_madmin::{InfoMessage, StorageInfo};
use s3s::S3ErrorCode;
use std::collections::HashSet;
use std::sync::Arc;
use tracing::{debug, error, info, warn};
@@ -212,6 +214,16 @@ impl DefaultAdminUsecase {
result.map_err(|_| Self::app_error(S3ErrorCode::InternalError, "load_data_usage_from_backend failed"))
}
fn data_usage_snapshot_covers_namespace(info: &DataUsageInfo, buckets: impl IntoIterator<Item = String>) -> bool {
if !info.is_complete_bucket_usage_snapshot() {
return false;
}
let current_buckets: HashSet<String> = buckets.into_iter().filter(|bucket| !bucket.starts_with('.')).collect();
current_buckets.len() == info.buckets_usage.len()
&& current_buckets.iter().all(|bucket| info.buckets_usage.contains_key(bucket))
}
async fn refresh_rebalance_status_snapshot(store: &ECStore) -> AdminUsecaseResult<()> {
store.refresh_rebalance_status_meta().await.map_err(|err| {
error!("refresh rebalance metadata for pool status failed: {:?}", err);
@@ -255,6 +267,20 @@ impl DefaultAdminUsecase {
pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
let mut info = Self::map_data_usage_load_result(load_data_usage_from_backend_cached(store.clone()).await)?;
apply_bucket_usage_memory_overlay(&mut info).await;
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
no_metadata: true,
..Default::default()
})
.await
.map_err(|_| Self::app_error(S3ErrorCode::InternalError, "list_bucket failed"))?;
if !Self::data_usage_snapshot_covers_namespace(&info, buckets.into_iter().map(|bucket| bucket.name)) {
return Err(Self::app_error(
S3ErrorCode::ServiceUnavailable,
"authoritative data usage snapshot is not available yet",
));
}
let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
@@ -645,6 +671,32 @@ mod tests {
});
}
#[test]
fn data_usage_snapshot_must_cover_the_current_bucket_namespace() {
let mut info = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
usage_snapshot_complete: true,
buckets_count: 1,
..Default::default()
};
info.buckets_usage.insert("bucket-a".to_string(), Default::default());
assert!(DefaultAdminUsecase::data_usage_snapshot_covers_namespace(
&info,
["bucket-a".to_string(), ".rustfs.sys".to_string()]
));
assert!(!DefaultAdminUsecase::data_usage_snapshot_covers_namespace(
&info,
["bucket-a".to_string(), "bucket-b".to_string()]
));
info.usage_snapshot_complete = false;
assert!(!DefaultAdminUsecase::data_usage_snapshot_covers_namespace(
&info,
["bucket-a".to_string()]
));
}
#[tokio::test]
async fn execute_query_storage_info_returns_internal_error_when_store_uninitialized() {
let usecase = DefaultAdminUsecase::without_context();
rustfs/src/app/data_usage_snapshot_gating_test.rs
+220 -10
View File
@@ -25,23 +25,36 @@
//! while the response carries the seeded numbers.
use super::gating_test_env::shared_gating_ecstore;
use super::storage_api::test::StoragePutObjReader as PutObjReader;
use super::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions};
use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions, MakeBucketOptions};
use super::storage_api::test::contract::object::ObjectIO as _;
use super::storage_api::test::data_usage::{
compute_bucket_usage, live_bucket_usage_computations, load_data_usage_from_backend_cached, record_bucket_object_write_memory,
store_data_usage_in_backend,
};
use super::storage_api::test::{ECStore, StoragePutObjReader as PutObjReader};
use crate::app::admin_usecase::DefaultAdminUsecase;
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
use serial_test::serial;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
const SEEDED_BUCKET_SIZE: u64 = 123_456;
const SEEDED_BUCKET_OBJECTS: u64 = 42;
async fn next_snapshot_time(ecstore: Arc<ECStore>) -> SystemTime {
let now = SystemTime::now();
let latest = load_data_usage_from_backend_cached(ecstore)
.await
.ok()
.and_then(|info| info.last_update)
.unwrap_or(now);
latest.max(now) + Duration::from_nanos(1)
}
fn seeded_data_usage_info(bucket: &str, last_update: SystemTime) -> DataUsageInfo {
let usage = BucketUsageInfo {
size: SEEDED_BUCKET_SIZE,
@@ -58,6 +71,7 @@ fn seeded_data_usage_info(bucket: &str, last_update: SystemTime) -> DataUsageInf
..Default::default()
};
info.buckets_usage = HashMap::from([(bucket.to_string(), usage)]);
info.usage_snapshot_complete = true;
info.bucket_sizes = HashMap::from([(bucket.to_string(), SEEDED_BUCKET_SIZE)]);
info
}
@@ -78,10 +92,18 @@ fn sized_data_usage_info(bucket: &str, size: u64, last_update: SystemTime) -> Da
..Default::default()
};
info.buckets_usage = HashMap::from([(bucket.to_string(), usage)]);
info.usage_snapshot_complete = true;
info.bucket_sizes = HashMap::from([(bucket.to_string(), size)]);
info
}
async fn persist_scanner_snapshot(ecstore: Arc<ECStore>, info: DataUsageInfo) {
let (sender, receiver) = mpsc::channel(1);
sender.send(info).await.expect("enqueue scanner snapshot");
drop(sender);
rustfs_scanner::scanner::store_data_usage_in_backend(CancellationToken::new(), ecstore, receiver).await;
}
#[tokio::test]
#[serial]
async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
@@ -121,10 +143,34 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
.make_bucket(&seeded_bucket, &MakeBucketOptions::default())
.await
.expect("create seeded bucket");
let seeded_at = SystemTime::now();
store_data_usage_in_backend(seeded_data_usage_info(&seeded_bucket, seeded_at), ecstore.clone())
ecstore
.make_bucket(&overlay_bucket, &MakeBucketOptions::default())
.await
.expect("persist seeded data usage snapshot");
.expect("create overlay bucket");
let seeded_at = next_snapshot_time(ecstore.clone()).await;
let mut seeded_info = seeded_data_usage_info(&seeded_bucket, seeded_at);
for bucket in ecstore
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("list the complete live bucket namespace")
.into_iter()
.filter(|bucket| !bucket.name.starts_with('.'))
{
seeded_info.buckets_usage.entry(bucket.name.clone()).or_default();
seeded_info.bucket_sizes.entry(bucket.name).or_default();
}
seeded_info.bucket_sizes.insert(live_bucket.clone(), control_usage.size);
seeded_info.buckets_usage.insert(live_bucket.clone(), control_usage);
seeded_info
.buckets_usage
.insert(overlay_bucket.clone(), BucketUsageInfo::default());
seeded_info.bucket_sizes.insert(overlay_bucket.clone(), 0);
seeded_info.buckets_count = u64::try_from(seeded_info.buckets_usage.len()).expect("test bucket count should fit into u64");
seeded_info.calculate_totals();
persist_scanner_snapshot(ecstore.clone(), seeded_info).await;
record_bucket_object_write_memory(&overlay_bucket, None, 512).await;
@@ -182,6 +228,138 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
assert_eq!(overlay_usage.size, 512);
}
#[tokio::test]
#[serial]
async fn data_usage_endpoint_treats_legacy_partial_stats_as_unknown_until_covered() {
let ecstore = shared_gating_ecstore().await;
let agent_bucket = format!("usage-legacy-agent-{}", Uuid::new_v4());
let small_bucket = format!("usage-legacy-small-{}", Uuid::new_v4());
ecstore
.make_bucket(&agent_bucket, &MakeBucketOptions::default())
.await
.expect("create legacy partial snapshot bucket");
ecstore
.make_bucket(&small_bucket, &MakeBucketOptions::default())
.await
.expect("create legacy zero snapshot bucket");
let mut reader = PutObjReader::from_vec(b"object exists".to_vec());
ecstore
.put_object(&agent_bucket, "object.bin", &mut reader, &Default::default())
.await
.expect("put object before persisting legacy partial snapshot");
let mut small_reader = PutObjReader::from_vec(vec![b'x'; 57_344]);
ecstore
.put_object(&small_bucket, "small.bin", &mut small_reader, &Default::default())
.await
.expect("put object before persisting legacy zero snapshot");
let persisted_at = next_snapshot_time(ecstore.clone()).await;
let mut legacy = DataUsageInfo {
last_update: Some(persisted_at),
buckets_count: 2,
..Default::default()
};
legacy.buckets_usage.insert(
agent_bucket.clone(),
BucketUsageInfo {
size: 196_870_144,
objects_count: 1_502,
versions_count: 1_502,
..Default::default()
},
);
legacy.buckets_usage.insert(small_bucket.clone(), BucketUsageInfo::default());
legacy.bucket_sizes.insert(agent_bucket.clone(), 196_870_144);
legacy.bucket_sizes.insert(small_bucket.clone(), 0);
store_data_usage_in_backend(legacy, ecstore.clone())
.await
.expect("persist legacy partial snapshot");
let before_endpoint = live_bucket_usage_computations();
let unknown = DefaultAdminUsecase::query_data_usage_info_with_store(ecstore.clone())
.await
.expect_err("an uncovered snapshot must be reported as temporarily unavailable");
assert_eq!(unknown.code, s3s::S3ErrorCode::ServiceUnavailable);
assert_eq!(
live_bucket_usage_computations(),
before_endpoint,
"uncovered snapshot handling must not restore request-time object listings"
);
let completed_at = persisted_at + Duration::from_nanos(1);
let mut completed = DataUsageInfo {
last_update: Some(completed_at),
usage_snapshot_complete: true,
..Default::default()
};
for bucket in ecstore
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("list the complete live bucket namespace")
.into_iter()
.filter(|bucket| !bucket.name.starts_with('.'))
{
completed
.buckets_usage
.insert(bucket.name.clone(), BucketUsageInfo::default());
completed.bucket_sizes.insert(bucket.name, 0);
}
completed.buckets_usage.insert(
agent_bucket.clone(),
BucketUsageInfo {
size: 222_822_400,
objects_count: 1_700,
versions_count: 1_700,
..Default::default()
},
);
completed.bucket_sizes.insert(agent_bucket.clone(), 222_822_400);
completed.buckets_usage.insert(
small_bucket.clone(),
BucketUsageInfo {
size: 57_344,
objects_count: 1,
versions_count: 1,
..Default::default()
},
);
completed.bucket_sizes.insert(small_bucket.clone(), 57_344);
completed.buckets_count = u64::try_from(completed.buckets_usage.len()).expect("test bucket count should fit into u64");
completed.calculate_totals();
store_data_usage_in_backend(completed, ecstore.clone())
.await
.expect("persist complete covered snapshot");
let covered = DefaultAdminUsecase::query_data_usage_info_with_store(ecstore)
.await
.expect("query covered data usage info");
assert_eq!(covered.last_update, Some(completed_at));
assert!(covered.usage_snapshot_complete);
assert_eq!(
covered
.buckets_usage
.get(&agent_bucket)
.map(|usage| (usage.objects_count, usage.size)),
Some((1_700, 222_822_400))
);
assert_eq!(
covered
.buckets_usage
.get(&small_bucket)
.map(|usage| (usage.objects_count, usage.size)),
Some((1, 57_344))
);
assert_eq!(
live_bucket_usage_computations(),
before_endpoint,
"covered snapshot publication must not trigger request-time object listings"
);
}
/// Revert detector for the snapshot-cache invalidation in
/// `save_data_usage_in_backend` (rustfs/backlog#1306): a fresh scanner save must
/// be visible to the very next cached read, not deferred until the 30s TTL
@@ -194,11 +372,11 @@ async fn save_data_usage_invalidates_snapshot_cache() {
let ecstore = shared_gating_ecstore().await;
let bucket = format!("usage-invalidate-{}", Uuid::new_v4());
// Future-date both snapshots beyond the stale-guard's 5min tolerance so the
// persists win deterministically over whatever snapshot a sibling serial
// test may have left behind, regardless of test execution order.
let warm_at = SystemTime::now() + Duration::from_secs(600);
let changed_at = warm_at + Duration::from_secs(1);
// Advance beyond the latest persisted timestamp by the smallest possible
// increment so this save wins the monotonicity guard without masking a
// subsequent in-memory mutation as older than the snapshot.
let warm_at = next_snapshot_time(ecstore.clone()).await;
let changed_at = warm_at + Duration::from_nanos(1);
const WARM_SIZE: u64 = 111_000;
const CHANGED_SIZE: u64 = 222_000;
@@ -236,6 +414,38 @@ async fn save_data_usage_invalidates_snapshot_cache() {
);
}
#[tokio::test]
#[serial]
async fn scanner_save_invalidates_snapshot_cache() {
let ecstore = shared_gating_ecstore().await;
let bucket = format!("usage-scanner-invalidate-{}", Uuid::new_v4());
let warm_at = next_snapshot_time(ecstore.clone()).await;
let changed_at = warm_at + Duration::from_nanos(1);
const WARM_SIZE: u64 = 333_000;
const CHANGED_SIZE: u64 = 444_000;
store_data_usage_in_backend(sized_data_usage_info(&bucket, WARM_SIZE, warm_at), ecstore.clone())
.await
.expect("persist initial snapshot");
let warmed = load_data_usage_from_backend_cached(ecstore.clone())
.await
.expect("warm cached snapshot");
assert_eq!(warmed.buckets_usage.get(&bucket).map(|usage| usage.size), Some(WARM_SIZE));
persist_scanner_snapshot(ecstore.clone(), sized_data_usage_info(&bucket, CHANGED_SIZE, changed_at)).await;
let reloaded = load_data_usage_from_backend_cached(ecstore)
.await
.expect("reload cached snapshot after scanner save");
assert_eq!(
reloaded.buckets_usage.get(&bucket).map(|usage| usage.size),
Some(CHANGED_SIZE),
"scanner persistence must invalidate the process-local snapshot cache"
);
assert_eq!(reloaded.last_update, Some(changed_at));
}
/// Wire pin for the no-snapshot response shape (rustfs/backlog#1306): a
/// default `DataUsageInfo` must keep serializing `last_update` as `null` with
/// empty bucket maps, so "no snapshot yet" stays distinguishable from real
rustfs/src/app/multipart_usecase.rs
+22 -14
View File
@@ -58,7 +58,9 @@ use crate::app::object_data_cache::{
ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success,
invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation,
};
use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write};
use crate::app::object_usecase::{
build_put_like_object_lock_metadata, map_quota_check_outcome, validate_existing_object_lock_for_write,
};
use crate::app::runtime_sources::{
AppContext, current_app_context, current_object_data_cache_for_context, current_object_store_handle_for_context,
};
@@ -516,6 +518,12 @@ impl DefaultMultipartUsecase {
_ => None,
};
let quota_metadata_sys = self.bucket_metadata_sys();
if let Some(metadata_sys) = quota_metadata_sys.as_ref() {
let quota_checker = QuotaChecker::new(metadata_sys.clone());
map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?;
}
let obj_info = store
.clone()
.complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, &opts)
@@ -524,17 +532,18 @@ impl DefaultMultipartUsecase {
let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await;
record_capacity_write(Some(capacity_scope_token)).await;
// check quota after completing multipart upload
if let Some(metadata_sys) = self.bucket_metadata_sys() {
if let Some(metadata_sys) = quota_metadata_sys {
let quota_checker = QuotaChecker::new(metadata_sys);
match quota_checker
.check_quota(&bucket, QuotaOperation::PutObject, obj_info.size as u64)
.check_quota(&bucket, QuotaOperation::PutObject, obj_info.size.max(0) as u64)
.await
{
Ok(check_result) => {
if !check_result.allowed {
// Quota exceeded, delete the completed object
// Preserve the established compensation behavior for
// a known over-quota result. Unknown usage is rejected
// by the preflight check before the upload is committed.
let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await;
let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await;
return Err(S3Error::with_message(
@@ -546,18 +555,17 @@ impl DefaultMultipartUsecase {
),
));
}
// Update quota tracking after successful multipart upload
if versioned {
record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64)
.await;
} else {
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
}
}
Err(e) => {
warn!("Quota check failed for bucket {}: {}, allowing operation", bucket, e);
Err(err) => {
warn!("Quota check failed for bucket {} after multipart completion: {}", bucket, err);
}
}
if versioned {
record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
} else {
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
}
}
enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
rustfs/src/app/object_usecase.rs
+49 -26
View File
@@ -501,7 +501,7 @@ fn request_body_is_aws_chunked_framed(headers: &HeaderMap) -> bool {
/// Map a bucket-quota checker outcome onto the S3 admission result.
///
/// Hard is the only supported quota type, so a checker fault (bucket-config read, config parse, or usage lookup) must fail closed rather than admit the write: allowing it would silently bypass a configured hard quota. The no-quota happy path never reaches the error arm — `QuotaChecker::check_quota` returns `Ok(allowed)` via the zero-extra-I/O fast path when no quota is configured, so failing closed here cannot penalise buckets without a quota. A fault surfaces as a retryable `ServiceUnavailable` and is counted; the client-facing message stays generic so internal config/usage details are not leaked.
fn map_quota_check_outcome(bucket: &str, outcome: Result<QuotaCheckResult, QuotaError>) -> S3Result<()> {
pub(super) fn map_quota_check_outcome(bucket: &str, outcome: Result<QuotaCheckResult, QuotaError>) -> S3Result<QuotaCheckResult> {
match outcome {
Ok(result) if !result.allowed => Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
@@ -513,13 +513,17 @@ fn map_quota_check_outcome(bucket: &str, outcome: Result<QuotaCheckResult, Quota
)),
Err(e) => {
counter!("rustfs_bucket_quota_check_failed_total").increment(1);
warn!(bucket, error = %e, state = "checker_failed", "Bucket quota check failed closed");
if matches!(&e, QuotaError::UsageUnavailable { .. }) {
debug!(bucket, error = %e, state = "usage_pending", "Bucket quota check waiting for authoritative usage");
} else {
warn!(bucket, error = %e, state = "checker_failed", "Bucket quota check failed closed");
}
Err(S3Error::with_message(
S3ErrorCode::ServiceUnavailable,
"Bucket quota check temporarily unavailable, please retry".to_string(),
))
}
_ => Ok(()),
Ok(result) => Ok(result),
}
}
@@ -3263,7 +3267,7 @@ impl DefaultObjectUsecase {
return Ok(());
};
let quota_checker = QuotaChecker::new(metadata_sys);
map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await)
map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(|_| ())
}
fn build_memory_bytes_blob(
@@ -7983,25 +7987,9 @@ impl DefaultObjectUsecase {
let extract_limits = put_object_extract_limits();
let extract_quota_snapshot = if let Some(metadata_sys) = self.bucket_metadata_sys() {
let quota_checker = QuotaChecker::new(metadata_sys);
match quota_checker.get_quota_config(&bucket).await {
Ok(quota) => {
if let Some(limit) = quota.quota {
match quota_checker.get_real_time_usage(&bucket).await {
Ok(current_usage) => Some((current_usage, limit)),
Err(err) => {
warn!(bucket, error = %err, state = "extract_usage_snapshot_failed", "Bucket quota snapshot degraded to allow");
None
}
}
} else {
None
}
}
Err(err) => {
warn!(bucket, error = %err, state = "extract_quota_snapshot_failed", "Bucket quota snapshot degraded to allow");
None
}
}
let check_result =
map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?;
check_result.current_usage.zip(check_result.quota_limit)
} else {
None
};
@@ -8191,8 +8179,11 @@ impl DefaultObjectUsecase {
let cache_adapter = self.object_data_cache();
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &fpath).await;
let obj_info = match store.put_object(&bucket, &fpath, &mut reader, &opts).await {
Ok(info) => info,
let (obj_info, backfilled_old_current_size) = match store
.put_object_with_old_current_size(&bucket, &fpath, &mut reader, &opts)
.await
{
Ok(result) => result,
Err(e) => {
if extract_options.ignore_errors {
warn!(error = %e, "Archive object write skipped due to ignore-errors");
@@ -8201,6 +8192,21 @@ impl DefaultObjectUsecase {
return Err(ApiError::from(e).into());
}
};
let extract_versioned = BucketVersioningSys::prefix_enabled(&bucket, &fpath).await;
match previous_current_size_from_backfill(backfilled_old_current_size) {
Some(previous_current_size) => {
if extract_versioned {
record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64)
.await;
} else {
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
}
}
None => {
record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, extract_versioned)
.await;
}
}
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &fpath).await;
if !wrote_any_entry {
rustfs_scanner::record_dirty_usage_bucket(&bucket);
@@ -14064,7 +14070,12 @@ mod tests {
#[test]
fn quota_admission_allows_within_limit() {
map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write");
let result = map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write");
assert_eq!(result.current_usage, Some(1024));
assert_eq!(result.quota_limit, Some(2048));
assert_eq!(result.operation_size, 512);
assert_eq!(result.remaining, Some(512));
}
#[test]
@@ -14085,4 +14096,16 @@ mod tests {
.expect_err("a checker fault must fail closed");
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
}
#[test]
fn quota_admission_fails_closed_on_unknown_authoritative_usage() {
let err = map_quota_check_outcome(
"bucket",
Err(QuotaError::UsageUnavailable {
bucket: "bucket".to_string(),
}),
)
.expect_err("unknown authoritative usage must not admit a quota-controlled write");
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
}
}
rustfs/src/app/storage_api.rs
+4
View File
@@ -931,6 +931,10 @@ pub(crate) mod s3_api {
pub(crate) mod admin_usecase {
pub(crate) mod contract {
pub(crate) mod bucket {
pub(crate) use super::super::super::storage_contracts::{BucketOperations, BucketOptions};
}
pub(crate) use super::super::storage_contracts::StorageAdminApi;
}
rustfs/src/error.rs
+11
View File
@@ -358,6 +358,7 @@ impl From<QuotaError> for ApiError {
let code = match &err {
QuotaError::QuotaExceeded { .. } => S3ErrorCode::InvalidRequest,
QuotaError::ConfigNotFound { .. } => S3ErrorCode::NoSuchBucket,
QuotaError::UsageUnavailable { .. } => S3ErrorCode::ServiceUnavailable,
QuotaError::InvalidConfig { .. } => S3ErrorCode::InvalidArgument,
QuotaError::StorageError(_) => S3ErrorCode::InternalError,
};
@@ -486,6 +487,16 @@ mod tests {
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
}
#[test]
fn test_unknown_authoritative_quota_usage_maps_to_retryable_error() {
let api_error = ApiError::from(QuotaError::UsageUnavailable {
bucket: "bucket".to_string(),
});
assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
}
#[test]
fn test_api_error_from_storage_error_mappings() {
let test_cases = vec![