mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
fix(scanner): require authoritative usage snapshots (#5333)
This commit is contained in:
@@ -32,12 +32,12 @@ use std::{
|
|||||||
/// save forever and freeze admin usage stats; callers must bypass the skip instead.
|
/// save forever and freeze admin usage stats; callers must bypass the skip instead.
|
||||||
pub const USAGE_LAST_UPDATE_FUTURE_TOLERANCE: Duration = Duration::from_secs(5 * 60);
|
pub const USAGE_LAST_UPDATE_FUTURE_TOLERANCE: Duration = Duration::from_secs(5 * 60);
|
||||||
|
|
||||||
/// Authoritative cluster-wide usage snapshot written by coordinated scanners.
|
/// Cluster-wide usage snapshot written by coordinated scanners.
|
||||||
///
|
///
|
||||||
/// This object name is intentionally distinct from the legacy unfenced
|
/// `usage_snapshot_complete` is an additive JSON field: older readers ignore
|
||||||
/// snapshot. Older binaries can continue writing the legacy object during a
|
/// it, while current readers treat snapshots from older writers as unknown.
|
||||||
/// rolling upgrade without overwriting a snapshot produced by the current
|
/// Keeping the existing object name preserves rolling-upgrade and rollback
|
||||||
/// scanner protocol.
|
/// compatibility without allowing an ambiguous snapshot to become authoritative.
|
||||||
pub const DATA_USAGE_OBJECT_NAME: &str = ".usage.v2.json";
|
pub const DATA_USAGE_OBJECT_NAME: &str = ".usage.v2.json";
|
||||||
|
|
||||||
/// Usage snapshot written by scanner implementations predating distributed
|
/// Usage snapshot written by scanner implementations predating distributed
|
||||||
@@ -190,6 +190,12 @@ pub struct DataUsageInfo {
|
|||||||
pub buckets_count: u64,
|
pub buckets_count: u64,
|
||||||
/// Buckets usage info provides following information across all buckets
|
/// Buckets usage info provides following information across all buckets
|
||||||
pub buckets_usage: HashMap<String, BucketUsageInfo>,
|
pub buckets_usage: HashMap<String, BucketUsageInfo>,
|
||||||
|
/// Whether this snapshot covers the complete bucket namespace.
|
||||||
|
///
|
||||||
|
/// Legacy snapshots default to `false`. A complete snapshot contains an
|
||||||
|
/// explicit entry for every bucket, including confirmed-empty buckets.
|
||||||
|
#[serde(default)]
|
||||||
|
pub usage_snapshot_complete: bool,
|
||||||
/// Deprecated kept here for backward compatibility reasons
|
/// Deprecated kept here for backward compatibility reasons
|
||||||
pub bucket_sizes: HashMap<String, u64>,
|
pub bucket_sizes: HashMap<String, u64>,
|
||||||
/// Per-disk snapshot information when available
|
/// Per-disk snapshot information when available
|
||||||
@@ -681,6 +687,12 @@ pub struct DataUsageCacheInfo {
|
|||||||
pub skip_healing: bool,
|
pub skip_healing: bool,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub failed_objects: HashMap<String, u64>,
|
pub failed_objects: HashMap<String, u64>,
|
||||||
|
/// Whether this per-set cache was produced by a completed scanner pass.
|
||||||
|
///
|
||||||
|
/// Older cache writers omit this field and therefore deserialize as
|
||||||
|
/// incomplete instead of exposing partial set totals as confirmed zeros.
|
||||||
|
#[serde(default)]
|
||||||
|
pub snapshot_complete: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Data usage cache
|
/// Data usage cache
|
||||||
@@ -1000,6 +1012,7 @@ impl DataUsageCache {
|
|||||||
objects_total_size: flat.size as u64,
|
objects_total_size: flat.size as u64,
|
||||||
buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX),
|
buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX),
|
||||||
buckets_usage,
|
buckets_usage,
|
||||||
|
usage_snapshot_complete: self.info.snapshot_complete,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1078,6 +1091,13 @@ impl DataUsageInfo {
|
|||||||
Self::default()
|
Self::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether this snapshot authoritatively covers every reported bucket.
|
||||||
|
pub fn is_complete_bucket_usage_snapshot(&self) -> bool {
|
||||||
|
self.usage_snapshot_complete
|
||||||
|
&& self.last_update.is_some()
|
||||||
|
&& u64::try_from(self.buckets_usage.len()).ok() == Some(self.buckets_count)
|
||||||
|
}
|
||||||
|
|
||||||
/// Add object metadata to data usage statistics
|
/// Add object metadata to data usage statistics
|
||||||
pub fn add_object(&mut self, object_path: &str, meta_object: &rustfs_filemeta::MetaObject) {
|
pub fn add_object(&mut self, object_path: &str, meta_object: &rustfs_filemeta::MetaObject) {
|
||||||
// This method is kept for backward compatibility
|
// This method is kept for backward compatibility
|
||||||
@@ -1442,6 +1462,35 @@ pub struct CompressionTotalInfo {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct LegacyUsageReader {
|
||||||
|
buckets_count: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn completeness_marker_is_additive_for_legacy_named_readers() {
|
||||||
|
let current = DataUsageInfo {
|
||||||
|
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||||
|
usage_snapshot_complete: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let encoded = rmp_serde::to_vec_named(¤t).expect("encode current data usage snapshot");
|
||||||
|
let legacy: LegacyUsageReader = rmp_serde::from_slice(&encoded).expect("legacy reader should ignore additive fields");
|
||||||
|
|
||||||
|
assert_eq!(legacy.buckets_count, 0);
|
||||||
|
assert!(current.is_complete_bucket_usage_snapshot());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn completeness_marker_requires_a_snapshot_timestamp() {
|
||||||
|
let untimestamped = DataUsageInfo {
|
||||||
|
usage_snapshot_complete: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(!untimestamped.is_complete_bucket_usage_snapshot());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_usage_last_update_future_tolerance_boundary() {
|
fn test_usage_last_update_future_tolerance_boundary() {
|
||||||
let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
|
let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
|
||||||
|
|||||||
@@ -287,10 +287,10 @@ pub mod config {
|
|||||||
pub mod data_usage {
|
pub mod data_usage {
|
||||||
pub use crate::data_usage::{
|
pub use crate::data_usage::{
|
||||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||||
init_compression_total_memory_from_backend, live_bucket_usage_computations, load_compression_total_from_memory,
|
init_compression_total_memory_from_backend, invalidate_data_usage_snapshot_cache, live_bucket_usage_computations,
|
||||||
load_data_usage_from_backend, load_data_usage_from_backend_cached, record_bucket_delete_marker_memory,
|
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
|
||||||
record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
|
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
|
||||||
record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
||||||
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
|
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
|
||||||
remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
|
remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
|
||||||
store_data_usage_in_backend,
|
store_data_usage_in_backend,
|
||||||
|
|||||||
@@ -167,7 +167,7 @@ impl QuotaChecker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let quota = self.get_quota_config(bucket).await?;
|
let quota = self.get_quota_config(bucket).await?;
|
||||||
let current_usage = self.get_real_time_usage(bucket).await.unwrap_or(0);
|
let current_usage = self.get_real_time_usage(bucket).await?;
|
||||||
|
|
||||||
Ok((quota, Some(current_usage)))
|
Ok((quota, Some(current_usage)))
|
||||||
}
|
}
|
||||||
@@ -177,7 +177,11 @@ impl QuotaChecker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_real_time_usage(&self, bucket: &str) -> Result<u64, QuotaError> {
|
pub async fn get_real_time_usage(&self, bucket: &str) -> Result<u64, QuotaError> {
|
||||||
Ok(get_bucket_usage_memory(bucket).await.unwrap_or(0))
|
get_bucket_usage_memory(bucket)
|
||||||
|
.await
|
||||||
|
.ok_or_else(|| QuotaError::UsageUnavailable {
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -185,6 +189,8 @@ impl QuotaChecker {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
|
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
|
||||||
|
use serial_test::serial;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
/// Regression (PR #5307 / s3-tests `test_100_continue_error_retry`): a
|
/// Regression (PR #5307 / s3-tests `test_100_continue_error_retry`): a
|
||||||
/// bucket with no persisted metadata has no quota, so the admission check
|
/// bucket with no persisted metadata has no quota, so the admission check
|
||||||
@@ -205,6 +211,26 @@ mod tests {
|
|||||||
assert_eq!(result.quota_limit, None);
|
assert_eq!(result.quota_limit, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn quota_usage_rejects_an_unknown_mutation_baseline() {
|
||||||
|
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||||
|
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
|
||||||
|
let checker = QuotaChecker::new(sys);
|
||||||
|
let bucket = format!("quota-unknown-{}", Uuid::new_v4().simple());
|
||||||
|
|
||||||
|
crate::data_usage::record_bucket_object_write_memory(&bucket, None, 42).await;
|
||||||
|
let result = checker.get_real_time_usage(&bucket).await;
|
||||||
|
crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
|
||||||
|
.await
|
||||||
|
.expect("test usage cache cleanup should succeed");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
matches!(result, Err(QuotaError::UsageUnavailable { bucket: failed_bucket }) if failed_bucket == bucket),
|
||||||
|
"quota decisions must fail closed without an authoritative usage baseline"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_quota_check_no_limit() {
|
async fn test_quota_check_no_limit() {
|
||||||
let result = QuotaCheckResult {
|
let result = QuotaCheckResult {
|
||||||
|
|||||||
@@ -110,6 +110,8 @@ pub enum QuotaError {
|
|||||||
QuotaExceeded { current: u64, limit: u64, operation: u64 },
|
QuotaExceeded { current: u64, limit: u64, operation: u64 },
|
||||||
#[error("Quota configuration not found for bucket: {bucket}")]
|
#[error("Quota configuration not found for bucket: {bucket}")]
|
||||||
ConfigNotFound { bucket: String },
|
ConfigNotFound { bucket: String },
|
||||||
|
#[error("Authoritative data usage is unavailable for bucket: {bucket}")]
|
||||||
|
UsageUnavailable { bucket: String },
|
||||||
#[error("Invalid quota configuration: {reason}")]
|
#[error("Invalid quota configuration: {reason}")]
|
||||||
InvalidConfig { reason: String },
|
InvalidConfig { reason: String },
|
||||||
#[error("Storage error: {0}")]
|
#[error("Storage error: {0}")]
|
||||||
@@ -155,7 +157,7 @@ impl QuotaErrorResponse {
|
|||||||
request_id: request_id.to_string(),
|
request_id: request_id.to_string(),
|
||||||
host_id: host_id.to_string(),
|
host_id: host_id.to_string(),
|
||||||
},
|
},
|
||||||
QuotaError::StorageError(_) => Self {
|
QuotaError::UsageUnavailable { .. } | QuotaError::StorageError(_) => Self {
|
||||||
code: QUOTA_INTERNAL_ERROR_CODE.to_string(),
|
code: QUOTA_INTERNAL_ERROR_CODE.to_string(),
|
||||||
message: quota_error.to_string(),
|
message: quota_error.to_string(),
|
||||||
resource: QUOTA_API_PATH.to_string(),
|
resource: QUOTA_API_PATH.to_string(),
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ use std::{
|
|||||||
time::{Duration, SystemTime},
|
time::{Duration, SystemTime},
|
||||||
};
|
};
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::{Mutex as TokioMutex, RwLock};
|
||||||
use tracing::{debug, error, info, instrument};
|
use tracing::{debug, error, info, instrument};
|
||||||
|
|
||||||
// Data usage storage constants
|
// Data usage storage constants
|
||||||
@@ -63,12 +63,17 @@ const DATA_USAGE_REMOVE_CAS_RETRIES: usize = 3;
|
|||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
struct CachedBucketUsage {
|
struct CachedBucketUsage {
|
||||||
usage: BucketUsageInfo,
|
usage: BucketUsageInfo,
|
||||||
|
authoritative: bool,
|
||||||
refreshed_at: SystemTime,
|
refreshed_at: SystemTime,
|
||||||
usage_updated_at: SystemTime,
|
usage_updated_at: SystemTime,
|
||||||
// Set by request-path mutations until a scanner snapshot catches up to the same core counts.
|
// Set by request-path mutations until a scanner snapshot catches up to the same core counts.
|
||||||
dirty: bool,
|
dirty: bool,
|
||||||
// Set when a newer scanner snapshot was observed but did not include the dirty counts yet.
|
// Set when a newer scanner snapshot was observed but did not include the dirty counts yet.
|
||||||
stale_snapshot_pending: bool,
|
stale_snapshot_pending: bool,
|
||||||
|
// First complete scanner generation observed after an unknown-baseline
|
||||||
|
// mutation. A strictly later generation is required before the mutation
|
||||||
|
// evidence can be discarded.
|
||||||
|
pending_scanner_position: Option<(u64, u64)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
type UsageMemoryCache = Arc<RwLock<HashMap<String, CachedBucketUsage>>>;
|
type UsageMemoryCache = Arc<RwLock<HashMap<String, CachedBucketUsage>>>;
|
||||||
@@ -78,6 +83,7 @@ type LiveBucketUsageCache = moka::future::Cache<String, BucketUsageInfo>;
|
|||||||
static USAGE_MEMORY_CACHE: OnceLock<UsageMemoryCache> = OnceLock::new();
|
static USAGE_MEMORY_CACHE: OnceLock<UsageMemoryCache> = OnceLock::new();
|
||||||
static USAGE_CACHE_UPDATING: OnceLock<CacheUpdating> = OnceLock::new();
|
static USAGE_CACHE_UPDATING: OnceLock<CacheUpdating> = OnceLock::new();
|
||||||
static LIVE_BUCKET_USAGE_CACHE: OnceLock<LiveBucketUsageCache> = OnceLock::new();
|
static LIVE_BUCKET_USAGE_CACHE: OnceLock<LiveBucketUsageCache> = OnceLock::new();
|
||||||
|
static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||||
|
|
||||||
/// Cached copy of the last persisted data usage snapshot, served to admin
|
/// Cached copy of the last persisted data usage snapshot, served to admin
|
||||||
/// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
|
/// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
|
||||||
@@ -110,8 +116,13 @@ fn cache_data_usage_snapshot_result(
|
|||||||
cache: &mut Option<CachedDataUsageSnapshot>,
|
cache: &mut Option<CachedDataUsageSnapshot>,
|
||||||
result: Result<DataUsageInfo, Error>,
|
result: Result<DataUsageInfo, Error>,
|
||||||
loaded_at: tokio::time::Instant,
|
loaded_at: tokio::time::Instant,
|
||||||
) -> Result<DataUsageInfo, Error> {
|
refresh_generation: u64,
|
||||||
match result {
|
) -> Option<Result<DataUsageInfo, Error>> {
|
||||||
|
if data_usage_snapshot_generation() != refresh_generation {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(match result {
|
||||||
Ok(info) => {
|
Ok(info) => {
|
||||||
*cache = Some(CachedDataUsageSnapshot {
|
*cache = Some(CachedDataUsageSnapshot {
|
||||||
info: Some(info.clone()),
|
info: Some(info.clone()),
|
||||||
@@ -123,12 +134,14 @@ fn cache_data_usage_snapshot_result(
|
|||||||
*cache = Some(CachedDataUsageSnapshot { info: None, loaded_at });
|
*cache = Some(CachedDataUsageSnapshot { info: None, loaded_at });
|
||||||
Err(e)
|
Err(e)
|
||||||
}
|
}
|
||||||
}
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
type DataUsageSnapshotCache = Arc<RwLock<Option<CachedDataUsageSnapshot>>>;
|
type DataUsageSnapshotCache = Arc<RwLock<Option<CachedDataUsageSnapshot>>>;
|
||||||
|
|
||||||
static DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
|
static DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
|
||||||
|
static DATA_USAGE_SNAPSHOT_REFRESH: OnceLock<Arc<TokioMutex<()>>> = OnceLock::new();
|
||||||
|
static DATA_USAGE_SNAPSHOT_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||||
|
|
||||||
// Always-on revert detector for rustfs/backlog#1306: one relaxed increment per
|
// Always-on revert detector for rustfs/backlog#1306: one relaxed increment per
|
||||||
// full-bucket version listing is negligible and lets tests prove that admin
|
// full-bucket version listing is negligible and lets tests prove that admin
|
||||||
@@ -183,6 +196,15 @@ fn data_usage_snapshot_cache() -> &'static DataUsageSnapshotCache {
|
|||||||
DATA_USAGE_SNAPSHOT_CACHE.get_or_init(|| Arc::new(RwLock::new(None)))
|
DATA_USAGE_SNAPSHOT_CACHE.get_or_init(|| Arc::new(RwLock::new(None)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn data_usage_snapshot_generation() -> u64 {
|
||||||
|
DATA_USAGE_SNAPSHOT_GENERATION.load(Ordering::Acquire)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clear_data_usage_snapshot_cache(cache: &mut Option<CachedDataUsageSnapshot>) {
|
||||||
|
DATA_USAGE_SNAPSHOT_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||||
|
*cache = None;
|
||||||
|
}
|
||||||
|
|
||||||
fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
|
fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
|
||||||
LIVE_BUCKET_USAGE_CACHE.get_or_init(|| {
|
LIVE_BUCKET_USAGE_CACHE.get_or_init(|| {
|
||||||
moka::future::Cache::builder()
|
moka::future::Cache::builder()
|
||||||
@@ -191,6 +213,10 @@ fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn usage_memory_generation() -> u64 {
|
||||||
|
USAGE_MEMORY_GENERATION.load(Ordering::Acquire)
|
||||||
|
}
|
||||||
|
|
||||||
// Data usage storage paths
|
// Data usage storage paths
|
||||||
lazy_static::lazy_static! {
|
lazy_static::lazy_static! {
|
||||||
pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}",
|
pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}",
|
||||||
@@ -245,9 +271,21 @@ fn stale_data_usage_persist_reason(incoming: &DataUsageInfo, existing: &DataUsag
|
|||||||
enum UsageSnapshotSource {
|
enum UsageSnapshotSource {
|
||||||
Primary,
|
Primary,
|
||||||
Backup,
|
Backup,
|
||||||
|
LegacyPrimary,
|
||||||
|
LegacyBackup,
|
||||||
Missing,
|
Missing,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl UsageSnapshotSource {
|
||||||
|
fn is_authoritative(self) -> bool {
|
||||||
|
matches!(self, Self::Primary | Self::Backup)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_backup(self) -> bool {
|
||||||
|
matches!(self, Self::Backup | Self::LegacyBackup)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn stale_data_usage_persist_reason_for_source(
|
fn stale_data_usage_persist_reason_for_source(
|
||||||
incoming: &DataUsageInfo,
|
incoming: &DataUsageInfo,
|
||||||
existing: &DataUsageInfo,
|
existing: &DataUsageInfo,
|
||||||
@@ -255,7 +293,7 @@ fn stale_data_usage_persist_reason_for_source(
|
|||||||
now: SystemTime,
|
now: SystemTime,
|
||||||
) -> Option<&'static str> {
|
) -> Option<&'static str> {
|
||||||
let reason = stale_data_usage_persist_reason(incoming, existing, now);
|
let reason = stale_data_usage_persist_reason(incoming, existing, now);
|
||||||
if source == UsageSnapshotSource::Backup && incoming.last_update == existing.last_update {
|
if source.is_backup() && incoming.last_update == existing.last_update {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
reason
|
reason
|
||||||
@@ -267,6 +305,7 @@ fn stale_data_usage_persist_reason_for_source(
|
|||||||
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
|
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
|
||||||
// Prevent older data from overwriting newer persisted stats
|
// Prevent older data from overwriting newer persisted stats
|
||||||
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
|
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
|
||||||
|
&& source.is_authoritative()
|
||||||
&& let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
|
&& let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
|
||||||
{
|
{
|
||||||
info!(
|
info!(
|
||||||
@@ -292,7 +331,7 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
|
|||||||
// next request instead of waiting out the remaining TTL. The next cached
|
// next request instead of waiting out the remaining TTL. The next cached
|
||||||
// read reloads through `load_data_usage_from_backend`, keeping its
|
// read reloads through `load_data_usage_from_backend`, keeping its
|
||||||
// backward-compatibility post-processing.
|
// backward-compatibility post-processing.
|
||||||
*data_usage_snapshot_cache().write().await = None;
|
invalidate_data_usage_snapshot_cache().await;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -343,12 +382,13 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
|
|||||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||||
) -> Result<(), Error> {
|
) -> Result<(), Error> {
|
||||||
ensure_bucket_namespace_guard(guard, bucket, "data usage cache cleanup")?;
|
ensure_bucket_namespace_guard(guard, bucket, "data usage cache cleanup")?;
|
||||||
|
let _ = USAGE_MEMORY_GENERATION.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)));
|
||||||
live_bucket_usage_cache().invalidate(bucket).await;
|
live_bucket_usage_cache().invalidate(bucket).await;
|
||||||
clear_bucket_usage_memory(bucket, guard).await?;
|
clear_bucket_usage_memory(bucket, guard).await?;
|
||||||
|
|
||||||
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
|
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
|
||||||
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache cleanup")?;
|
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache cleanup")?;
|
||||||
*snapshot_cache = None;
|
clear_data_usage_snapshot_cache(&mut snapshot_cache);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -363,7 +403,7 @@ where
|
|||||||
let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
|
let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
|
||||||
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
|
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
|
||||||
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
|
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
|
||||||
*snapshot_cache = None;
|
clear_data_usage_snapshot_cache(&mut snapshot_cache);
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -386,7 +426,9 @@ where
|
|||||||
.filter(|etag| !etag.is_empty())
|
.filter(|etag| !etag.is_empty())
|
||||||
.map(str::to_owned)
|
.map(str::to_owned)
|
||||||
.ok_or_else(|| Error::other("data usage snapshot has no ETag"))?;
|
.ok_or_else(|| Error::other("data usage snapshot has no ETag"))?;
|
||||||
let data_usage_info = normalize_loaded_data_usage(parse_usage_snapshot(&reader.read_all().await?)?).await;
|
let mut data_usage_info = parse_usage_snapshot(&reader.read_all().await?)?;
|
||||||
|
populate_backward_compatible_usage_maps(&mut data_usage_info);
|
||||||
|
validate_complete_usage_snapshot(&mut data_usage_info);
|
||||||
Ok(Some((data_usage_info, revision)))
|
Ok(Some((data_usage_info, revision)))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -468,12 +510,15 @@ async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<DataUs
|
|||||||
where
|
where
|
||||||
S: EcstoreObjectIO + ?Sized,
|
S: EcstoreObjectIO + ?Sized,
|
||||||
{
|
{
|
||||||
for object in [
|
for (object, authoritative) in [
|
||||||
DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
|
(DATA_USAGE_OBJ_BACKUP_PATH.as_str(), true),
|
||||||
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
(LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), false),
|
||||||
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
|
(LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(), false),
|
||||||
] {
|
] {
|
||||||
if let Some((data_usage_info, _)) = load_data_usage_for_bucket_removal(store, object).await? {
|
if let Some((mut data_usage_info, _)) = load_data_usage_for_bucket_removal(store, object).await? {
|
||||||
|
if !authoritative {
|
||||||
|
data_usage_info.usage_snapshot_complete = false;
|
||||||
|
}
|
||||||
return Ok(data_usage_info);
|
return Ok(data_usage_info);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -692,13 +737,21 @@ async fn load_data_usage_snapshot_from_store<S: EcstoreObjectIO>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let legacy_primary = read_config_preserve_empty(store.clone(), &LEGACY_DATA_USAGE_OBJ_NAME_PATH).await;
|
let legacy_primary = read_config_preserve_empty(store.clone(), &LEGACY_DATA_USAGE_OBJ_NAME_PATH).await;
|
||||||
resolve_loaded_snapshot_pair_with_source(
|
let legacy = resolve_loaded_snapshot_pair_with_source(
|
||||||
legacy_primary,
|
legacy_primary,
|
||||||
async move { read_config_preserve_empty(store, &LEGACY_DATA_USAGE_OBJ_BACKUP_PATH).await },
|
async move { read_config_preserve_empty(store, &LEGACY_DATA_USAGE_OBJ_BACKUP_PATH).await },
|
||||||
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||||
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
|
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
|
||||||
)
|
)
|
||||||
.await
|
.await?;
|
||||||
|
Ok((
|
||||||
|
legacy.0,
|
||||||
|
match legacy.1 {
|
||||||
|
UsageSnapshotSource::Primary => UsageSnapshotSource::LegacyPrimary,
|
||||||
|
UsageSnapshotSource::Backup => UsageSnapshotSource::LegacyBackup,
|
||||||
|
source => source,
|
||||||
|
},
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo, UsageSnapshotSource), Error> {
|
async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo, UsageSnapshotSource), Error> {
|
||||||
@@ -708,13 +761,27 @@ async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo,
|
|||||||
/// Load data usage info from backend storage
|
/// Load data usage info from backend storage
|
||||||
#[instrument(skip(store))]
|
#[instrument(skip(store))]
|
||||||
pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||||
Ok(normalize_loaded_data_usage(load_data_usage_snapshot(store).await?.0).await)
|
let (data_usage_info, source) = load_data_usage_snapshot(store).await?;
|
||||||
|
Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo) -> DataUsageInfo {
|
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
|
||||||
info!("Loaded data usage info from backend with {} buckets", data_usage_info.buckets_count);
|
if !data_usage_info.is_complete_bucket_usage_snapshot() {
|
||||||
|
data_usage_info.usage_snapshot_complete = false;
|
||||||
|
data_usage_info.buckets_usage.clear();
|
||||||
|
data_usage_info.bucket_sizes.clear();
|
||||||
|
data_usage_info.buckets_count = 0;
|
||||||
|
data_usage_info.calculate_totals();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Handle backward compatibility
|
fn validate_complete_usage_snapshot(data_usage_info: &mut DataUsageInfo) {
|
||||||
|
if data_usage_info.usage_snapshot_complete && !data_usage_info.is_complete_bucket_usage_snapshot() {
|
||||||
|
data_usage_info.usage_snapshot_complete = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn populate_backward_compatible_usage_maps(data_usage_info: &mut DataUsageInfo) {
|
||||||
if data_usage_info.buckets_usage.is_empty() {
|
if data_usage_info.buckets_usage.is_empty() {
|
||||||
data_usage_info.buckets_usage = data_usage_info
|
data_usage_info.buckets_usage = data_usage_info
|
||||||
.bucket_sizes
|
.bucket_sizes
|
||||||
@@ -738,6 +805,17 @@ async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo) -> Data
|
|||||||
.map(|(bucket, bui)| (bucket.clone(), bui.size))
|
.map(|(bucket, bui)| (bucket.clone(), bui.size))
|
||||||
.collect();
|
.collect();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo, authoritative_format: bool) -> DataUsageInfo {
|
||||||
|
info!("Loaded data usage info from backend with {} buckets", data_usage_info.buckets_count);
|
||||||
|
|
||||||
|
if !authoritative_format {
|
||||||
|
data_usage_info.usage_snapshot_complete = false;
|
||||||
|
}
|
||||||
|
populate_backward_compatible_usage_maps(&mut data_usage_info);
|
||||||
|
validate_complete_usage_snapshot(&mut data_usage_info);
|
||||||
|
discard_incomplete_bucket_usage(&mut data_usage_info);
|
||||||
|
|
||||||
// Handle replication info
|
// Handle replication info
|
||||||
for (bucket, bui) in &data_usage_info.buckets_usage {
|
for (bucket, bui) in &data_usage_info.buckets_usage {
|
||||||
@@ -774,22 +852,43 @@ async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo) -> Data
|
|||||||
pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||||
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||||
|
|
||||||
{
|
loop {
|
||||||
let cache = data_usage_snapshot_cache().read().await;
|
{
|
||||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
let cache = data_usage_snapshot_cache().read().await;
|
||||||
|
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Serialize refreshes without holding the cache lock across backend I/O.
|
||||||
|
let refresh_guard = DATA_USAGE_SNAPSHOT_REFRESH
|
||||||
|
.get_or_init(|| Arc::new(TokioMutex::new(())))
|
||||||
|
.lock()
|
||||||
|
.await;
|
||||||
|
{
|
||||||
|
let cache = data_usage_snapshot_cache().read().await;
|
||||||
|
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let refresh_generation = data_usage_snapshot_generation();
|
||||||
|
let result = load_data_usage_from_backend(store.clone()).await;
|
||||||
|
let loaded_at = tokio::time::Instant::now();
|
||||||
|
let mut cache = data_usage_snapshot_cache().write().await;
|
||||||
|
if let Some(result) = cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation) {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
drop(cache);
|
||||||
|
drop(refresh_guard);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Re-check under the write lock so concurrent expirations trigger a single
|
/// Invalidate the process-local persisted usage snapshot cache after a durable
|
||||||
// backend read instead of a stampede.
|
/// scanner write.
|
||||||
|
pub async fn invalidate_data_usage_snapshot_cache() {
|
||||||
let mut cache = data_usage_snapshot_cache().write().await;
|
let mut cache = data_usage_snapshot_cache().write().await;
|
||||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
clear_data_usage_snapshot_cache(&mut cache);
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
let result = load_data_usage_from_backend(store).await;
|
|
||||||
cache_data_usage_snapshot_result(&mut cache, result, tokio::time::Instant::now())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Aggregate usage information from local disk snapshots.
|
/// Aggregate usage information from local disk snapshots.
|
||||||
@@ -1114,8 +1213,12 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn apply_live_bucket_usage_to_response(data_usage_info: &mut DataUsageInfo, bucket: &str, usage: &BucketUsageInfo) {
|
fn apply_live_bucket_usage_to_response(data_usage_info: &mut DataUsageInfo, bucket: &str, usage: &BucketUsageInfo) {
|
||||||
|
let inserted_bucket = !data_usage_info.buckets_usage.contains_key(bucket);
|
||||||
data_usage_info.bucket_sizes.insert(bucket.to_string(), usage.size);
|
data_usage_info.bucket_sizes.insert(bucket.to_string(), usage.size);
|
||||||
data_usage_info.buckets_usage.insert(bucket.to_string(), usage.clone());
|
data_usage_info.buckets_usage.insert(bucket.to_string(), usage.clone());
|
||||||
|
if inserted_bucket {
|
||||||
|
data_usage_info.usage_snapshot_complete = false;
|
||||||
|
}
|
||||||
set_buckets_count_from_usage(data_usage_info);
|
set_buckets_count_from_usage(data_usage_info);
|
||||||
data_usage_info.calculate_totals();
|
data_usage_info.calculate_totals();
|
||||||
}
|
}
|
||||||
@@ -1180,13 +1283,15 @@ async fn ensure_bucket_usage_cached(bucket: &str) {
|
|||||||
update_usage_cache_if_needed().await;
|
update_usage_cache_if_needed().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTime) -> CachedBucketUsage {
|
fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTime, authoritative: bool) -> CachedBucketUsage {
|
||||||
CachedBucketUsage {
|
CachedBucketUsage {
|
||||||
usage,
|
usage,
|
||||||
|
authoritative,
|
||||||
refreshed_at: SystemTime::now(),
|
refreshed_at: SystemTime::now(),
|
||||||
usage_updated_at: updated_at,
|
usage_updated_at: updated_at,
|
||||||
dirty: false,
|
dirty: false,
|
||||||
stale_snapshot_pending: false,
|
stale_snapshot_pending: false,
|
||||||
|
pending_scanner_position: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1194,10 +1299,12 @@ fn cached_bucket_usage_now(usage: BucketUsageInfo) -> CachedBucketUsage {
|
|||||||
let now = SystemTime::now();
|
let now = SystemTime::now();
|
||||||
CachedBucketUsage {
|
CachedBucketUsage {
|
||||||
usage,
|
usage,
|
||||||
|
authoritative: false,
|
||||||
refreshed_at: now,
|
refreshed_at: now,
|
||||||
usage_updated_at: now,
|
usage_updated_at: now,
|
||||||
dirty: false,
|
dirty: false,
|
||||||
stale_snapshot_pending: false,
|
stale_snapshot_pending: false,
|
||||||
|
pending_scanner_position: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1212,6 +1319,30 @@ fn bucket_usage_counts_match(left: &BucketUsageInfo, right: &BucketUsageInfo) ->
|
|||||||
&& left.delete_markers_count == right.delete_markers_count
|
&& left.delete_markers_count == right.delete_markers_count
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn preserve_unknown_dirty_usage(
|
||||||
|
existing: &CachedBucketUsage,
|
||||||
|
snapshot_position: Option<(u64, u64)>,
|
||||||
|
snapshot_update: SystemTime,
|
||||||
|
) -> Option<CachedBucketUsage> {
|
||||||
|
if existing.authoritative || !existing.dirty {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if existing
|
||||||
|
.pending_scanner_position
|
||||||
|
.zip(snapshot_position)
|
||||||
|
.is_some_and(|(previous, current)| current.0 > previous.0 || (current.0 == previous.0 && current.1 > previous.1))
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut preserved = existing.clone();
|
||||||
|
preserved.stale_snapshot_pending = true;
|
||||||
|
if preserved.pending_scanner_position.is_none() && snapshot_update >= existing.usage_updated_at {
|
||||||
|
preserved.pending_scanner_position = snapshot_position;
|
||||||
|
}
|
||||||
|
Some(preserved)
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: BucketUsageInfo, refresh_started_at: SystemTime) {
|
async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: BucketUsageInfo, refresh_started_at: SystemTime) {
|
||||||
let mut cache = memory_cache().write().await;
|
let mut cache = memory_cache().write().await;
|
||||||
@@ -1221,7 +1352,7 @@ async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: Buc
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
cache.insert(bucket.to_string(), cached_bucket_usage_from_backend(usage, refresh_started_at));
|
cache.insert(bucket.to_string(), cached_bucket_usage_from_backend(usage, refresh_started_at, true));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fast in-memory update for immediate quota and admin usage consistency.
|
/// Fast in-memory update for immediate quota and admin usage consistency.
|
||||||
@@ -1271,6 +1402,7 @@ async fn record_bucket_object_write_memory_inner(
|
|||||||
entry.usage_updated_at = now;
|
entry.usage_updated_at = now;
|
||||||
entry.dirty = true;
|
entry.dirty = true;
|
||||||
entry.stale_snapshot_pending = false;
|
entry.stale_snapshot_pending = false;
|
||||||
|
entry.pending_scanner_position = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Degraded in-memory update for an object write whose previous current size
|
/// Degraded in-memory update for an object write whose previous current size
|
||||||
@@ -1299,6 +1431,7 @@ pub async fn record_bucket_object_write_unknown_previous_memory(bucket: &str, ne
|
|||||||
entry.usage_updated_at = now;
|
entry.usage_updated_at = now;
|
||||||
entry.dirty = true;
|
entry.dirty = true;
|
||||||
entry.stale_snapshot_pending = false;
|
entry.stale_snapshot_pending = false;
|
||||||
|
entry.pending_scanner_position = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fast in-memory increment for immediate quota consistency.
|
/// Fast in-memory increment for immediate quota consistency.
|
||||||
@@ -1326,6 +1459,7 @@ pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64,
|
|||||||
entry.usage_updated_at = now;
|
entry.usage_updated_at = now;
|
||||||
entry.dirty = true;
|
entry.dirty = true;
|
||||||
entry.stale_snapshot_pending = false;
|
entry.stale_snapshot_pending = false;
|
||||||
|
entry.pending_scanner_position = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fast in-memory update for successful delete marker creation.
|
/// Fast in-memory update for successful delete marker creation.
|
||||||
@@ -1344,6 +1478,7 @@ pub async fn record_bucket_delete_marker_memory(bucket: &str) {
|
|||||||
entry.usage_updated_at = now;
|
entry.usage_updated_at = now;
|
||||||
entry.dirty = true;
|
entry.dirty = true;
|
||||||
entry.stale_snapshot_pending = false;
|
entry.stale_snapshot_pending = false;
|
||||||
|
entry.pending_scanner_position = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fast in-memory decrement for immediate quota consistency
|
/// Fast in-memory decrement for immediate quota consistency
|
||||||
@@ -1354,8 +1489,8 @@ pub async fn decrement_bucket_usage_memory(bucket: &str, size_decrement: u64) {
|
|||||||
/// Get bucket usage from the authoritative cache for this topology.
|
/// Get bucket usage from the authoritative cache for this topology.
|
||||||
async fn get_persisted_bucket_usage(bucket: &str) -> Option<u64> {
|
async fn get_persisted_bucket_usage(bucket: &str) -> Option<u64> {
|
||||||
let store = runtime_sources::object_store_handle()?;
|
let store = runtime_sources::object_store_handle()?;
|
||||||
let data_usage_info = load_data_usage_from_backend_cached(store).await.ok()?;
|
let info = load_data_usage_from_backend_cached(store).await.ok()?;
|
||||||
data_usage_info.buckets_usage.get(bucket).map(|usage| usage.size)
|
info.buckets_usage.get(bucket).map(|usage| usage.size)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
|
pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
|
||||||
@@ -1371,7 +1506,10 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
|
|||||||
update_usage_cache_if_needed().await;
|
update_usage_cache_if_needed().await;
|
||||||
|
|
||||||
let cache = memory_cache().read().await;
|
let cache = memory_cache().read().await;
|
||||||
cache.get(bucket).map(|cached| cached.usage.size)
|
cache
|
||||||
|
.get(bucket)
|
||||||
|
.filter(|cached| cached.authoritative)
|
||||||
|
.map(|cached| cached.usage.size)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn update_usage_cache_if_needed() {
|
async fn update_usage_cache_if_needed() {
|
||||||
@@ -1401,11 +1539,12 @@ async fn update_usage_cache_if_needed() {
|
|||||||
drop(updating);
|
drop(updating);
|
||||||
|
|
||||||
let updating_clone = (*cache_updating()).clone();
|
let updating_clone = (*cache_updating()).clone();
|
||||||
|
let refresh_generation = usage_memory_generation();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if let Some(store) = runtime_sources::object_store_handle()
|
if let Some(store) = runtime_sources::object_store_handle()
|
||||||
&& let Ok(data_usage_info) = load_data_usage_from_backend(store).await
|
&& let Ok(data_usage_info) = load_data_usage_from_backend(store).await
|
||||||
{
|
{
|
||||||
replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
replace_bucket_usage_memory_from_info_if_generation(&data_usage_info, Some(refresh_generation)).await;
|
||||||
}
|
}
|
||||||
let mut updating = updating_clone.write().await;
|
let mut updating = updating_clone.write().await;
|
||||||
*updating = false;
|
*updating = false;
|
||||||
@@ -1426,10 +1565,11 @@ async fn update_usage_cache_if_needed() {
|
|||||||
*updating = true;
|
*updating = true;
|
||||||
drop(updating);
|
drop(updating);
|
||||||
|
|
||||||
|
let refresh_generation = usage_memory_generation();
|
||||||
if let Some(store) = runtime_sources::object_store_handle()
|
if let Some(store) = runtime_sources::object_store_handle()
|
||||||
&& let Ok(data_usage_info) = load_data_usage_from_backend(store).await
|
&& let Ok(data_usage_info) = load_data_usage_from_backend(store).await
|
||||||
{
|
{
|
||||||
replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
replace_bucket_usage_memory_from_info_if_generation(&data_usage_info, Some(refresh_generation)).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut updating = cache_updating().write().await;
|
let mut updating = cache_updating().write().await;
|
||||||
@@ -1437,40 +1577,57 @@ async fn update_usage_cache_if_needed() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn replace_bucket_usage_memory_from_info(data_usage_info: &DataUsageInfo) {
|
pub async fn replace_bucket_usage_memory_from_info(data_usage_info: &DataUsageInfo) {
|
||||||
let usage_updated_at = data_usage_info_updated_at(data_usage_info);
|
replace_bucket_usage_memory_from_info_if_generation(data_usage_info, None).await;
|
||||||
let mut cache = memory_cache().write().await;
|
}
|
||||||
let mut next_cache = HashMap::new();
|
|
||||||
|
|
||||||
for (bucket, bucket_usage) in data_usage_info.buckets_usage.iter() {
|
async fn replace_bucket_usage_memory_from_info_if_generation(data_usage_info: &DataUsageInfo, expected_generation: Option<u64>) {
|
||||||
if let Some(existing) = cache.get(bucket) {
|
if !data_usage_info.is_complete_bucket_usage_snapshot() {
|
||||||
if existing.usage_updated_at > usage_updated_at {
|
return;
|
||||||
next_cache.insert(bucket.clone(), existing.clone());
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if existing.dirty && !bucket_usage_counts_match(&existing.usage, bucket_usage) {
|
|
||||||
// A scanner snapshot can be saved after newer writes but still miss them if it listed the bucket earlier.
|
|
||||||
let mut preserved = existing.clone();
|
|
||||||
preserved.stale_snapshot_pending = true;
|
|
||||||
next_cache.insert(bucket.clone(), preserved);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
next_cache.insert(bucket.clone(), cached_bucket_usage_from_backend(bucket_usage.clone(), usage_updated_at));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for (bucket, existing) in cache.iter() {
|
let usage_updated_at = data_usage_info_updated_at(data_usage_info);
|
||||||
if !data_usage_info.buckets_usage.contains_key(bucket) {
|
let snapshot_position = data_usage_info.scanner_epoch.zip(data_usage_info.scanner_cycle);
|
||||||
if existing.usage_updated_at > usage_updated_at {
|
let mut next_cache = HashMap::with_capacity(data_usage_info.buckets_usage.len());
|
||||||
next_cache.insert(bucket.clone(), existing.clone());
|
for (bucket, bucket_usage) in data_usage_info.buckets_usage.iter() {
|
||||||
continue;
|
next_cache.insert(
|
||||||
}
|
bucket.clone(),
|
||||||
|
cached_bucket_usage_from_backend(bucket_usage.clone(), usage_updated_at, true),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if existing.dirty {
|
let mut cache = memory_cache().write().await;
|
||||||
let mut preserved = existing.clone();
|
if expected_generation.is_some_and(|expected| usage_memory_generation() != expected) {
|
||||||
preserved.stale_snapshot_pending = true;
|
return;
|
||||||
next_cache.insert(bucket.clone(), preserved);
|
}
|
||||||
|
for (bucket, existing) in cache.iter() {
|
||||||
|
match next_cache.entry(bucket.clone()) {
|
||||||
|
Entry::Occupied(mut candidate) => {
|
||||||
|
if let Some(preserved) = preserve_unknown_dirty_usage(existing, snapshot_position, usage_updated_at) {
|
||||||
|
candidate.insert(preserved);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if existing.authoritative && existing.usage_updated_at > usage_updated_at {
|
||||||
|
candidate.insert(existing.clone());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if existing.authoritative && existing.dirty && !bucket_usage_counts_match(&existing.usage, &candidate.get().usage)
|
||||||
|
{
|
||||||
|
// A scanner snapshot can be saved after newer writes but still miss them if it listed the bucket earlier.
|
||||||
|
let mut preserved = existing.clone();
|
||||||
|
preserved.stale_snapshot_pending = true;
|
||||||
|
candidate.insert(preserved);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Entry::Vacant(candidate) => {
|
||||||
|
if let Some(preserved) = preserve_unknown_dirty_usage(existing, snapshot_position, usage_updated_at) {
|
||||||
|
candidate.insert(preserved);
|
||||||
|
} else if existing.authoritative && existing.usage_updated_at > usage_updated_at {
|
||||||
|
candidate.insert(existing.clone());
|
||||||
|
} else if existing.authoritative && existing.dirty {
|
||||||
|
let mut preserved = existing.clone();
|
||||||
|
preserved.stale_snapshot_pending = true;
|
||||||
|
candidate.insert(preserved);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1490,18 +1647,29 @@ async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mu
|
|||||||
|
|
||||||
let persisted_update = data_usage_info.last_update;
|
let persisted_update = data_usage_info.last_update;
|
||||||
let mut changed = false;
|
let mut changed = false;
|
||||||
|
let mut added_bucket = false;
|
||||||
|
|
||||||
for (bucket, cached) in cache.iter() {
|
for (bucket, cached) in cache.iter() {
|
||||||
|
if !cached.authoritative {
|
||||||
|
if cached.dirty {
|
||||||
|
data_usage_info.usage_snapshot_complete = false;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if !cached.stale_snapshot_pending && persisted_update.is_some_and(|persisted| cached.usage_updated_at <= persisted) {
|
if !cached.stale_snapshot_pending && persisted_update.is_some_and(|persisted| cached.usage_updated_at <= persisted) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
added_bucket |= !data_usage_info.buckets_usage.contains_key(bucket);
|
||||||
data_usage_info.buckets_usage.insert(bucket.clone(), cached.usage.clone());
|
data_usage_info.buckets_usage.insert(bucket.clone(), cached.usage.clone());
|
||||||
data_usage_info.bucket_sizes.insert(bucket.clone(), cached.usage.size);
|
data_usage_info.bucket_sizes.insert(bucket.clone(), cached.usage.size);
|
||||||
changed = true;
|
changed = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if changed {
|
if changed {
|
||||||
|
if added_bucket {
|
||||||
|
data_usage_info.usage_snapshot_complete = false;
|
||||||
|
}
|
||||||
data_usage_info.buckets_count = data_usage_info.buckets_usage.len() as u64;
|
data_usage_info.buckets_count = data_usage_info.buckets_usage.len() as u64;
|
||||||
data_usage_info.calculate_totals();
|
data_usage_info.calculate_totals();
|
||||||
}
|
}
|
||||||
@@ -2061,6 +2229,7 @@ mod tests {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
info.usage_snapshot_complete = true;
|
||||||
info.bucket_sizes.insert(bucket.to_string(), size);
|
info.bucket_sizes.insert(bucket.to_string(), size);
|
||||||
info.buckets_count = info.buckets_usage.len() as u64;
|
info.buckets_count = info.buckets_usage.len() as u64;
|
||||||
info.calculate_totals();
|
info.calculate_totals();
|
||||||
@@ -2184,12 +2353,140 @@ mod tests {
|
|||||||
assert!(!err.to_string().contains("secret-marker"));
|
assert!(!err.to_string().contains("secret-marker"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn legacy_snapshot_bucket_usage_is_unknown_without_completeness_marker() {
|
||||||
|
let mut legacy = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
|
||||||
|
legacy.usage_snapshot_complete = true;
|
||||||
|
legacy.buckets_usage.insert("large".to_string(), BucketUsageInfo::default());
|
||||||
|
legacy.bucket_sizes.insert("large".to_string(), 0);
|
||||||
|
legacy.buckets_count = 2;
|
||||||
|
|
||||||
|
let normalized = normalize_loaded_data_usage(legacy, false).await;
|
||||||
|
|
||||||
|
assert_eq!(normalized.buckets_count, 0);
|
||||||
|
assert!(normalized.buckets_usage.is_empty());
|
||||||
|
assert!(normalized.bucket_sizes.is_empty());
|
||||||
|
assert_eq!(normalized.objects_total_count, 0);
|
||||||
|
assert_eq!(normalized.objects_total_size, 0);
|
||||||
|
assert!(!normalized.usage_snapshot_complete);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
fn legacy_snapshot_json_defaults_completeness_to_unknown() {
|
||||||
|
let info = data_usage_info_for_test("legacy", 1, 42, SystemTime::UNIX_EPOCH);
|
||||||
|
let mut value = serde_json::to_value(info).expect("serialize data usage fixture");
|
||||||
|
let object = value.as_object_mut().expect("data usage fixture should be a JSON object");
|
||||||
|
object.remove("usage_snapshot_complete");
|
||||||
|
|
||||||
|
let decoded: DataUsageInfo = serde_json::from_value(value).expect("deserialize legacy data usage fixture");
|
||||||
|
|
||||||
|
assert!(!decoded.usage_snapshot_complete);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn complete_snapshot_json_is_additive_for_legacy_readers() {
|
||||||
|
#[derive(serde::Deserialize)]
|
||||||
|
struct LegacyUsageReader {
|
||||||
|
buckets_count: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
let info = data_usage_info_for_test("current", 1, 42, SystemTime::UNIX_EPOCH);
|
||||||
|
let encoded = serde_json::to_vec(&info).expect("serialize current data usage fixture");
|
||||||
|
let legacy: LegacyUsageReader =
|
||||||
|
serde_json::from_slice(&encoded).expect("legacy JSON reader should ignore additive fields");
|
||||||
|
|
||||||
|
assert_eq!(legacy.buckets_count, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn complete_snapshot_preserves_confirmed_empty_bucket() {
|
||||||
|
let mut info = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
|
||||||
|
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
|
||||||
|
info.buckets_count = 2;
|
||||||
|
|
||||||
|
let normalized = normalize_loaded_data_usage(info, true).await;
|
||||||
|
|
||||||
|
assert_eq!(normalized.buckets_count, 2);
|
||||||
|
assert!(normalized.usage_snapshot_complete);
|
||||||
|
assert_eq!(
|
||||||
|
normalized
|
||||||
|
.buckets_usage
|
||||||
|
.get("control")
|
||||||
|
.map(|usage| (usage.objects_count, usage.size)),
|
||||||
|
Some((10, 10_285))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
normalized
|
||||||
|
.buckets_usage
|
||||||
|
.get("empty")
|
||||||
|
.map(|usage| (usage.objects_count, usage.size)),
|
||||||
|
Some((0, 0))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn malformed_complete_snapshot_is_rejected_as_a_whole() {
|
||||||
|
let mut info = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
|
||||||
|
info.buckets_usage.insert(
|
||||||
|
"partial".to_string(),
|
||||||
|
BucketUsageInfo {
|
||||||
|
objects_count: 1_502,
|
||||||
|
versions_count: 1_502,
|
||||||
|
size: 196_870_144,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
info.bucket_sizes.insert("partial".to_string(), 196_870_144);
|
||||||
|
info.buckets_count = 1;
|
||||||
|
|
||||||
|
let normalized = normalize_loaded_data_usage(info, true).await;
|
||||||
|
|
||||||
|
assert_eq!(normalized.buckets_count, 0);
|
||||||
|
assert!(!normalized.buckets_usage.contains_key("control"));
|
||||||
|
assert!(!normalized.buckets_usage.contains_key("partial"));
|
||||||
|
assert!(!normalized.bucket_sizes.contains_key("partial"));
|
||||||
|
assert!(!normalized.usage_snapshot_complete);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn completeness_marker_does_not_fabricate_a_missing_bucket() {
|
||||||
|
let mut info = data_usage_info_for_test("control", 10, 10_285, SystemTime::UNIX_EPOCH);
|
||||||
|
info.buckets_count = 2;
|
||||||
|
|
||||||
|
assert!(!data_usage_contains_bucket(&info, "missing"));
|
||||||
|
let normalized = normalize_loaded_data_usage(info, true).await;
|
||||||
|
|
||||||
|
assert!(!normalized.usage_snapshot_complete);
|
||||||
|
assert!(normalized.buckets_usage.is_empty());
|
||||||
|
assert!(!normalized.buckets_usage.contains_key("missing"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn complete_empty_snapshot_remains_authoritative() {
|
||||||
|
let normalized = normalize_loaded_data_usage(
|
||||||
|
DataUsageInfo {
|
||||||
|
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||||
|
usage_snapshot_complete: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(normalized.usage_snapshot_complete);
|
||||||
|
assert_eq!(normalized.buckets_count, 0);
|
||||||
|
assert!(normalized.buckets_usage.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
fn cached_snapshot_failure_is_reused_until_ttl_expires() {
|
fn cached_snapshot_failure_is_reused_until_ttl_expires() {
|
||||||
let loaded_at = tokio::time::Instant::now();
|
let loaded_at = tokio::time::Instant::now();
|
||||||
let mut cache = None;
|
let mut cache = None;
|
||||||
|
let refresh_generation = data_usage_snapshot_generation();
|
||||||
|
|
||||||
let first = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at);
|
let first = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at, refresh_generation)
|
||||||
|
.expect("an uninterrupted refresh should populate the cache");
|
||||||
assert!(matches!(first, Err(Error::ErasureReadQuorum)));
|
assert!(matches!(first, Err(Error::ErasureReadQuorum)));
|
||||||
|
|
||||||
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
||||||
@@ -2200,13 +2497,16 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
#[serial]
|
||||||
fn cached_snapshot_success_is_reused_until_ttl_expires() {
|
fn cached_snapshot_success_is_reused_until_ttl_expires() {
|
||||||
let loaded_at = tokio::time::Instant::now();
|
let loaded_at = tokio::time::Instant::now();
|
||||||
let expected = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH);
|
let expected = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH);
|
||||||
let mut cache = None;
|
let mut cache = None;
|
||||||
|
let refresh_generation = data_usage_snapshot_generation();
|
||||||
|
|
||||||
let first =
|
let first = cache_data_usage_snapshot_result(&mut cache, Ok(expected), loaded_at, refresh_generation)
|
||||||
cache_data_usage_snapshot_result(&mut cache, Ok(expected), loaded_at).expect("successful load must be returned");
|
.expect("an uninterrupted refresh should populate the cache")
|
||||||
|
.expect("successful load must be returned");
|
||||||
assert_snapshot_bucket(&first, "bucket");
|
assert_snapshot_bucket(&first, "bucket");
|
||||||
|
|
||||||
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
||||||
@@ -2215,6 +2515,31 @@ mod tests {
|
|||||||
assert_snapshot_bucket(&cached, "bucket");
|
assert_snapshot_bucket(&cached, "bucket");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn cache_invalidation_during_refresh_prevents_stale_snapshot_resurrection() {
|
||||||
|
let loaded_at = tokio::time::Instant::now();
|
||||||
|
let refresh_generation = data_usage_snapshot_generation();
|
||||||
|
let mut cache = Some(CachedDataUsageSnapshot {
|
||||||
|
info: Some(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||||
|
loaded_at,
|
||||||
|
});
|
||||||
|
clear_data_usage_snapshot_cache(&mut cache);
|
||||||
|
|
||||||
|
let stale_result = cache_data_usage_snapshot_result(
|
||||||
|
&mut cache,
|
||||||
|
Ok(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||||
|
loaded_at,
|
||||||
|
refresh_generation,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(stale_result.is_none());
|
||||||
|
assert!(
|
||||||
|
cache.is_none(),
|
||||||
|
"an in-flight load must not repopulate a cache invalidated after it started"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn resolve_snapshot_primary_ok_is_used_without_backup_read() {
|
async fn resolve_snapshot_primary_ok_is_used_without_backup_read() {
|
||||||
let backup_read = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
let backup_read = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||||
@@ -2312,7 +2637,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn authoritative_usage_loader_migrates_only_when_v2_is_absent() {
|
async fn authoritative_usage_loader_prefers_v2_over_legacy_input() {
|
||||||
let store = Arc::new(UsageCasStore {
|
let store = Arc::new(UsageCasStore {
|
||||||
state: Mutex::new(UsageCasState {
|
state: Mutex::new(UsageCasState {
|
||||||
legacy_object: Some((snapshot_bytes("legacy-bucket"), 1)),
|
legacy_object: Some((snapshot_bytes("legacy-bucket"), 1)),
|
||||||
@@ -2320,18 +2645,20 @@ mod tests {
|
|||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
let (legacy, _) = load_data_usage_snapshot_from_store(store.clone())
|
let (legacy, source) = load_data_usage_snapshot_from_store(store.clone())
|
||||||
.await
|
.await
|
||||||
.expect("a legacy snapshot should seed an upgrade when v2 is absent");
|
.expect("a legacy snapshot should seed an upgrade when newer formats are absent");
|
||||||
assert_snapshot_bucket(&legacy, "legacy-bucket");
|
assert_snapshot_bucket(&legacy, "legacy-bucket");
|
||||||
|
assert_eq!(source, UsageSnapshotSource::LegacyPrimary);
|
||||||
|
|
||||||
store.state.lock().await.object = Some((snapshot_bytes("v2-bucket"), 1));
|
store.state.lock().await.object = Some((snapshot_bytes("v2-bucket"), 2));
|
||||||
let (authoritative, _) = load_data_usage_snapshot_from_store(store.clone())
|
let (authoritative, source) = load_data_usage_snapshot_from_store(store.clone())
|
||||||
.await
|
.await
|
||||||
.expect("a v2 snapshot should be authoritative");
|
.expect("a v2 snapshot should supersede the legacy migration input");
|
||||||
assert_snapshot_bucket(&authoritative, "v2-bucket");
|
assert_snapshot_bucket(&authoritative, "v2-bucket");
|
||||||
|
assert_eq!(source, UsageSnapshotSource::Primary);
|
||||||
|
|
||||||
store.state.lock().await.object = Some((b"corrupt-v2".to_vec(), 2));
|
store.state.lock().await.object = Some((b"corrupt-v2".to_vec(), 3));
|
||||||
let err = load_data_usage_snapshot_from_store(store)
|
let err = load_data_usage_snapshot_from_store(store)
|
||||||
.await
|
.await
|
||||||
.expect_err("corrupt v2 state must not fall back to a legacy writer");
|
.expect_err("corrupt v2 state must not fall back to a legacy writer");
|
||||||
@@ -2686,6 +3013,140 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn memory_overlay_does_not_publish_delta_from_unknown_baseline() {
|
||||||
|
clear_usage_memory_cache_for_test().await;
|
||||||
|
|
||||||
|
let bucket = "unknown-baseline";
|
||||||
|
memory_cache().write().await.insert(
|
||||||
|
bucket.to_string(),
|
||||||
|
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
|
||||||
|
);
|
||||||
|
record_bucket_object_write_memory(bucket, None, 42).await;
|
||||||
|
|
||||||
|
let mut response = DataUsageInfo::default();
|
||||||
|
apply_bucket_usage_memory_overlay(&mut response).await;
|
||||||
|
|
||||||
|
assert!(!response.buckets_usage.contains_key(bucket));
|
||||||
|
assert_eq!(get_bucket_usage_memory(bucket).await, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn unknown_dirty_usage_requires_a_followup_scanner_generation() {
|
||||||
|
clear_usage_memory_cache_for_test().await;
|
||||||
|
|
||||||
|
let bucket = "unknown-baseline";
|
||||||
|
memory_cache().write().await.insert(
|
||||||
|
bucket.to_string(),
|
||||||
|
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
|
||||||
|
);
|
||||||
|
record_bucket_object_write_memory(bucket, None, 42).await;
|
||||||
|
let mutation_update = memory_cache()
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.get(bucket)
|
||||||
|
.expect("unknown usage mutation should remain cached")
|
||||||
|
.usage_updated_at;
|
||||||
|
|
||||||
|
let mut first_snapshot = data_usage_info_for_test(bucket, 10, 420, mutation_update + Duration::from_nanos(1));
|
||||||
|
first_snapshot.scanner_epoch = Some(7);
|
||||||
|
first_snapshot.scanner_cycle = Some(10);
|
||||||
|
replace_bucket_usage_memory_from_info(&first_snapshot).await;
|
||||||
|
|
||||||
|
let mut first_response = first_snapshot.clone();
|
||||||
|
apply_bucket_usage_memory_overlay_if_authoritative(&mut first_response, true).await;
|
||||||
|
assert!(
|
||||||
|
!first_response.usage_snapshot_complete,
|
||||||
|
"the first scanner generation after an unknown mutation cannot prove that it observed the mutation"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
first_response
|
||||||
|
.buckets_usage
|
||||||
|
.get(bucket)
|
||||||
|
.map(|usage| (usage.objects_count, usage.size)),
|
||||||
|
Some((10, 420)),
|
||||||
|
"unknown memory deltas must not be overlaid as absolute usage"
|
||||||
|
);
|
||||||
|
|
||||||
|
replace_bucket_usage_memory_from_info(&first_snapshot).await;
|
||||||
|
let mut repeated_response = first_snapshot.clone();
|
||||||
|
apply_bucket_usage_memory_overlay_if_authoritative(&mut repeated_response, true).await;
|
||||||
|
assert!(
|
||||||
|
!repeated_response.usage_snapshot_complete,
|
||||||
|
"reloading the same scanner generation must not clear unknown mutation evidence"
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut successor = data_usage_info_for_test(bucket, 11, 462, mutation_update + Duration::from_nanos(2));
|
||||||
|
successor.scanner_epoch = Some(7);
|
||||||
|
successor.scanner_cycle = Some(11);
|
||||||
|
replace_bucket_usage_memory_from_info(&successor).await;
|
||||||
|
|
||||||
|
let mut successor_response = successor;
|
||||||
|
apply_bucket_usage_memory_overlay_if_authoritative(&mut successor_response, true).await;
|
||||||
|
assert!(successor_response.is_complete_bucket_usage_snapshot());
|
||||||
|
assert_eq!(
|
||||||
|
successor_response
|
||||||
|
.buckets_usage
|
||||||
|
.get(bucket)
|
||||||
|
.map(|usage| (usage.objects_count, usage.size)),
|
||||||
|
Some((11, 462))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn a_new_unknown_mutation_restarts_the_scanner_generation_fence() {
|
||||||
|
clear_usage_memory_cache_for_test().await;
|
||||||
|
|
||||||
|
let bucket = "unknown-baseline";
|
||||||
|
memory_cache().write().await.insert(
|
||||||
|
bucket.to_string(),
|
||||||
|
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
|
||||||
|
);
|
||||||
|
record_bucket_object_write_memory(bucket, None, 42).await;
|
||||||
|
let first_mutation_update = memory_cache()
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.get(bucket)
|
||||||
|
.expect("unknown usage mutation should remain cached")
|
||||||
|
.usage_updated_at;
|
||||||
|
|
||||||
|
let mut first_snapshot = data_usage_info_for_test(bucket, 10, 420, first_mutation_update + Duration::from_nanos(1));
|
||||||
|
first_snapshot.scanner_epoch = Some(7);
|
||||||
|
first_snapshot.scanner_cycle = Some(10);
|
||||||
|
replace_bucket_usage_memory_from_info(&first_snapshot).await;
|
||||||
|
|
||||||
|
record_bucket_object_write_memory(bucket, None, 42).await;
|
||||||
|
let second_mutation_update = memory_cache()
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.get(bucket)
|
||||||
|
.expect("second unknown usage mutation should remain cached")
|
||||||
|
.usage_updated_at;
|
||||||
|
let mut next_snapshot = data_usage_info_for_test(bucket, 11, 462, second_mutation_update + Duration::from_nanos(1));
|
||||||
|
next_snapshot.scanner_epoch = Some(7);
|
||||||
|
next_snapshot.scanner_cycle = Some(11);
|
||||||
|
replace_bucket_usage_memory_from_info(&next_snapshot).await;
|
||||||
|
|
||||||
|
let mut next_response = next_snapshot.clone();
|
||||||
|
apply_bucket_usage_memory_overlay_if_authoritative(&mut next_response, true).await;
|
||||||
|
assert!(
|
||||||
|
!next_response.usage_snapshot_complete,
|
||||||
|
"a mutation after the first observation must require another scanner generation"
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut final_snapshot = data_usage_info_for_test(bucket, 12, 504, second_mutation_update + Duration::from_nanos(2));
|
||||||
|
final_snapshot.scanner_epoch = Some(7);
|
||||||
|
final_snapshot.scanner_cycle = Some(12);
|
||||||
|
replace_bucket_usage_memory_from_info(&final_snapshot).await;
|
||||||
|
|
||||||
|
let mut final_response = final_snapshot;
|
||||||
|
apply_bucket_usage_memory_overlay_if_authoritative(&mut final_response, true).await;
|
||||||
|
assert!(final_response.is_complete_bucket_usage_snapshot());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn remove_bucket_usage_from_info_drops_bucket_and_recomputes_totals() {
|
fn remove_bucket_usage_from_info_drops_bucket_and_recomputes_totals() {
|
||||||
let last_update = SystemTime::now();
|
let last_update = SystemTime::now();
|
||||||
@@ -3109,7 +3570,7 @@ mod tests {
|
|||||||
"fresh successor snapshot cache must not be erased after namespace lock loss"
|
"fresh successor snapshot cache must not be erased after namespace lock loss"
|
||||||
);
|
);
|
||||||
drop(snapshot_cache);
|
drop(snapshot_cache);
|
||||||
*data_usage_snapshot_cache().write().await = None;
|
invalidate_data_usage_snapshot_cache().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
@@ -3168,7 +3629,7 @@ mod tests {
|
|||||||
"fresh successor snapshot cache must not be erased by final invalidation after lock loss"
|
"fresh successor snapshot cache must not be erased by final invalidation after lock loss"
|
||||||
);
|
);
|
||||||
drop(snapshot_cache);
|
drop(snapshot_cache);
|
||||||
*data_usage_snapshot_cache().write().await = None;
|
invalidate_data_usage_snapshot_cache().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -3255,6 +3716,7 @@ mod tests {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
legacy.bucket_sizes.insert("bucket-b".to_string(), 126);
|
legacy.bucket_sizes.insert("bucket-b".to_string(), 126);
|
||||||
|
legacy.usage_snapshot_complete = false;
|
||||||
legacy.buckets_count = 2;
|
legacy.buckets_count = 2;
|
||||||
legacy.calculate_totals();
|
legacy.calculate_totals();
|
||||||
let store = Arc::new(UsageCasStore {
|
let store = Arc::new(UsageCasStore {
|
||||||
@@ -3288,6 +3750,13 @@ mod tests {
|
|||||||
Some((3, 126))
|
Some((3, 126))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
for (data, _) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() {
|
||||||
|
let saved = serde_json::from_slice::<DataUsageInfo>(data).expect("migrated usage snapshot should decode");
|
||||||
|
assert!(
|
||||||
|
!saved.usage_snapshot_complete,
|
||||||
|
"legacy usage must not be promoted to an authoritative snapshot"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -3428,6 +3897,7 @@ mod tests {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
backup.bucket_sizes.insert("bucket-b".to_string(), 126);
|
backup.bucket_sizes.insert("bucket-b".to_string(), 126);
|
||||||
|
backup.buckets_count = 2;
|
||||||
backup.calculate_totals();
|
backup.calculate_totals();
|
||||||
let store = Arc::new(UsageCasStore {
|
let store = Arc::new(UsageCasStore {
|
||||||
state: Mutex::new(UsageCasState {
|
state: Mutex::new(UsageCasState {
|
||||||
|
|||||||
@@ -57,14 +57,14 @@ fn apply_data_usage_result(
|
|||||||
usage: &mut rustfs_madmin::Usage,
|
usage: &mut rustfs_madmin::Usage,
|
||||||
) {
|
) {
|
||||||
match result {
|
match result {
|
||||||
Ok(info) => {
|
Ok(info) if info.is_complete_bucket_usage_snapshot() => {
|
||||||
buckets.count = info.buckets_count;
|
buckets.count = info.buckets_count;
|
||||||
objects.count = info.objects_total_count;
|
objects.count = info.objects_total_count;
|
||||||
versions.count = info.versions_total_count;
|
versions.count = info.versions_total_count;
|
||||||
delete_markers.count = info.delete_markers_total_count;
|
delete_markers.count = info.delete_markers_total_count;
|
||||||
usage.size = info.objects_total_size;
|
usage.size = info.objects_total_size;
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Ok(_) | Err(_) => {
|
||||||
buckets.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
|
buckets.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
|
||||||
objects.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
|
objects.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
|
||||||
versions.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
|
versions.error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
|
||||||
@@ -649,18 +649,15 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
|
|||||||
|
|
||||||
if erasure_set.id == 0 {
|
if erasure_set.id == 0 {
|
||||||
erasure_set.id = d.set_index;
|
erasure_set.id = d.set_index;
|
||||||
if let Ok(cache) = load_data_usage_cache(
|
match load_data_usage_cache(
|
||||||
&store.pools[d.pool_index as usize].disk_set[d.set_index as usize].clone(),
|
&store.pools[d.pool_index as usize].disk_set[d.set_index as usize].clone(),
|
||||||
DATA_USAGE_CACHE_NAME,
|
DATA_USAGE_CACHE_NAME,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::<String>::new());
|
Ok(cache) => apply_erasure_set_usage(&cache, erasure_set),
|
||||||
erasure_set.objects_count = data_usage_info.objects_total_count;
|
Err(_) => erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
|
||||||
erasure_set.versions_count = data_usage_info.versions_total_count;
|
}
|
||||||
erasure_set.delete_markers_count = data_usage_info.delete_markers_total_count;
|
|
||||||
erasure_set.usage = data_usage_info.objects_total_size;
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
erasure_set.raw_capacity += d.total_space;
|
erasure_set.raw_capacity += d.total_space;
|
||||||
@@ -672,6 +669,20 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
|
|||||||
Ok(pools_info)
|
Ok(pools_info)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn apply_erasure_set_usage(cache: &rustfs_data_usage::DataUsageCache, erasure_set: &mut ErasureSetInfo) {
|
||||||
|
let data_usage_info = cache.dui(DATA_USAGE_ROOT, &[]);
|
||||||
|
if !data_usage_info.is_complete_bucket_usage_snapshot() {
|
||||||
|
erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
erasure_set.objects_count = data_usage_info.objects_total_count;
|
||||||
|
erasure_set.versions_count = data_usage_info.versions_total_count;
|
||||||
|
erasure_set.delete_markers_count = data_usage_info.delete_markers_total_count;
|
||||||
|
erasure_set.usage = data_usage_info.objects_total_size;
|
||||||
|
erasure_set.usage_error = None;
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::const_is_empty)]
|
#[allow(clippy::const_is_empty)]
|
||||||
pub fn get_commit_id() -> String {
|
pub fn get_commit_id() -> String {
|
||||||
let ver = if !build::TAG.is_empty() {
|
let ver = if !build::TAG.is_empty() {
|
||||||
@@ -697,8 +708,9 @@ mod tests {
|
|||||||
use rustfs_madmin::{Disk, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, ServerProperties};
|
use rustfs_madmin::{Disk, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, ServerProperties};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
DATA_USAGE_UNAVAILABLE_ERROR, apply_data_usage_result, get_local_server_property, get_online_offline_disks_stats,
|
DATA_USAGE_ROOT, DATA_USAGE_UNAVAILABLE_ERROR, apply_data_usage_result, apply_erasure_set_usage,
|
||||||
get_server_info, reconcile_servers_with_endpoint_topology, server_topology_completeness_report,
|
get_local_server_property, get_online_offline_disks_stats, get_server_info, reconcile_servers_with_endpoint_topology,
|
||||||
|
server_topology_completeness_report,
|
||||||
};
|
};
|
||||||
|
|
||||||
fn disk_with_state(endpoint: &str, state: &str) -> Disk {
|
fn disk_with_state(endpoint: &str, state: &str) -> Disk {
|
||||||
@@ -900,11 +912,19 @@ mod tests {
|
|||||||
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
|
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
|
||||||
let mut usage = rustfs_madmin::Usage::default();
|
let mut usage = rustfs_madmin::Usage::default();
|
||||||
let info = rustfs_data_usage::DataUsageInfo {
|
let info = rustfs_data_usage::DataUsageInfo {
|
||||||
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
|
||||||
buckets_count: 2,
|
buckets_count: 2,
|
||||||
objects_total_count: 3,
|
objects_total_count: 3,
|
||||||
versions_total_count: 4,
|
versions_total_count: 4,
|
||||||
delete_markers_total_count: 5,
|
delete_markers_total_count: 5,
|
||||||
objects_total_size: 6,
|
objects_total_size: 6,
|
||||||
|
buckets_usage: [
|
||||||
|
("bucket-a".to_string(), rustfs_data_usage::BucketUsageInfo::default()),
|
||||||
|
("bucket-b".to_string(), rustfs_data_usage::BucketUsageInfo::default()),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.collect(),
|
||||||
|
usage_snapshot_complete: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -917,6 +937,71 @@ mod tests {
|
|||||||
assert_eq!(usage.size, 6);
|
assert_eq!(usage.size, 6);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn incomplete_data_usage_is_unavailable_in_server_info() {
|
||||||
|
let mut buckets = rustfs_madmin::Buckets::default();
|
||||||
|
let mut objects = rustfs_madmin::Objects::default();
|
||||||
|
let mut versions = rustfs_madmin::Versions::default();
|
||||||
|
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
|
||||||
|
let mut usage = rustfs_madmin::Usage::default();
|
||||||
|
let info = rustfs_data_usage::DataUsageInfo {
|
||||||
|
buckets_count: 1,
|
||||||
|
objects_total_count: 100,
|
||||||
|
objects_total_size: 1024,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
apply_data_usage_result(Ok(info), &mut buckets, &mut objects, &mut versions, &mut delete_markers, &mut usage);
|
||||||
|
|
||||||
|
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||||
|
assert_eq!(objects.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||||
|
assert_eq!(versions.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||||
|
assert_eq!(delete_markers.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||||
|
assert_eq!(usage.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn incomplete_erasure_set_cache_is_not_reported_as_zero() {
|
||||||
|
let mut cache = rustfs_data_usage::DataUsageCache::default();
|
||||||
|
cache.info.name = DATA_USAGE_ROOT.to_string();
|
||||||
|
cache.replace(DATA_USAGE_ROOT, "", rustfs_data_usage::DataUsageEntry::default());
|
||||||
|
let mut set = rustfs_madmin::ErasureSetInfo::default();
|
||||||
|
|
||||||
|
apply_erasure_set_usage(&cache, &mut set);
|
||||||
|
|
||||||
|
assert_eq!(set.usage_error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||||
|
assert_eq!(set.objects_count, 0);
|
||||||
|
assert_eq!(set.usage, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn complete_erasure_set_cache_is_reported() {
|
||||||
|
let mut cache = rustfs_data_usage::DataUsageCache::default();
|
||||||
|
cache.info.name = DATA_USAGE_ROOT.to_string();
|
||||||
|
cache.info.last_update = Some(std::time::SystemTime::UNIX_EPOCH);
|
||||||
|
cache.info.snapshot_complete = true;
|
||||||
|
cache.replace(
|
||||||
|
DATA_USAGE_ROOT,
|
||||||
|
"",
|
||||||
|
rustfs_data_usage::DataUsageEntry {
|
||||||
|
size: 512,
|
||||||
|
objects: 3,
|
||||||
|
versions: 4,
|
||||||
|
delete_markers: 1,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let mut set = rustfs_madmin::ErasureSetInfo::default();
|
||||||
|
|
||||||
|
apply_erasure_set_usage(&cache, &mut set);
|
||||||
|
|
||||||
|
assert_eq!(set.objects_count, 3);
|
||||||
|
assert_eq!(set.versions_count, 4);
|
||||||
|
assert_eq!(set.delete_markers_count, 1);
|
||||||
|
assert_eq!(set.usage, 512);
|
||||||
|
assert!(set.usage_error.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[serial]
|
#[serial]
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn server_info_includes_global_deployment_id() {
|
async fn server_info_includes_global_deployment_id() {
|
||||||
|
|||||||
@@ -326,6 +326,13 @@ impl ECStore {
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||||
|
// A scanner may have sampled the first fence before the bucket
|
||||||
|
// became visible. Fence again after metadata initialization so
|
||||||
|
// that snapshot cannot publish as complete.
|
||||||
|
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -545,6 +552,9 @@ impl ECStore {
|
|||||||
"physical bucket deletion succeeded but metadata cleanup remains pending"
|
"physical bucket deletion succeeded but metadata cleanup remains pending"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
// A scanner may have sampled the first fence before the physical
|
||||||
|
// namespace disappeared. The completion fence invalidates that scan.
|
||||||
|
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -839,8 +849,8 @@ mod tests {
|
|||||||
.expect("bucket should be created");
|
.expect("bucket should be created");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
ecstore.scanner_namespace_mutation_generation(),
|
ecstore.scanner_namespace_mutation_generation(),
|
||||||
generation_before_make.saturating_add(1),
|
generation_before_make.saturating_add(2),
|
||||||
"successful bucket creation should advance scanner namespace activity"
|
"successful bucket creation should fence scanner namespace activity before and after creation"
|
||||||
);
|
);
|
||||||
|
|
||||||
let generation_before_put = ecstore.scanner_namespace_mutation_generation();
|
let generation_before_put = ecstore.scanner_namespace_mutation_generation();
|
||||||
@@ -1057,8 +1067,8 @@ mod tests {
|
|||||||
.expect("MarkDelete should remove an empty bucket");
|
.expect("MarkDelete should remove an empty bucket");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
ecstore.scanner_namespace_mutation_generation(),
|
ecstore.scanner_namespace_mutation_generation(),
|
||||||
generation_before_delete.saturating_add(1),
|
generation_before_delete.saturating_add(2),
|
||||||
"successful bucket deletion should advance scanner namespace activity"
|
"successful bucket deletion should fence scanner namespace activity before and after deletion"
|
||||||
);
|
);
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
@@ -1179,8 +1189,8 @@ mod tests {
|
|||||||
.expect("Purge should force-delete bucket data");
|
.expect("Purge should force-delete bucket data");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
ecstore.scanner_namespace_mutation_generation(),
|
ecstore.scanner_namespace_mutation_generation(),
|
||||||
generation_before_delete.saturating_add(1),
|
generation_before_delete.saturating_add(2),
|
||||||
"successful bucket purge should advance scanner namespace activity"
|
"successful bucket purge should fence scanner namespace activity before and after deletion"
|
||||||
);
|
);
|
||||||
|
|
||||||
assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
|
assert!(!any_disk_path_exists(&disk_paths, &bucket).await, "Purge should remove the bucket volume");
|
||||||
@@ -1249,7 +1259,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn bucket_delete_finishes_usage_cleanup_before_same_name_recreation() {
|
async fn bucket_recreation_does_not_publish_unverified_usage() {
|
||||||
let (_, ecstore) = setup_bucket_delete_test_env().await;
|
let (_, ecstore) = setup_bucket_delete_test_env().await;
|
||||||
let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple());
|
let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple());
|
||||||
ecstore
|
ecstore
|
||||||
@@ -1271,6 +1281,7 @@ mod tests {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
snapshot.usage_snapshot_complete = true;
|
||||||
snapshot.bucket_sizes.insert(bucket.clone(), 42);
|
snapshot.bucket_sizes.insert(bucket.clone(), 42);
|
||||||
snapshot.calculate_totals();
|
snapshot.calculate_totals();
|
||||||
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
|
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
|
||||||
@@ -1296,13 +1307,11 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.expect("recreated bucket usage base should load");
|
.expect("recreated bucket usage base should load");
|
||||||
crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await;
|
crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await;
|
||||||
assert_eq!(
|
assert!(
|
||||||
recreated
|
!recreated.buckets_usage.contains_key(&bucket),
|
||||||
.buckets_usage
|
"a request-path delta without an authoritative baseline must remain unavailable"
|
||||||
.get(&bucket)
|
|
||||||
.map(|usage| (usage.objects_count, usage.versions_count, usage.size)),
|
|
||||||
Some((1, 1, 84))
|
|
||||||
);
|
);
|
||||||
|
assert_eq!(crate::data_usage::get_bucket_usage_memory(&bucket).await, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1324,6 +1333,7 @@ mod tests {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
snapshot.usage_snapshot_complete = true;
|
||||||
snapshot.bucket_sizes.insert(bucket.clone(), 42);
|
snapshot.bucket_sizes.insert(bucket.clone(), 42);
|
||||||
snapshot.calculate_totals();
|
snapshot.calculate_totals();
|
||||||
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
|
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
|
||||||
@@ -1373,6 +1383,7 @@ mod tests {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
snapshot.usage_snapshot_complete = true;
|
||||||
snapshot.bucket_sizes.insert(bucket.clone(), 42);
|
snapshot.bucket_sizes.insert(bucket.clone(), 42);
|
||||||
snapshot.calculate_totals();
|
snapshot.calculate_totals();
|
||||||
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
|
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
|
||||||
|
|||||||
@@ -317,6 +317,8 @@ pub struct ErasureSetInfo {
|
|||||||
pub versions_count: u64,
|
pub versions_count: u64,
|
||||||
#[serde(rename = "deleteMarkersCount")]
|
#[serde(rename = "deleteMarkersCount")]
|
||||||
pub delete_markers_count: u64,
|
pub delete_markers_count: u64,
|
||||||
|
#[serde(rename = "usageError", default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub usage_error: Option<String>,
|
||||||
#[serde(rename = "healDisks")]
|
#[serde(rename = "healDisks")]
|
||||||
pub heal_disks: i32,
|
pub heal_disks: i32,
|
||||||
}
|
}
|
||||||
@@ -1107,6 +1109,7 @@ mod tests {
|
|||||||
assert_eq!(erasure_set.objects_count, 0);
|
assert_eq!(erasure_set.objects_count, 0);
|
||||||
assert_eq!(erasure_set.versions_count, 0);
|
assert_eq!(erasure_set.versions_count, 0);
|
||||||
assert_eq!(erasure_set.delete_markers_count, 0);
|
assert_eq!(erasure_set.delete_markers_count, 0);
|
||||||
|
assert!(erasure_set.usage_error.is_none());
|
||||||
assert_eq!(erasure_set.heal_disks, 0);
|
assert_eq!(erasure_set.heal_disks, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1120,6 +1123,7 @@ mod tests {
|
|||||||
objects_count: 10000,
|
objects_count: 10000,
|
||||||
versions_count: 15000,
|
versions_count: 15000,
|
||||||
delete_markers_count: 500,
|
delete_markers_count: 500,
|
||||||
|
usage_error: None,
|
||||||
heal_disks: 2,
|
heal_disks: 2,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1133,6 +1137,38 @@ mod tests {
|
|||||||
assert_eq!(erasure_set.heal_disks, 2);
|
assert_eq!(erasure_set.heal_disks, 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn erasure_set_usage_error_is_additive_on_the_wire() {
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct LegacyErasureSetInfo {
|
||||||
|
id: i32,
|
||||||
|
usage: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
let legacy = r#"{"id":1,"rawUsage":2,"rawCapacity":3,"usage":4,"objectsCount":5,"versionsCount":6,"deleteMarkersCount":7,"healDisks":8}"#;
|
||||||
|
let decoded: ErasureSetInfo = serde_json::from_str(legacy).expect("legacy erasure set payload should decode");
|
||||||
|
assert!(decoded.usage_error.is_none());
|
||||||
|
|
||||||
|
let encoded = serde_json::to_value(decoded).expect("erasure set payload should encode");
|
||||||
|
assert!(
|
||||||
|
encoded.get("usageError").is_none(),
|
||||||
|
"an absent usage error must not change the legacy wire shape"
|
||||||
|
);
|
||||||
|
|
||||||
|
let current = ErasureSetInfo {
|
||||||
|
id: 9,
|
||||||
|
usage: 10,
|
||||||
|
usage_error: Some("data usage unavailable".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let encoded = serde_json::to_value(current).expect("current erasure set payload should encode");
|
||||||
|
let legacy: LegacyErasureSetInfo =
|
||||||
|
serde_json::from_value(encoded).expect("legacy erasure set reader should ignore additive fields");
|
||||||
|
|
||||||
|
assert_eq!(legacy.id, 9);
|
||||||
|
assert_eq!(legacy.usage, 10);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_backend_type_default() {
|
fn test_backend_type_default() {
|
||||||
let backend_type = BackendType::default();
|
let backend_type = BackendType::default();
|
||||||
|
|||||||
@@ -33,10 +33,10 @@ use std::borrow::Cow;
|
|||||||
pub struct BucketStats {
|
pub struct BucketStats {
|
||||||
/// Name of the bucket
|
/// Name of the bucket
|
||||||
pub name: String,
|
pub name: String,
|
||||||
/// Total size of all objects in the bucket (bytes)
|
/// Total size of all objects in the bucket (bytes), when authoritative.
|
||||||
pub size_bytes: u64,
|
pub size_bytes: Option<u64>,
|
||||||
/// Number of objects in the bucket
|
/// Number of objects in the bucket, when authoritative.
|
||||||
pub objects_count: u64,
|
pub objects_count: Option<u64>,
|
||||||
/// Quota limit for the bucket (bytes), 0 if no quota
|
/// Quota limit for the bucket (bytes), 0 if no quota
|
||||||
pub quota_bytes: u64,
|
pub quota_bytes: u64,
|
||||||
}
|
}
|
||||||
@@ -54,15 +54,19 @@ pub fn collect_bucket_metrics(buckets: &[BucketStats]) -> Vec<PrometheusMetric>
|
|||||||
for bucket in buckets {
|
for bucket in buckets {
|
||||||
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone());
|
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone());
|
||||||
|
|
||||||
metrics.push(
|
if let Some(size_bytes) = bucket.size_bytes {
|
||||||
PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, bucket.size_bytes as f64)
|
metrics.push(
|
||||||
.with_label("bucket", bucket_label.clone()),
|
PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, size_bytes as f64)
|
||||||
);
|
.with_label("bucket", bucket_label.clone()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
metrics.push(
|
if let Some(objects_count) = bucket.objects_count {
|
||||||
PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, bucket.objects_count as f64)
|
metrics.push(
|
||||||
.with_label("bucket", bucket_label.clone()),
|
PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, objects_count as f64)
|
||||||
);
|
.with_label("bucket", bucket_label.clone()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
metrics.push(
|
metrics.push(
|
||||||
PrometheusMetric::from_descriptor(&BUCKET_QUOTA_BYTES_MD, bucket.quota_bytes as f64)
|
PrometheusMetric::from_descriptor(&BUCKET_QUOTA_BYTES_MD, bucket.quota_bytes as f64)
|
||||||
@@ -83,14 +87,14 @@ mod tests {
|
|||||||
let buckets = vec![
|
let buckets = vec![
|
||||||
BucketStats {
|
BucketStats {
|
||||||
name: "test-bucket".to_string(),
|
name: "test-bucket".to_string(),
|
||||||
size_bytes: 1000,
|
size_bytes: Some(1000),
|
||||||
objects_count: 100,
|
objects_count: Some(100),
|
||||||
quota_bytes: 0,
|
quota_bytes: 0,
|
||||||
},
|
},
|
||||||
BucketStats {
|
BucketStats {
|
||||||
name: "another-bucket".to_string(),
|
name: "another-bucket".to_string(),
|
||||||
size_bytes: 2000,
|
size_bytes: Some(2000),
|
||||||
objects_count: 200,
|
objects_count: Some(200),
|
||||||
quota_bytes: 0,
|
quota_bytes: 0,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
@@ -114,8 +118,8 @@ mod tests {
|
|||||||
fn test_collect_bucket_metrics_with_quotas() {
|
fn test_collect_bucket_metrics_with_quotas() {
|
||||||
let buckets = vec![BucketStats {
|
let buckets = vec![BucketStats {
|
||||||
name: "quota-bucket".to_string(),
|
name: "quota-bucket".to_string(),
|
||||||
size_bytes: 500,
|
size_bytes: Some(500),
|
||||||
objects_count: 10,
|
objects_count: Some(10),
|
||||||
quota_bytes: 10000,
|
quota_bytes: 10000,
|
||||||
}];
|
}];
|
||||||
|
|
||||||
@@ -146,8 +150,8 @@ mod tests {
|
|||||||
fn test_collect_bucket_metrics_zero_quota_always_reported() {
|
fn test_collect_bucket_metrics_zero_quota_always_reported() {
|
||||||
let buckets = vec![BucketStats {
|
let buckets = vec![BucketStats {
|
||||||
name: "no-quota-bucket".to_string(),
|
name: "no-quota-bucket".to_string(),
|
||||||
size_bytes: 100,
|
size_bytes: Some(100),
|
||||||
objects_count: 5,
|
objects_count: Some(5),
|
||||||
quota_bytes: 0,
|
quota_bytes: 0,
|
||||||
}];
|
}];
|
||||||
|
|
||||||
@@ -165,12 +169,26 @@ mod tests {
|
|||||||
assert!(quota_metric.is_some());
|
assert!(quota_metric.is_some());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_usage_only_reports_quota() {
|
||||||
|
let metrics = collect_bucket_metrics(&[BucketStats {
|
||||||
|
name: "unknown-usage".to_string(),
|
||||||
|
size_bytes: None,
|
||||||
|
objects_count: None,
|
||||||
|
quota_bytes: 4096,
|
||||||
|
}]);
|
||||||
|
|
||||||
|
assert_eq!(metrics.len(), 1);
|
||||||
|
assert_eq!(metrics[0].name, BUCKET_QUOTA_BYTES_MD.get_full_metric_name());
|
||||||
|
assert_eq!(metrics[0].value, 4096.0);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_bucket_stats_default() {
|
fn test_bucket_stats_default() {
|
||||||
let stats = BucketStats::default();
|
let stats = BucketStats::default();
|
||||||
assert!(stats.name.is_empty());
|
assert!(stats.name.is_empty());
|
||||||
assert_eq!(stats.size_bytes, 0);
|
assert_eq!(stats.size_bytes, None);
|
||||||
assert_eq!(stats.objects_count, 0);
|
assert_eq!(stats.objects_count, None);
|
||||||
assert_eq!(stats.quota_bytes, 0);
|
assert_eq!(stats.quota_bytes, 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,9 +43,9 @@ pub struct ClusterStats {
|
|||||||
/// Number of drives with no capacity observation
|
/// Number of drives with no capacity observation
|
||||||
pub missing_capacity_drives: u64,
|
pub missing_capacity_drives: u64,
|
||||||
/// Total number of objects in the cluster
|
/// Total number of objects in the cluster
|
||||||
pub objects_count: u64,
|
pub objects_count: Option<u64>,
|
||||||
/// Total number of buckets in the cluster
|
/// Total number of buckets in the cluster
|
||||||
pub buckets_count: u64,
|
pub buckets_count: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Collects cluster-wide metrics from the provided cluster statistics.
|
/// Collects cluster-wide metrics from the provided cluster statistics.
|
||||||
@@ -53,16 +53,21 @@ pub struct ClusterStats {
|
|||||||
/// Uses the metric descriptors from `metrics_type::cluster` module.
|
/// Uses the metric descriptors from `metrics_type::cluster` module.
|
||||||
/// Returns a vector of Prometheus metrics for cluster statistics.
|
/// Returns a vector of Prometheus metrics for cluster statistics.
|
||||||
pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
|
pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
|
||||||
vec![
|
let mut metrics = vec![
|
||||||
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD, stats.raw_capacity_bytes as f64),
|
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD, stats.raw_capacity_bytes as f64),
|
||||||
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD, stats.usable_capacity_bytes as f64),
|
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD, stats.usable_capacity_bytes as f64),
|
||||||
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USED_BYTES_MD, stats.used_bytes as f64),
|
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USED_BYTES_MD, stats.used_bytes as f64),
|
||||||
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_FREE_BYTES_MD, stats.free_bytes as f64),
|
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_FREE_BYTES_MD, stats.free_bytes as f64),
|
||||||
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_STALE_DRIVES_MD, stats.stale_capacity_drives as f64),
|
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_STALE_DRIVES_MD, stats.stale_capacity_drives as f64),
|
||||||
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_MISSING_DRIVES_MD, stats.missing_capacity_drives as f64),
|
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_MISSING_DRIVES_MD, stats.missing_capacity_drives as f64),
|
||||||
PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, stats.objects_count as f64),
|
];
|
||||||
PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, stats.buckets_count as f64),
|
if let Some(objects_count) = stats.objects_count {
|
||||||
]
|
metrics.push(PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, objects_count as f64));
|
||||||
|
}
|
||||||
|
if let Some(buckets_count) = stats.buckets_count {
|
||||||
|
metrics.push(PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, buckets_count as f64));
|
||||||
|
}
|
||||||
|
metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -79,8 +84,8 @@ mod tests {
|
|||||||
free_bytes: 1300,
|
free_bytes: 1300,
|
||||||
stale_capacity_drives: 1,
|
stale_capacity_drives: 1,
|
||||||
missing_capacity_drives: 0,
|
missing_capacity_drives: 0,
|
||||||
objects_count: 100,
|
objects_count: Some(100),
|
||||||
buckets_count: 5,
|
buckets_count: Some(5),
|
||||||
};
|
};
|
||||||
|
|
||||||
let metrics = collect_cluster_metrics(&stats);
|
let metrics = collect_cluster_metrics(&stats);
|
||||||
@@ -116,7 +121,7 @@ mod tests {
|
|||||||
let metrics = collect_cluster_metrics(&stats);
|
let metrics = collect_cluster_metrics(&stats);
|
||||||
report_metrics(&metrics);
|
report_metrics(&metrics);
|
||||||
|
|
||||||
assert_eq!(metrics.len(), 8);
|
assert_eq!(metrics.len(), 6);
|
||||||
|
|
||||||
// All values should be zero
|
// All values should be zero
|
||||||
for metric in &metrics {
|
for metric in &metrics {
|
||||||
@@ -134,7 +139,27 @@ mod tests {
|
|||||||
assert_eq!(stats.free_bytes, 0);
|
assert_eq!(stats.free_bytes, 0);
|
||||||
assert_eq!(stats.stale_capacity_drives, 0);
|
assert_eq!(stats.stale_capacity_drives, 0);
|
||||||
assert_eq!(stats.missing_capacity_drives, 0);
|
assert_eq!(stats.missing_capacity_drives, 0);
|
||||||
assert_eq!(stats.objects_count, 0);
|
assert_eq!(stats.objects_count, None);
|
||||||
assert_eq!(stats.buckets_count, 0);
|
assert_eq!(stats.buckets_count, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_cluster_cardinality_is_omitted() {
|
||||||
|
let metrics = collect_cluster_metrics(&ClusterStats {
|
||||||
|
objects_count: None,
|
||||||
|
buckets_count: None,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
metrics
|
||||||
|
.iter()
|
||||||
|
.all(|metric| metric.name != CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name())
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
metrics
|
||||||
|
.iter()
|
||||||
|
.all(|metric| metric.name != CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name())
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -79,10 +79,13 @@ use crate::metrics::schema::audit::{AUDIT_FAILED_MESSAGES_MD, AUDIT_TARGET_QUEUE
|
|||||||
use crate::metrics::schema::bucket_replication::{
|
use crate::metrics::schema::bucket_replication::{
|
||||||
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, TARGET_ARN_L,
|
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, TARGET_ARN_L,
|
||||||
};
|
};
|
||||||
|
use crate::metrics::schema::cluster::{CLUSTER_BUCKETS_TOTAL_MD, CLUSTER_OBJECTS_TOTAL_MD};
|
||||||
use crate::metrics::schema::cluster_usage::{
|
use crate::metrics::schema::cluster_usage::{
|
||||||
BUCKET_LABEL as USAGE_BUCKET_LABEL, RANGE_LABEL as USAGE_RANGE_LABEL, USAGE_BUCKET_DELETE_MARKERS_COUNT_MD,
|
BUCKET_LABEL as USAGE_BUCKET_LABEL, RANGE_LABEL as USAGE_RANGE_LABEL, USAGE_BUCKET_DELETE_MARKERS_COUNT_MD,
|
||||||
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, USAGE_BUCKET_OBJECTS_TOTAL_MD,
|
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, USAGE_BUCKET_OBJECTS_TOTAL_MD,
|
||||||
USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, USAGE_BUCKET_TOTAL_BYTES_MD, USAGE_BUCKET_VERSIONS_COUNT_MD,
|
USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, USAGE_BUCKET_TOTAL_BYTES_MD, USAGE_BUCKET_VERSIONS_COUNT_MD, USAGE_BUCKETS_COUNT_MD,
|
||||||
|
USAGE_DELETE_MARKERS_COUNT_MD, USAGE_OBJECTS_COUNT_MD, USAGE_OBJECTS_DISTRIBUTION_MD, USAGE_SINCE_LAST_UPDATE_SECONDS_MD,
|
||||||
|
USAGE_TOTAL_BYTES_MD, USAGE_VERSIONS_COUNT_MD, USAGE_VERSIONS_DISTRIBUTION_MD,
|
||||||
};
|
};
|
||||||
use crate::metrics::schema::node_bucket::{BUCKET_OBJECTS_TOTAL_MD, BUCKET_QUOTA_BYTES_MD, BUCKET_USAGE_BYTES_MD};
|
use crate::metrics::schema::node_bucket::{BUCKET_OBJECTS_TOTAL_MD, BUCKET_QUOTA_BYTES_MD, BUCKET_USAGE_BYTES_MD};
|
||||||
use crate::metrics::schema::notification_target::{
|
use crate::metrics::schema::notification_target::{
|
||||||
@@ -659,6 +662,26 @@ fn bucket_live_keys(stats: &[crate::metrics::collectors::BucketStats]) -> HashSe
|
|||||||
stats.iter().map(|stat| stat.name.clone()).collect()
|
stats.iter().map(|stat| stat.name.clone()).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn bucket_observation_live_keys(stats: &[crate::metrics::collectors::BucketStats]) -> HashSet<BucketKey> {
|
||||||
|
stats
|
||||||
|
.iter()
|
||||||
|
.filter(|stat| stat.size_bytes.is_some() || stat.objects_count.is_some())
|
||||||
|
.map(|stat| stat.name.clone())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bucket_observation_retire_keys(
|
||||||
|
previous_observations: &HashSet<BucketKey>,
|
||||||
|
current_buckets: &HashSet<BucketKey>,
|
||||||
|
current_observations: &HashSet<BucketKey>,
|
||||||
|
) -> Vec<BucketKey> {
|
||||||
|
previous_observations
|
||||||
|
.difference(current_observations)
|
||||||
|
.filter(|bucket| current_buckets.contains(*bucket))
|
||||||
|
.cloned()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8>) -> Vec<PrometheusMetric> {
|
fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8>) -> Vec<PrometheusMetric> {
|
||||||
if zero_tombstones.is_empty() {
|
if zero_tombstones.is_empty() {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
@@ -677,6 +700,50 @@ fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8
|
|||||||
zero_metrics
|
zero_metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct BucketSeriesState {
|
||||||
|
has_seen_snapshot: bool,
|
||||||
|
live_keys: HashSet<BucketKey>,
|
||||||
|
observation_keys: HashSet<BucketKey>,
|
||||||
|
zero_tombstones: HashMap<BucketKey, u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct BucketSeriesUpdate {
|
||||||
|
metrics: Vec<PrometheusMetric>,
|
||||||
|
retire_observations: Vec<BucketKey>,
|
||||||
|
retire_buckets: Vec<BucketKey>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BucketSeriesState {
|
||||||
|
fn observe(
|
||||||
|
&mut self,
|
||||||
|
stats: Option<&[crate::metrics::collectors::BucketStats]>,
|
||||||
|
tombstone_cycles: u8,
|
||||||
|
) -> Option<BucketSeriesUpdate> {
|
||||||
|
let stats = stats?;
|
||||||
|
let current_bucket_keys = bucket_live_keys(stats);
|
||||||
|
let current_observation_keys = bucket_observation_live_keys(stats);
|
||||||
|
let retire_observations =
|
||||||
|
bucket_observation_retire_keys(&self.observation_keys, ¤t_bucket_keys, ¤t_observation_keys);
|
||||||
|
self.observation_keys = current_observation_keys;
|
||||||
|
update_series_zero_tombstones(
|
||||||
|
&mut self.has_seen_snapshot,
|
||||||
|
&mut self.live_keys,
|
||||||
|
&mut self.zero_tombstones,
|
||||||
|
current_bucket_keys,
|
||||||
|
tombstone_cycles,
|
||||||
|
);
|
||||||
|
let mut metrics = collect_bucket_metrics(stats);
|
||||||
|
metrics.extend(collect_bucket_zero_tombstone_metrics(&self.zero_tombstones));
|
||||||
|
let retire_buckets = expire_series_zero_tombstones(&mut self.zero_tombstones);
|
||||||
|
Some(BucketSeriesUpdate {
|
||||||
|
metrics,
|
||||||
|
retire_observations,
|
||||||
|
retire_buckets,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn retire_bucket_metric_series(bucket: &str) -> usize {
|
fn retire_bucket_metric_series(bucket: &str) -> usize {
|
||||||
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.to_string());
|
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.to_string());
|
||||||
let labels = [("bucket", bucket_label.clone())];
|
let labels = [("bucket", bucket_label.clone())];
|
||||||
@@ -685,6 +752,32 @@ fn retire_bucket_metric_series(bucket: &str) -> usize {
|
|||||||
+ retire_metric_series(&BUCKET_QUOTA_BYTES_MD.get_full_metric_name(), &labels)
|
+ retire_metric_series(&BUCKET_QUOTA_BYTES_MD.get_full_metric_name(), &labels)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn retire_bucket_observation_metric_series(bucket: &str) -> usize {
|
||||||
|
let labels = [("bucket", Cow::Owned(bucket.to_string()))];
|
||||||
|
retire_metric_series(&BUCKET_USAGE_BYTES_MD.get_full_metric_name(), &labels)
|
||||||
|
+ retire_metric_series(&BUCKET_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn retire_cluster_usage_metric_series() -> usize {
|
||||||
|
let labels: [(&'static str, Cow<'static, str>); 0] = [];
|
||||||
|
[
|
||||||
|
USAGE_SINCE_LAST_UPDATE_SECONDS_MD.get_full_metric_name(),
|
||||||
|
USAGE_TOTAL_BYTES_MD.get_full_metric_name(),
|
||||||
|
USAGE_OBJECTS_COUNT_MD.get_full_metric_name(),
|
||||||
|
USAGE_VERSIONS_COUNT_MD.get_full_metric_name(),
|
||||||
|
USAGE_DELETE_MARKERS_COUNT_MD.get_full_metric_name(),
|
||||||
|
USAGE_BUCKETS_COUNT_MD.get_full_metric_name(),
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.map(|name| retire_metric_series(name, &labels))
|
||||||
|
.sum()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn retire_cluster_usage_distribution_series(metric_name: String, range: &str) -> usize {
|
||||||
|
let labels = [(USAGE_RANGE_LABEL, Cow::Owned(range.to_string()))];
|
||||||
|
retire_metric_series(&metric_name, &labels)
|
||||||
|
}
|
||||||
|
|
||||||
fn bucket_usage_live_keys(stats: &[crate::metrics::collectors::BucketUsageStats]) -> HashSet<BucketKey> {
|
fn bucket_usage_live_keys(stats: &[crate::metrics::collectors::BucketUsageStats]) -> HashSet<BucketKey> {
|
||||||
stats.iter().map(|stat| stat.bucket.clone()).collect()
|
stats.iter().map(|stat| stat.bucket.clone()).collect()
|
||||||
}
|
}
|
||||||
@@ -973,11 +1066,23 @@ pub fn init_metrics_runtime(token: CancellationToken) {
|
|||||||
let token_clone = token.clone();
|
let token_clone = token.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let mut interval = metrics_interval(cluster_interval, Duration::ZERO);
|
let mut interval = metrics_interval(cluster_interval, Duration::ZERO);
|
||||||
|
let mut objects_count_was_authoritative = false;
|
||||||
|
let mut buckets_count_was_authoritative = false;
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = interval.tick() => {
|
_ = interval.tick() => {
|
||||||
run_metrics_collector_tick(health, MetricsCollectorTaskId::ClusterStats, "cluster_stats", async {
|
run_metrics_collector_tick(health, MetricsCollectorTaskId::ClusterStats, "cluster_stats", async {
|
||||||
let (stats, cluster_health) = collect_cluster_and_health_stats().await;
|
let (stats, cluster_health) = collect_cluster_and_health_stats().await;
|
||||||
|
if objects_count_was_authoritative && stats.objects_count.is_none() {
|
||||||
|
let labels: [(&'static str, Cow<'static, str>); 0] = [];
|
||||||
|
let _ = retire_metric_series(&CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels);
|
||||||
|
}
|
||||||
|
if buckets_count_was_authoritative && stats.buckets_count.is_none() {
|
||||||
|
let labels: [(&'static str, Cow<'static, str>); 0] = [];
|
||||||
|
let _ = retire_metric_series(&CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name(), &labels);
|
||||||
|
}
|
||||||
|
objects_count_was_authoritative = stats.objects_count.is_some();
|
||||||
|
buckets_count_was_authoritative = stats.buckets_count.is_some();
|
||||||
let mut metrics = collect_cluster_metrics(&stats);
|
let mut metrics = collect_cluster_metrics(&stats);
|
||||||
metrics.extend(collect_cluster_health_metrics(&cluster_health));
|
metrics.extend(collect_cluster_health_metrics(&cluster_health));
|
||||||
report_metrics(&metrics);
|
report_metrics(&metrics);
|
||||||
@@ -1004,6 +1109,8 @@ pub fn init_metrics_runtime(token: CancellationToken) {
|
|||||||
let mut bucket_usage_object_size_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
|
let mut bucket_usage_object_size_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
|
||||||
let mut prev_bucket_usage_version_keys: HashSet<BucketRangeKey> = HashSet::new();
|
let mut prev_bucket_usage_version_keys: HashSet<BucketRangeKey> = HashSet::new();
|
||||||
let mut bucket_usage_version_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
|
let mut bucket_usage_version_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
|
||||||
|
let mut prev_cluster_usage_object_size_keys: HashSet<String> = HashSet::new();
|
||||||
|
let mut prev_cluster_usage_version_keys: HashSet<String> = HashSet::new();
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = interval.tick() => {
|
_ = interval.tick() => {
|
||||||
@@ -1028,6 +1135,30 @@ pub fn init_metrics_runtime(token: CancellationToken) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some((cluster_usage, bucket_usage)) = collect_cluster_usage_metric_stats().await {
|
if let Some((cluster_usage, bucket_usage)) = collect_cluster_usage_metric_stats().await {
|
||||||
|
let current_cluster_usage_object_size_keys = cluster_usage
|
||||||
|
.object_size_distribution
|
||||||
|
.iter()
|
||||||
|
.map(|(range, _)| range.clone())
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
for range in prev_cluster_usage_object_size_keys.difference(¤t_cluster_usage_object_size_keys) {
|
||||||
|
let _ = retire_cluster_usage_distribution_series(
|
||||||
|
USAGE_OBJECTS_DISTRIBUTION_MD.get_full_metric_name(),
|
||||||
|
range,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
prev_cluster_usage_object_size_keys = current_cluster_usage_object_size_keys;
|
||||||
|
let current_cluster_usage_version_keys = cluster_usage
|
||||||
|
.versions_distribution
|
||||||
|
.iter()
|
||||||
|
.map(|(range, _)| range.clone())
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
for range in prev_cluster_usage_version_keys.difference(¤t_cluster_usage_version_keys) {
|
||||||
|
let _ = retire_cluster_usage_distribution_series(
|
||||||
|
USAGE_VERSIONS_DISTRIBUTION_MD.get_full_metric_name(),
|
||||||
|
range,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
prev_cluster_usage_version_keys = current_cluster_usage_version_keys;
|
||||||
metrics.extend(collect_cluster_usage_metrics(&cluster_usage));
|
metrics.extend(collect_cluster_usage_metrics(&cluster_usage));
|
||||||
update_series_zero_tombstones(
|
update_series_zero_tombstones(
|
||||||
&mut has_seen_bucket_usage_snapshot,
|
&mut has_seen_bucket_usage_snapshot,
|
||||||
@@ -1073,6 +1204,41 @@ pub fn init_metrics_runtime(token: CancellationToken) {
|
|||||||
&range,
|
&range,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
} else if has_seen_bucket_usage_snapshot {
|
||||||
|
let _ = retire_cluster_usage_metric_series();
|
||||||
|
for range in prev_cluster_usage_object_size_keys.drain() {
|
||||||
|
let _ = retire_cluster_usage_distribution_series(
|
||||||
|
USAGE_OBJECTS_DISTRIBUTION_MD.get_full_metric_name(),
|
||||||
|
&range,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for range in prev_cluster_usage_version_keys.drain() {
|
||||||
|
let _ = retire_cluster_usage_distribution_series(
|
||||||
|
USAGE_VERSIONS_DISTRIBUTION_MD.get_full_metric_name(),
|
||||||
|
&range,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for bucket in prev_bucket_usage_keys.drain() {
|
||||||
|
let _ = retire_bucket_usage_metric_series(&bucket);
|
||||||
|
}
|
||||||
|
for (bucket, range) in prev_bucket_usage_object_size_keys.drain() {
|
||||||
|
let _ = retire_bucket_usage_distribution_series(
|
||||||
|
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD.get_full_metric_name(),
|
||||||
|
&bucket,
|
||||||
|
&range,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (bucket, range) in prev_bucket_usage_version_keys.drain() {
|
||||||
|
let _ = retire_bucket_usage_distribution_series(
|
||||||
|
USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD.get_full_metric_name(),
|
||||||
|
&bucket,
|
||||||
|
&range,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
bucket_usage_zero_tombstones.clear();
|
||||||
|
bucket_usage_object_size_zero_tombstones.clear();
|
||||||
|
bucket_usage_version_zero_tombstones.clear();
|
||||||
|
has_seen_bucket_usage_snapshot = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if !metrics.is_empty() {
|
if !metrics.is_empty() {
|
||||||
@@ -1094,25 +1260,20 @@ pub fn init_metrics_runtime(token: CancellationToken) {
|
|||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let mut interval = metrics_interval(bucket_interval, Duration::ZERO);
|
let mut interval = metrics_interval(bucket_interval, Duration::ZERO);
|
||||||
let tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
|
let tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
|
||||||
let mut has_seen_bucket_snapshot = false;
|
let mut series_state = BucketSeriesState::default();
|
||||||
let mut prev_bucket_keys: HashSet<BucketKey> = HashSet::new();
|
|
||||||
let mut bucket_zero_tombstones: HashMap<BucketKey, u8> = HashMap::new();
|
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = interval.tick() => {
|
_ = interval.tick() => {
|
||||||
run_metrics_collector_tick(health, MetricsCollectorTaskId::BucketStats, "bucket_stats", async {
|
run_metrics_collector_tick(health, MetricsCollectorTaskId::BucketStats, "bucket_stats", async {
|
||||||
let stats = collect_bucket_stats().await;
|
let stats = collect_bucket_stats().await;
|
||||||
update_series_zero_tombstones(
|
let Some(update) = series_state.observe(stats.as_deref(), tombstone_cycles) else {
|
||||||
&mut has_seen_bucket_snapshot,
|
return;
|
||||||
&mut prev_bucket_keys,
|
};
|
||||||
&mut bucket_zero_tombstones,
|
for bucket in update.retire_observations {
|
||||||
bucket_live_keys(&stats),
|
let _ = retire_bucket_observation_metric_series(&bucket);
|
||||||
tombstone_cycles,
|
}
|
||||||
);
|
report_metrics(&update.metrics);
|
||||||
let mut metrics = collect_bucket_metrics(&stats);
|
for bucket in update.retire_buckets {
|
||||||
metrics.extend(collect_bucket_zero_tombstone_metrics(&bucket_zero_tombstones));
|
|
||||||
report_metrics(&metrics);
|
|
||||||
for bucket in expire_series_zero_tombstones(&mut bucket_zero_tombstones) {
|
|
||||||
let _ = retire_bucket_metric_series(&bucket);
|
let _ = retire_bucket_metric_series(&bucket);
|
||||||
}
|
}
|
||||||
}).await;
|
}).await;
|
||||||
@@ -1850,8 +2011,8 @@ mod tests {
|
|||||||
let mut zero_tombstones = HashMap::new();
|
let mut zero_tombstones = HashMap::new();
|
||||||
let live_stats = vec![crate::metrics::collectors::BucketStats {
|
let live_stats = vec![crate::metrics::collectors::BucketStats {
|
||||||
name: "tmp".to_string(),
|
name: "tmp".to_string(),
|
||||||
size_bytes: 1024,
|
size_bytes: Some(1024),
|
||||||
objects_count: 8,
|
objects_count: Some(8),
|
||||||
quota_bytes: 2048,
|
quota_bytes: 2048,
|
||||||
}];
|
}];
|
||||||
|
|
||||||
@@ -1885,6 +2046,50 @@ mod tests {
|
|||||||
assert!(zero_tombstones.is_empty());
|
assert!(zero_tombstones.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bucket_observation_retirement_distinguishes_unknown_usage_from_deletion() {
|
||||||
|
let previous = HashSet::from(["bucket".to_string()]);
|
||||||
|
let unknown_stats = vec![crate::metrics::collectors::BucketStats {
|
||||||
|
name: "bucket".to_string(),
|
||||||
|
size_bytes: None,
|
||||||
|
objects_count: None,
|
||||||
|
quota_bytes: 1024,
|
||||||
|
}];
|
||||||
|
let current_buckets = bucket_live_keys(&unknown_stats);
|
||||||
|
let current_observations = bucket_observation_live_keys(&unknown_stats);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
bucket_observation_retire_keys(&previous, ¤t_buckets, ¤t_observations),
|
||||||
|
vec!["bucket".to_string()],
|
||||||
|
"an existing bucket with unknown usage must retire its previous usage observations"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
bucket_observation_retire_keys(&previous, &HashSet::new(), &HashSet::new()).is_empty(),
|
||||||
|
"a deleted bucket remains governed by the zero-tombstone lifecycle"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unavailable_bucket_snapshot_preserves_metric_series_state() {
|
||||||
|
let mut state = BucketSeriesState::default();
|
||||||
|
let initial = [crate::metrics::collectors::BucketStats {
|
||||||
|
name: "bucket".to_string(),
|
||||||
|
size_bytes: Some(512),
|
||||||
|
objects_count: Some(2),
|
||||||
|
quota_bytes: 1024,
|
||||||
|
}];
|
||||||
|
assert!(state.observe(Some(&initial), 2).is_some());
|
||||||
|
|
||||||
|
let live_keys = state.live_keys.clone();
|
||||||
|
let observation_keys = state.observation_keys.clone();
|
||||||
|
let zero_tombstones = state.zero_tombstones.clone();
|
||||||
|
|
||||||
|
assert!(state.observe(None, 2).is_none());
|
||||||
|
assert_eq!(state.live_keys, live_keys);
|
||||||
|
assert_eq!(state.observation_keys, observation_keys);
|
||||||
|
assert_eq!(state.zero_tombstones, zero_tombstones);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn parse_system_metrics_interval_rounds_legacy_millis_up_to_one_second() {
|
fn parse_system_metrics_interval_rounds_legacy_millis_up_to_one_second() {
|
||||||
temp_env::with_vars(
|
temp_env::with_vars(
|
||||||
|
|||||||
@@ -41,7 +41,11 @@ use rustfs_io_metrics::{
|
|||||||
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_resource_and_system,
|
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_resource_and_system,
|
||||||
snapshot_process_resource_and_system_with,
|
snapshot_process_resource_and_system_with,
|
||||||
};
|
};
|
||||||
use std::{collections::HashMap, sync::Arc, time::SystemTime};
|
use std::{
|
||||||
|
collections::{HashMap, HashSet},
|
||||||
|
sync::Arc,
|
||||||
|
time::SystemTime,
|
||||||
|
};
|
||||||
use sysinfo::{Networks, System};
|
use sysinfo::{Networks, System};
|
||||||
use tracing::{instrument, warn};
|
use tracing::{instrument, warn};
|
||||||
|
|
||||||
@@ -52,8 +56,10 @@ const EVENT_METRICS_COLLECTOR_STATE: &str = "metrics_collector_state";
|
|||||||
type ObsStorageInfo = <ObsStore as StorageAdminApi>::StorageInfo;
|
type ObsStorageInfo = <ObsStore as StorageAdminApi>::StorageInfo;
|
||||||
type ObsBackendInfo = <ObsStore as StorageAdminApi>::BackendInfo;
|
type ObsBackendInfo = <ObsStore as StorageAdminApi>::BackendInfo;
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
struct ObsDataUsageInfo {
|
struct ObsDataUsageInfo {
|
||||||
last_update: Option<SystemTime>,
|
last_update: Option<SystemTime>,
|
||||||
|
usage_snapshot_complete: bool,
|
||||||
buckets_count: u64,
|
buckets_count: u64,
|
||||||
objects_total_count: u64,
|
objects_total_count: u64,
|
||||||
versions_total_count: u64,
|
versions_total_count: u64,
|
||||||
@@ -62,6 +68,7 @@ struct ObsDataUsageInfo {
|
|||||||
buckets_usage: HashMap<String, ObsBucketUsageInfo>,
|
buckets_usage: HashMap<String, ObsBucketUsageInfo>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
struct ObsBucketUsageInfo {
|
struct ObsBucketUsageInfo {
|
||||||
size: u64,
|
size: u64,
|
||||||
objects_count: u64,
|
objects_count: u64,
|
||||||
@@ -85,9 +92,11 @@ fn usize_to_u64_saturating(value: usize) -> u64 {
|
|||||||
|
|
||||||
async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreResult<ObsDataUsageInfo> {
|
async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreResult<ObsDataUsageInfo> {
|
||||||
let data_usage = obs_load_data_usage_from_backend(store).await?;
|
let data_usage = obs_load_data_usage_from_backend(store).await?;
|
||||||
|
let usage_snapshot_complete = data_usage.is_complete_bucket_usage_snapshot();
|
||||||
|
|
||||||
Ok(ObsDataUsageInfo {
|
Ok(ObsDataUsageInfo {
|
||||||
last_update: data_usage.last_update,
|
last_update: data_usage.last_update,
|
||||||
|
usage_snapshot_complete,
|
||||||
buckets_count: data_usage.buckets_count,
|
buckets_count: data_usage.buckets_count,
|
||||||
objects_total_count: data_usage.objects_total_count,
|
objects_total_count: data_usage.objects_total_count,
|
||||||
versions_total_count: data_usage.versions_total_count,
|
versions_total_count: data_usage.versions_total_count,
|
||||||
@@ -113,6 +122,21 @@ async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreRes
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn bucket_usage_metric_values(data_usage: Option<&ObsDataUsageInfo>, bucket: &str) -> (Option<u64>, Option<u64>) {
|
||||||
|
data_usage
|
||||||
|
.filter(|usage| usage.usage_snapshot_complete)
|
||||||
|
.and_then(|usage| usage.buckets_usage.get(bucket))
|
||||||
|
.map(|usage| (Some(usage.size), Some(usage.objects_count)))
|
||||||
|
.unwrap_or((None, None))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn data_usage_snapshot_covers_bucket_namespace(data_usage: &ObsDataUsageInfo, buckets: &HashSet<String>) -> bool {
|
||||||
|
data_usage.usage_snapshot_complete
|
||||||
|
&& u64::try_from(buckets.len()).ok() == Some(data_usage.buckets_count)
|
||||||
|
&& buckets.len() == data_usage.buckets_usage.len()
|
||||||
|
&& buckets.iter().all(|bucket| data_usage.buckets_usage.contains_key(bucket))
|
||||||
|
}
|
||||||
|
|
||||||
fn resolve_obs_object_store_handle() -> Option<Arc<ObsStore>> {
|
fn resolve_obs_object_store_handle() -> Option<Arc<ObsStore>> {
|
||||||
obs_resolve_object_store_handle()
|
obs_resolve_object_store_handle()
|
||||||
}
|
}
|
||||||
@@ -365,25 +389,16 @@ pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthS
|
|||||||
})
|
})
|
||||||
.count() as u64;
|
.count() as u64;
|
||||||
|
|
||||||
// Get bucket and object counts from data usage info.
|
let data_usage = match load_obs_data_usage_from_backend(store).await {
|
||||||
let (buckets_count, objects_count) = match load_obs_data_usage_from_backend(store.clone()).await {
|
Ok(data_usage) if data_usage.usage_snapshot_complete => Some(data_usage),
|
||||||
Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
|
Ok(_) => None,
|
||||||
Err(e) => {
|
Err(error) => {
|
||||||
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "data_usage_load_failed", error = %e, "metrics collector state changed");
|
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "data_usage_load_failed", error = %error, "metrics collector state changed");
|
||||||
// Fall back to bucket list for buckets_count, objects_count stays 0.
|
None
|
||||||
let buckets = store
|
|
||||||
.list_bucket(&BucketOptions {
|
|
||||||
cached: true,
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.unwrap_or_else(|err| {
|
|
||||||
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "bucket_list_failed", error = %err, "metrics collector state changed");
|
|
||||||
Vec::new()
|
|
||||||
});
|
|
||||||
(buckets.len() as u64, 0)
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let buckets_count = data_usage.as_ref().map(|usage| usage.buckets_count);
|
||||||
|
let objects_count = data_usage.as_ref().map(|usage| usage.objects_total_count);
|
||||||
|
|
||||||
let mut online = 0u64;
|
let mut online = 0u64;
|
||||||
let mut offline = 0u64;
|
let mut offline = 0u64;
|
||||||
@@ -428,10 +443,12 @@ pub async fn collect_cluster_health_stats() -> ClusterHealthStats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Collect bucket statistics from the storage layer.
|
/// Collect bucket statistics from the storage layer.
|
||||||
pub async fn collect_bucket_stats() -> Vec<BucketStats> {
|
///
|
||||||
let Some(store) = resolve_obs_object_store_handle() else {
|
/// `None` means the bucket namespace could not be observed. Callers must keep
|
||||||
return Vec::new();
|
/// their prior metric-series state instead of treating that failure as an
|
||||||
};
|
/// authoritative empty namespace.
|
||||||
|
pub async fn collect_bucket_stats() -> Option<Vec<BucketStats>> {
|
||||||
|
let store = resolve_obs_object_store_handle()?;
|
||||||
|
|
||||||
// Load data usage info from backend to get bucket sizes and object counts
|
// Load data usage info from backend to get bucket sizes and object counts
|
||||||
let data_usage = match load_obs_data_usage_from_backend(store.clone()).await {
|
let data_usage = match load_obs_data_usage_from_backend(store.clone()).await {
|
||||||
@@ -453,7 +470,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
|
|||||||
Ok(buckets) => buckets,
|
Ok(buckets) => buckets,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "bucket_stats", result = "bucket_list_failed", error = %e, "metrics collector state changed");
|
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "bucket_stats", result = "bucket_list_failed", error = %e, "metrics collector state changed");
|
||||||
return Vec::new();
|
return None;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -465,11 +482,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get size and objects_count from data usage info
|
// Get size and objects_count from data usage info
|
||||||
let (size_bytes, objects_count) = data_usage
|
let (size_bytes, objects_count) = bucket_usage_metric_values(data_usage.as_ref(), &bucket.name);
|
||||||
.as_ref()
|
|
||||||
.and_then(|du| du.buckets_usage.get(&bucket.name))
|
|
||||||
.map(|bui| (bui.size, bui.objects_count))
|
|
||||||
.unwrap_or((0, 0));
|
|
||||||
|
|
||||||
// Get quota from bucket metadata
|
// Get quota from bucket metadata
|
||||||
let quota_bytes = obs_bucket_quota_limit_bytes(&bucket.name).await;
|
let quota_bytes = obs_bucket_quota_limit_bytes(&bucket.name).await;
|
||||||
@@ -482,7 +495,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
stats
|
Some(stats)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Collect bucket replication bandwidth stats from the global monitor.
|
/// Collect bucket replication bandwidth stats from the global monitor.
|
||||||
@@ -935,6 +948,29 @@ pub async fn collect_iam_stats() -> Option<IamStats> {
|
|||||||
pub async fn collect_cluster_usage_metric_stats() -> Option<(ClusterUsageStats, Vec<BucketUsageStats>)> {
|
pub async fn collect_cluster_usage_metric_stats() -> Option<(ClusterUsageStats, Vec<BucketUsageStats>)> {
|
||||||
let store = resolve_obs_object_store_handle()?;
|
let store = resolve_obs_object_store_handle()?;
|
||||||
let data_usage = load_obs_data_usage_from_backend(store.clone()).await.ok()?;
|
let data_usage = load_obs_data_usage_from_backend(store.clone()).await.ok()?;
|
||||||
|
let bucket_namespace = store
|
||||||
|
.list_bucket(&BucketOptions {
|
||||||
|
cached: true,
|
||||||
|
no_metadata: true,
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.ok()?
|
||||||
|
.into_iter()
|
||||||
|
.filter(|bucket| !bucket.name.starts_with('.'))
|
||||||
|
.map(|bucket| bucket.name)
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
collect_cluster_usage_metric_stats_from_data_usage(data_usage, &bucket_namespace).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn collect_cluster_usage_metric_stats_from_data_usage(
|
||||||
|
data_usage: ObsDataUsageInfo,
|
||||||
|
bucket_namespace: &HashSet<String>,
|
||||||
|
) -> Option<(ClusterUsageStats, Vec<BucketUsageStats>)> {
|
||||||
|
if !data_usage_snapshot_covers_bucket_namespace(&data_usage, bucket_namespace) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
let mut buckets = Vec::with_capacity(data_usage.buckets_usage.len());
|
let mut buckets = Vec::with_capacity(data_usage.buckets_usage.len());
|
||||||
|
|
||||||
for (bucket_name, usage) in &data_usage.buckets_usage {
|
for (bucket_name, usage) in &data_usage.buckets_usage {
|
||||||
@@ -1187,6 +1223,65 @@ mod tests {
|
|||||||
info
|
info
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bucket_usage_metrics_distinguish_unknown_from_confirmed_zero() {
|
||||||
|
assert_eq!(bucket_usage_metric_values(None, "bucket"), (None, None));
|
||||||
|
|
||||||
|
let mut data_usage = ObsDataUsageInfo {
|
||||||
|
usage_snapshot_complete: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
data_usage
|
||||||
|
.buckets_usage
|
||||||
|
.insert("bucket".to_string(), ObsBucketUsageInfo::default());
|
||||||
|
|
||||||
|
assert_eq!(bucket_usage_metric_values(Some(&data_usage), "bucket"), (Some(0), Some(0)));
|
||||||
|
assert_eq!(bucket_usage_metric_values(Some(&data_usage), "missing"), (None, None));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cluster_usage_metrics_skip_incomplete_snapshot() {
|
||||||
|
assert!(
|
||||||
|
collect_cluster_usage_metric_stats_from_data_usage(ObsDataUsageInfo::default(), &HashSet::new())
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cluster_usage_metrics_publish_complete_empty_snapshot() {
|
||||||
|
let (cluster, buckets) = collect_cluster_usage_metric_stats_from_data_usage(
|
||||||
|
ObsDataUsageInfo {
|
||||||
|
usage_snapshot_complete: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
&HashSet::new(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("complete empty usage should remain publishable");
|
||||||
|
|
||||||
|
assert_eq!(cluster.buckets_count, 0);
|
||||||
|
assert_eq!(cluster.objects_count, 0);
|
||||||
|
assert_eq!(cluster.total_bytes, 0);
|
||||||
|
assert!(buckets.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cluster_usage_metrics_skip_snapshot_for_a_different_bucket_namespace() {
|
||||||
|
let data_usage = ObsDataUsageInfo {
|
||||||
|
usage_snapshot_complete: true,
|
||||||
|
buckets_count: 1,
|
||||||
|
buckets_usage: HashMap::from([("stale-bucket".to_string(), ObsBucketUsageInfo::default())]),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
collect_cluster_usage_metric_stats_from_data_usage(data_usage, &HashSet::from(["live-bucket".to_string()]))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cluster_config_stats_accept_homogeneous_backend_parities() {
|
fn cluster_config_stats_accept_homogeneous_backend_parities() {
|
||||||
let stats = cluster_config_stats_from_backend_parities(Some(1), Some(2))
|
let stats = cluster_config_stats_from_backend_parities(Some(1), Some(2))
|
||||||
|
|||||||
@@ -35,10 +35,10 @@ use storage_api::owner::{
|
|||||||
EcstoreTierConfig, EcstoreVersioningApi, HTTPPreconditions, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete,
|
EcstoreTierConfig, EcstoreVersioningApi, HTTPPreconditions, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete,
|
||||||
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
|
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
|
||||||
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
|
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
|
||||||
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd,
|
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_data_usage_snapshot_cache,
|
||||||
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
|
ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
|
||||||
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
|
ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
|
||||||
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
|
ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
|
||||||
scanner_replication_config_for_lifecycle_eval,
|
scanner_replication_config_for_lifecycle_eval,
|
||||||
};
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -462,6 +462,10 @@ pub(crate) async fn replace_bucket_usage_memory_from_info(data_usage_info: &rust
|
|||||||
ecstore_replace_bucket_usage_memory_from_info(data_usage_info).await;
|
ecstore_replace_bucket_usage_memory_from_info(data_usage_info).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn invalidate_data_usage_snapshot_cache() {
|
||||||
|
ecstore_invalidate_data_usage_snapshot_cache().await;
|
||||||
|
}
|
||||||
|
|
||||||
pub trait ScannerObjectIO:
|
pub trait ScannerObjectIO:
|
||||||
ObjectIO<
|
ObjectIO<
|
||||||
Error = EcstoreError,
|
Error = EcstoreError,
|
||||||
|
|||||||
+135
-90
@@ -61,8 +61,9 @@ use crate::storage_api::scan::{
|
|||||||
};
|
};
|
||||||
use crate::{
|
use crate::{
|
||||||
ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _,
|
ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _,
|
||||||
get_lifecycle_config, get_replication_config, read_config, replace_bucket_usage_memory_from_info, save_config,
|
get_lifecycle_config, get_replication_config, invalidate_data_usage_snapshot_cache, read_config,
|
||||||
save_config_shared_with_preconditions, save_config_with_preconditions, scanner_is_erasure_sd,
|
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions,
|
||||||
|
scanner_is_erasure_sd,
|
||||||
};
|
};
|
||||||
|
|
||||||
const LOG_COMPONENT_SCANNER: &str = "scanner";
|
const LOG_COMPONENT_SCANNER: &str = "scanner";
|
||||||
@@ -985,7 +986,7 @@ fn initial_scanner_delay_for_startup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn data_usage_info_is_cold(info: &DataUsageInfo) -> bool {
|
fn data_usage_info_is_cold(info: &DataUsageInfo) -> bool {
|
||||||
info.last_update.is_none() || (info.buckets_usage.is_empty() && info.bucket_sizes.is_empty())
|
!info.is_complete_bucket_usage_snapshot()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn read_data_usage_config_for_startup(storeapi: &Arc<impl ScannerObjectIO>) -> Result<Option<Vec<u8>>, EcstoreError> {
|
async fn read_data_usage_config_for_startup(storeapi: &Arc<impl ScannerObjectIO>) -> Result<Option<Vec<u8>>, EcstoreError> {
|
||||||
@@ -1004,10 +1005,12 @@ async fn read_data_usage_config_for_startup(storeapi: &Arc<impl ScannerObjectIO>
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
match read_pair(storeapi, DATA_USAGE_OBJ_NAME_PATH.as_str()).await? {
|
for path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] {
|
||||||
Some(data) => Ok(Some(data)),
|
if let Some(data) = read_pair(storeapi, path).await? {
|
||||||
None => read_pair(storeapi, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()).await,
|
return Ok(Some(data));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool {
|
fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool {
|
||||||
@@ -3611,6 +3614,21 @@ async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
|
|||||||
data_usage_info.scanner_epoch = Some(leader_epoch);
|
data_usage_info.scanner_epoch = Some(leader_epoch);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !data_usage_info.is_complete_bucket_usage_snapshot() {
|
||||||
|
error!(
|
||||||
|
target: "rustfs::scanner",
|
||||||
|
event = EVENT_SCANNER_PERSIST_STATE,
|
||||||
|
component = LOG_COMPONENT_SCANNER,
|
||||||
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||||
|
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||||
|
state = "reject_incomplete_snapshot",
|
||||||
|
"Scanner refused to persist an incomplete data usage snapshot"
|
||||||
|
);
|
||||||
|
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed);
|
||||||
|
outcome = DataUsagePersistOutcome::Failed;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
let data = match serde_json::to_vec(&data_usage_info) {
|
let data = match serde_json::to_vec(&data_usage_info) {
|
||||||
Ok(data) => data,
|
Ok(data) => data,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@@ -3767,16 +3785,19 @@ async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
|
|||||||
|
|
||||||
match save_outcome {
|
match save_outcome {
|
||||||
DataUsagePersistOutcome::Current => {
|
DataUsagePersistOutcome::Current => {
|
||||||
|
invalidate_data_usage_snapshot_cache().await;
|
||||||
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::SkippedStale);
|
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::SkippedStale);
|
||||||
outcome = DataUsagePersistOutcome::Current;
|
outcome = DataUsagePersistOutcome::Current;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
DataUsagePersistOutcome::AlreadyDurable => {
|
DataUsagePersistOutcome::AlreadyDurable => {
|
||||||
|
invalidate_data_usage_snapshot_cache().await;
|
||||||
replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
||||||
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
||||||
outcome = DataUsagePersistOutcome::AlreadyDurable;
|
outcome = DataUsagePersistOutcome::AlreadyDurable;
|
||||||
}
|
}
|
||||||
DataUsagePersistOutcome::PriorCycleDurable => {
|
DataUsagePersistOutcome::PriorCycleDurable => {
|
||||||
|
invalidate_data_usage_snapshot_cache().await;
|
||||||
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
||||||
outcome = DataUsagePersistOutcome::PriorCycleDurable;
|
outcome = DataUsagePersistOutcome::PriorCycleDurable;
|
||||||
}
|
}
|
||||||
@@ -3786,6 +3807,7 @@ async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
DataUsagePersistOutcome::Saved => {
|
DataUsagePersistOutcome::Saved => {
|
||||||
|
invalidate_data_usage_snapshot_cache().await;
|
||||||
replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
||||||
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
||||||
outcome = DataUsagePersistOutcome::Saved;
|
outcome = DataUsagePersistOutcome::Saved;
|
||||||
@@ -4485,8 +4507,25 @@ mod tests {
|
|||||||
assert_eq!(epoch, 11);
|
assert_eq!(epoch, 11);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() {
|
||||||
|
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::now()), 1);
|
||||||
|
legacy.usage_snapshot_complete = false;
|
||||||
|
|
||||||
|
assert!(data_usage_info_is_cold(&legacy));
|
||||||
|
assert!(!data_usage_info_is_cold(&complete_usage_with_bucket_count(
|
||||||
|
Some(std::time::SystemTime::now()),
|
||||||
|
1,
|
||||||
|
)));
|
||||||
|
assert!(!data_usage_info_is_cold(&DataUsageInfo {
|
||||||
|
last_update: Some(std::time::SystemTime::now()),
|
||||||
|
usage_snapshot_complete: true,
|
||||||
|
..Default::default()
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn scanner_startup_migrates_legacy_usage_only_until_v2_exists() {
|
async fn scanner_startup_prefers_v2_over_legacy_usage() {
|
||||||
let store = Arc::new(MemoryConfigStore::default());
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
let legacy = DataUsageInfo {
|
let legacy = DataUsageInfo {
|
||||||
scanner_epoch: Some(19),
|
scanner_epoch: Some(19),
|
||||||
@@ -4517,9 +4556,10 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
let authoritative = DataUsageInfo {
|
let authoritative = DataUsageInfo {
|
||||||
scanner_epoch: Some(7),
|
scanner_epoch: Some(23),
|
||||||
scanner_cycle: Some(11),
|
scanner_cycle: Some(51),
|
||||||
last_update: Some(std::time::SystemTime::now()),
|
last_update: Some(std::time::SystemTime::now()),
|
||||||
|
usage_snapshot_complete: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode");
|
let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode");
|
||||||
@@ -4539,8 +4579,8 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.expect("v2 usage floor should be authoritative"),
|
.expect("v2 usage floor should be authoritative"),
|
||||||
PersistedUsageFloor {
|
PersistedUsageFloor {
|
||||||
next_cycle: 12,
|
next_cycle: 52,
|
||||||
leader_epoch: 7,
|
leader_epoch: 23,
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -4594,7 +4634,7 @@ mod tests {
|
|||||||
scanner_epoch: Some(1),
|
scanner_epoch: Some(1),
|
||||||
scanner_cycle: Some(cycle),
|
scanner_cycle: Some(cycle),
|
||||||
last_update: Some(std::time::SystemTime::now()),
|
last_update: Some(std::time::SystemTime::now()),
|
||||||
..Default::default()
|
..complete_usage_with_bucket_count(None, 0)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("usage update should queue");
|
.expect("usage update should queue");
|
||||||
@@ -4959,16 +4999,8 @@ mod tests {
|
|||||||
let (sender, receiver) = mpsc::channel(2);
|
let (sender, receiver) = mpsc::channel(2);
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
|
|
||||||
let newer = DataUsageInfo {
|
let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
let older = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
|
||||||
buckets_count: 2,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let older = DataUsageInfo {
|
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
|
||||||
buckets_count: 1,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
|
|
||||||
sender.send(newer).await.expect("newer usage snapshot should enqueue");
|
sender.send(newer).await.expect("newer usage snapshot should enqueue");
|
||||||
sender.send(older).await.expect("older usage snapshot should enqueue");
|
sender.send(older).await.expect("older usage snapshot should enqueue");
|
||||||
@@ -4993,16 +5025,8 @@ mod tests {
|
|||||||
let (sender, receiver) = mpsc::channel(1);
|
let (sender, receiver) = mpsc::channel(1);
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||||
let newer = DataUsageInfo {
|
let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
let stale = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
|
||||||
buckets_count: 2,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let stale = DataUsageInfo {
|
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
|
||||||
buckets_count: 1,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
store
|
store
|
||||||
.interleaving_puts
|
.interleaving_puts
|
||||||
.lock()
|
.lock()
|
||||||
@@ -5045,6 +5069,7 @@ mod tests {
|
|||||||
initial.bucket_sizes.insert("bucket-a".to_string(), 84);
|
initial.bucket_sizes.insert("bucket-a".to_string(), 84);
|
||||||
initial.buckets_count = 1;
|
initial.buckets_count = 1;
|
||||||
initial.calculate_totals();
|
initial.calculate_totals();
|
||||||
|
mark_usage_snapshot_complete(&mut initial);
|
||||||
let initial_data = serde_json::to_vec(&initial).expect("initial usage snapshot should encode");
|
let initial_data = serde_json::to_vec(&initial).expect("initial usage snapshot should encode");
|
||||||
store.objects.lock().await.insert(key.clone(), initial_data.clone());
|
store.objects.lock().await.insert(key.clone(), initial_data.clone());
|
||||||
store.revisions.lock().await.insert(key.clone(), 1);
|
store.revisions.lock().await.insert(key.clone(), 1);
|
||||||
@@ -5054,6 +5079,7 @@ mod tests {
|
|||||||
deleted.bucket_sizes.clear();
|
deleted.bucket_sizes.clear();
|
||||||
deleted.buckets_count = 0;
|
deleted.buckets_count = 0;
|
||||||
deleted.calculate_totals();
|
deleted.calculate_totals();
|
||||||
|
mark_usage_snapshot_complete(&mut deleted);
|
||||||
store
|
store
|
||||||
.interleaving_puts
|
.interleaving_puts
|
||||||
.lock()
|
.lock()
|
||||||
@@ -5102,7 +5128,7 @@ mod tests {
|
|||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
||||||
scanner_epoch: Some(8),
|
scanner_epoch: Some(8),
|
||||||
scanner_cycle: Some(1),
|
scanner_cycle: Some(1),
|
||||||
..Default::default()
|
..complete_usage_with_bucket_count(None, 0)
|
||||||
};
|
};
|
||||||
store.objects.lock().await.insert(
|
store.objects.lock().await.insert(
|
||||||
backup_key.clone(),
|
backup_key.clone(),
|
||||||
@@ -5129,6 +5155,7 @@ mod tests {
|
|||||||
incoming.bucket_sizes.insert("bucket-a".to_string(), 84);
|
incoming.bucket_sizes.insert("bucket-a".to_string(), 84);
|
||||||
incoming.buckets_count = 1;
|
incoming.buckets_count = 1;
|
||||||
incoming.calculate_totals();
|
incoming.calculate_totals();
|
||||||
|
mark_usage_snapshot_complete(&mut incoming);
|
||||||
sender.send(incoming).await.expect("usage snapshot should enqueue");
|
sender.send(incoming).await.expect("usage snapshot should enqueue");
|
||||||
}
|
}
|
||||||
drop(sender);
|
drop(sender);
|
||||||
@@ -5162,7 +5189,7 @@ mod tests {
|
|||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
|
||||||
scanner_epoch: Some(8),
|
scanner_epoch: Some(8),
|
||||||
scanner_cycle: Some(10),
|
scanner_cycle: Some(10),
|
||||||
..Default::default()
|
..complete_usage_with_bucket_count(None, 0)
|
||||||
};
|
};
|
||||||
let encoded = serde_json::to_vec(&durable).expect("usage snapshot should encode");
|
let encoded = serde_json::to_vec(&durable).expect("usage snapshot should encode");
|
||||||
store.objects.lock().await.insert(main_key.clone(), encoded.clone());
|
store.objects.lock().await.insert(main_key.clone(), encoded.clone());
|
||||||
@@ -5205,6 +5232,7 @@ mod tests {
|
|||||||
incoming.bucket_sizes.insert("bucket-a".to_string(), 84);
|
incoming.bucket_sizes.insert("bucket-a".to_string(), 84);
|
||||||
incoming.buckets_count = 1;
|
incoming.buckets_count = 1;
|
||||||
incoming.calculate_totals();
|
incoming.calculate_totals();
|
||||||
|
mark_usage_snapshot_complete(&mut incoming);
|
||||||
|
|
||||||
let mut deleted = incoming.clone();
|
let mut deleted = incoming.clone();
|
||||||
deleted.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(31));
|
deleted.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(31));
|
||||||
@@ -5212,6 +5240,7 @@ mod tests {
|
|||||||
deleted.bucket_sizes.clear();
|
deleted.bucket_sizes.clear();
|
||||||
deleted.buckets_count = 0;
|
deleted.buckets_count = 0;
|
||||||
deleted.calculate_totals();
|
deleted.calculate_totals();
|
||||||
|
mark_usage_snapshot_complete(&mut deleted);
|
||||||
store.replace_after_successful_puts.lock().await.insert(
|
store.replace_after_successful_puts.lock().await.insert(
|
||||||
main_key.clone(),
|
main_key.clone(),
|
||||||
(1, serde_json::to_vec(&deleted).expect("deleted primary snapshot should encode")),
|
(1, serde_json::to_vec(&deleted).expect("deleted primary snapshot should encode")),
|
||||||
@@ -5251,21 +5280,9 @@ mod tests {
|
|||||||
let (sender, receiver) = mpsc::channel(1);
|
let (sender, receiver) = mpsc::channel(1);
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||||
let initial = DataUsageInfo {
|
let initial = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 3);
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
let stale_winner = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 3);
|
||||||
buckets_count: 1,
|
let current = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), 3);
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let stale_winner = DataUsageInfo {
|
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
|
||||||
buckets_count: 2,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let current = DataUsageInfo {
|
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
|
|
||||||
buckets_count: 3,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
store
|
store
|
||||||
.objects
|
.objects
|
||||||
.lock()
|
.lock()
|
||||||
@@ -5298,21 +5315,13 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_store_data_usage_in_backend_rejects_untimestamped_stale_snapshot() {
|
async fn test_store_data_usage_in_backend_rejects_untimestamped_complete_snapshot() {
|
||||||
let store = Arc::new(MemoryConfigStore::default());
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
let (sender, receiver) = mpsc::channel(2);
|
let (sender, receiver) = mpsc::channel(2);
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
|
|
||||||
let timestamped = DataUsageInfo {
|
let timestamped = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2);
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
let untimestamped = complete_usage_with_bucket_count(None, 1);
|
||||||
buckets_count: 2,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let untimestamped = DataUsageInfo {
|
|
||||||
last_update: None,
|
|
||||||
buckets_count: 1,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
|
|
||||||
sender
|
sender
|
||||||
.send(timestamped)
|
.send(timestamped)
|
||||||
@@ -5334,7 +5343,7 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(saved.buckets_count, 2);
|
assert_eq!(saved.buckets_count, 2);
|
||||||
assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)));
|
assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)));
|
||||||
assert_eq!(outcome, DataUsagePersistOutcome::Current);
|
assert_eq!(outcome, DataUsagePersistOutcome::Failed);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -5346,8 +5355,7 @@ mod tests {
|
|||||||
let snapshot = DataUsageInfo {
|
let snapshot = DataUsageInfo {
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
||||||
scanner_cycle: Some(12),
|
scanner_cycle: Some(12),
|
||||||
buckets_count: 2,
|
..complete_usage_with_bucket_count(None, 2)
|
||||||
..Default::default()
|
|
||||||
};
|
};
|
||||||
store
|
store
|
||||||
.objects
|
.objects
|
||||||
@@ -5377,8 +5385,7 @@ mod tests {
|
|||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
|
||||||
scanner_epoch: Some(8),
|
scanner_epoch: Some(8),
|
||||||
scanner_cycle: Some(12),
|
scanner_cycle: Some(12),
|
||||||
buckets_count: 2,
|
..complete_usage_with_bucket_count(None, 2)
|
||||||
..Default::default()
|
|
||||||
};
|
};
|
||||||
store
|
store
|
||||||
.objects
|
.objects
|
||||||
@@ -5392,8 +5399,7 @@ mod tests {
|
|||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)),
|
||||||
scanner_epoch: Some(8),
|
scanner_epoch: Some(8),
|
||||||
scanner_cycle: Some(12),
|
scanner_cycle: Some(12),
|
||||||
buckets_count: 3,
|
..complete_usage_with_bucket_count(None, 3)
|
||||||
..Default::default()
|
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("changed retry snapshot should enqueue");
|
.expect("changed retry snapshot should enqueue");
|
||||||
@@ -5426,8 +5432,7 @@ mod tests {
|
|||||||
let existing = DataUsageInfo {
|
let existing = DataUsageInfo {
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)),
|
||||||
scanner_cycle: Some(12),
|
scanner_cycle: Some(12),
|
||||||
buckets_count: 2,
|
..complete_usage_with_bucket_count(None, 2)
|
||||||
..Default::default()
|
|
||||||
};
|
};
|
||||||
store
|
store
|
||||||
.objects
|
.objects
|
||||||
@@ -5440,8 +5445,7 @@ mod tests {
|
|||||||
let older = DataUsageInfo {
|
let older = DataUsageInfo {
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
|
||||||
scanner_cycle: Some(11),
|
scanner_cycle: Some(11),
|
||||||
buckets_count: 1,
|
..complete_usage_with_bucket_count(None, 1)
|
||||||
..Default::default()
|
|
||||||
};
|
};
|
||||||
older_sender.send(older).await.expect("older-cycle snapshot should enqueue");
|
older_sender.send(older).await.expect("older-cycle snapshot should enqueue");
|
||||||
drop(older_sender);
|
drop(older_sender);
|
||||||
@@ -5454,8 +5458,7 @@ mod tests {
|
|||||||
let newer = DataUsageInfo {
|
let newer = DataUsageInfo {
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
|
||||||
scanner_cycle: Some(13),
|
scanner_cycle: Some(13),
|
||||||
buckets_count: 3,
|
..complete_usage_with_bucket_count(None, 3)
|
||||||
..Default::default()
|
|
||||||
};
|
};
|
||||||
newer_sender
|
newer_sender
|
||||||
.send(newer.clone())
|
.send(newer.clone())
|
||||||
@@ -5490,8 +5493,7 @@ mod tests {
|
|||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)),
|
||||||
scanner_epoch: Some(8),
|
scanner_epoch: Some(8),
|
||||||
scanner_cycle: Some(12),
|
scanner_cycle: Some(12),
|
||||||
buckets_count: 2,
|
..complete_usage_with_bucket_count(None, 2)
|
||||||
..Default::default()
|
|
||||||
};
|
};
|
||||||
store
|
store
|
||||||
.objects
|
.objects
|
||||||
@@ -5506,8 +5508,7 @@ mod tests {
|
|||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
|
||||||
scanner_epoch: Some(7),
|
scanner_epoch: Some(7),
|
||||||
scanner_cycle: Some(99),
|
scanner_cycle: Some(99),
|
||||||
buckets_count: 1,
|
..complete_usage_with_bucket_count(None, 1)
|
||||||
..Default::default()
|
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("old-epoch snapshot should enqueue");
|
.expect("old-epoch snapshot should enqueue");
|
||||||
@@ -5523,8 +5524,7 @@ mod tests {
|
|||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
|
||||||
scanner_epoch: None,
|
scanner_epoch: None,
|
||||||
scanner_cycle: Some(1),
|
scanner_cycle: Some(1),
|
||||||
buckets_count: 3,
|
..complete_usage_with_bucket_count(None, 3)
|
||||||
..Default::default()
|
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("replacement-epoch snapshot should enqueue");
|
.expect("replacement-epoch snapshot should enqueue");
|
||||||
@@ -5554,8 +5554,7 @@ mod tests {
|
|||||||
let existing = DataUsageInfo {
|
let existing = DataUsageInfo {
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)),
|
||||||
scanner_cycle: Some(12),
|
scanner_cycle: Some(12),
|
||||||
buckets_count: 2,
|
..complete_usage_with_bucket_count(None, 2)
|
||||||
..Default::default()
|
|
||||||
};
|
};
|
||||||
store
|
store
|
||||||
.objects
|
.objects
|
||||||
@@ -5569,8 +5568,7 @@ mod tests {
|
|||||||
.send(DataUsageInfo {
|
.send(DataUsageInfo {
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
|
||||||
scanner_cycle: Some(12),
|
scanner_cycle: Some(12),
|
||||||
buckets_count: 3,
|
..complete_usage_with_bucket_count(None, 3)
|
||||||
..Default::default()
|
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("conflicting same-cycle snapshot should enqueue");
|
.expect("conflicting same-cycle snapshot should enqueue");
|
||||||
@@ -5595,11 +5593,59 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo {
|
#[tokio::test]
|
||||||
DataUsageInfo {
|
async fn test_store_data_usage_in_backend_rejects_incomplete_snapshot() {
|
||||||
|
let store = Arc::new(MemoryConfigStore::default());
|
||||||
|
let (sender, receiver) = mpsc::channel(2);
|
||||||
|
let complete_update = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10);
|
||||||
|
|
||||||
|
sender
|
||||||
|
.send(complete_usage_with_bucket_count(Some(complete_update), 1))
|
||||||
|
.await
|
||||||
|
.expect("complete usage snapshot should enqueue");
|
||||||
|
sender
|
||||||
|
.send(DataUsageInfo {
|
||||||
|
last_update: Some(complete_update + Duration::from_secs(1)),
|
||||||
|
buckets_count: 1,
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("incomplete usage snapshot should enqueue");
|
||||||
|
drop(sender);
|
||||||
|
|
||||||
|
let outcome = store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await;
|
||||||
|
|
||||||
|
let objects = store.objects.lock().await;
|
||||||
|
let saved = objects
|
||||||
|
.get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()))
|
||||||
|
.expect("complete data usage snapshot should remain saved");
|
||||||
|
let saved = serde_json::from_slice::<DataUsageInfo>(saved).expect("saved usage snapshot should decode");
|
||||||
|
assert_eq!(saved.last_update, Some(complete_update));
|
||||||
|
assert!(saved.is_complete_bucket_usage_snapshot());
|
||||||
|
assert_eq!(outcome, DataUsagePersistOutcome::Failed);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mark_usage_snapshot_complete(info: &mut DataUsageInfo) {
|
||||||
|
info.usage_snapshot_complete = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn complete_usage_with_bucket_count(last_update: Option<std::time::SystemTime>, buckets_count: u64) -> DataUsageInfo {
|
||||||
|
let mut info = DataUsageInfo {
|
||||||
last_update,
|
last_update,
|
||||||
|
buckets_count,
|
||||||
|
usage_snapshot_complete: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
|
};
|
||||||
|
for index in 0..buckets_count {
|
||||||
|
let bucket = format!("bucket-{index}");
|
||||||
|
info.buckets_usage.insert(bucket.clone(), Default::default());
|
||||||
|
info.bucket_sizes.insert(bucket, 0);
|
||||||
}
|
}
|
||||||
|
info
|
||||||
|
}
|
||||||
|
|
||||||
|
fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo {
|
||||||
|
complete_usage_with_bucket_count(last_update, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -5681,11 +5727,10 @@ mod tests {
|
|||||||
|
|
||||||
for idx in 1_u64..=11 {
|
for idx in 1_u64..=11 {
|
||||||
sender
|
sender
|
||||||
.send(DataUsageInfo {
|
.send(complete_usage_with_bucket_count(
|
||||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(idx)),
|
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(idx)),
|
||||||
buckets_count: idx,
|
idx,
|
||||||
..Default::default()
|
))
|
||||||
})
|
|
||||||
.await
|
.await
|
||||||
.expect("usage snapshot should enqueue");
|
.expect("usage snapshot should enqueue");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -846,7 +846,7 @@ fn apply_bucket_result_to_cache(cache: &mut DataUsageCache, result: DataUsageEnt
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn should_publish_completed_snapshot(completed_count: usize, total_count: usize, budget_elapsed: bool, cancelled: bool) -> bool {
|
fn should_publish_completed_snapshot(completed_count: usize, total_count: usize, budget_elapsed: bool, cancelled: bool) -> bool {
|
||||||
total_count > 0 && completed_count == total_count && !budget_elapsed && !cancelled
|
completed_count == total_count && !budget_elapsed && !cancelled
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
@@ -1015,6 +1015,10 @@ fn completed_data_usage_info(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let merged_last_update = results.iter().filter_map(|result| result.info.last_update).max()?;
|
let merged_last_update = results.iter().filter_map(|result| result.info.last_update).max()?;
|
||||||
|
let bucket_sizes = buckets_usage
|
||||||
|
.iter()
|
||||||
|
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||||
|
.collect();
|
||||||
let data_usage_info = DataUsageInfo {
|
let data_usage_info = DataUsageInfo {
|
||||||
last_update: Some(merged_last_update),
|
last_update: Some(merged_last_update),
|
||||||
scanner_cycle: Some(results.first()?.info.next_cycle),
|
scanner_cycle: Some(results.first()?.info.next_cycle),
|
||||||
@@ -1023,7 +1027,9 @@ fn completed_data_usage_info(
|
|||||||
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
|
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
|
||||||
objects_total_size: u64::try_from(total.size).ok()?,
|
objects_total_size: u64::try_from(total.size).ok()?,
|
||||||
buckets_count: u64::try_from(all_buckets.len()).ok()?,
|
buckets_count: u64::try_from(all_buckets.len()).ok()?,
|
||||||
|
bucket_sizes,
|
||||||
buckets_usage,
|
buckets_usage,
|
||||||
|
usage_snapshot_complete: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
Some((data_usage_info, merged_last_update))
|
Some((data_usage_info, merged_last_update))
|
||||||
@@ -1042,7 +1048,10 @@ mod publish_gate_tests {
|
|||||||
assert!(!should_publish_completed_snapshot(2, 3, false, false));
|
assert!(!should_publish_completed_snapshot(2, 3, false, false));
|
||||||
assert!(!should_publish_completed_snapshot(3, 3, true, false));
|
assert!(!should_publish_completed_snapshot(3, 3, true, false));
|
||||||
assert!(!should_publish_completed_snapshot(3, 3, false, true));
|
assert!(!should_publish_completed_snapshot(3, 3, false, true));
|
||||||
assert!(!should_publish_completed_snapshot(0, 0, false, false));
|
assert!(
|
||||||
|
should_publish_completed_snapshot(0, 0, false, false),
|
||||||
|
"a completed empty namespace is an authoritative zero snapshot"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn incomplete_scope_cache(source: DataUsageCacheSource) -> DataUsageCache {
|
fn incomplete_scope_cache(source: DataUsageCacheSource) -> DataUsageCache {
|
||||||
@@ -1121,11 +1130,13 @@ mod publish_gate_tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn completed_data_usage_info_requires_every_set_before_publish() {
|
fn completed_data_usage_info_requires_every_set_before_publish() {
|
||||||
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()];
|
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string(), "bucket-empty".to_string()];
|
||||||
let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
|
let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
|
||||||
first_set.replace("bucket-b", DATA_USAGE_ROOT, DataUsageEntry::default());
|
first_set.replace("bucket-b", DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||||
|
first_set.replace("bucket-empty", DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||||
let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0));
|
let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0));
|
||||||
second_set.replace("bucket-a", DATA_USAGE_ROOT, DataUsageEntry::default());
|
second_set.replace("bucket-a", DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||||
|
second_set.replace("bucket-empty", DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
completed_data_usage_info_for_test(&[first_set.clone(), DataUsageCache::default()], &all_buckets, false, false)
|
completed_data_usage_info_for_test(&[first_set.clone(), DataUsageCache::default()], &all_buckets, false, false)
|
||||||
@@ -1144,7 +1155,38 @@ mod publish_gate_tests {
|
|||||||
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
|
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
|
||||||
assert_eq!(data_usage_info.scanner_cycle, Some(0));
|
assert_eq!(data_usage_info.scanner_cycle, Some(0));
|
||||||
assert_eq!(data_usage_info.objects_total_count, 3);
|
assert_eq!(data_usage_info.objects_total_count, 3);
|
||||||
assert_eq!(data_usage_info.buckets_usage.len(), 2);
|
assert_eq!(data_usage_info.buckets_usage.len(), 3);
|
||||||
|
assert!(data_usage_info.usage_snapshot_complete);
|
||||||
|
assert_eq!(
|
||||||
|
data_usage_info
|
||||||
|
.buckets_usage
|
||||||
|
.get("bucket-empty")
|
||||||
|
.map(|usage| (usage.objects_count, usage.size)),
|
||||||
|
Some((0, 0))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn completed_data_usage_info_publishes_confirmed_empty_namespace() {
|
||||||
|
let mut completed_set = DataUsageCache {
|
||||||
|
info: DataUsageCacheInfo {
|
||||||
|
name: DATA_USAGE_ROOT.to_string(),
|
||||||
|
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
||||||
|
source: Some(DataUsageCacheSource::new(0, 0)),
|
||||||
|
snapshot_complete: true,
|
||||||
|
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
completed_set.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
|
||||||
|
|
||||||
|
let (data_usage_info, _) = completed_data_usage_info_for_test(&[completed_set], &[], false, false)
|
||||||
|
.expect("a completed empty namespace should produce an authoritative snapshot");
|
||||||
|
|
||||||
|
assert!(data_usage_info.is_complete_bucket_usage_snapshot());
|
||||||
|
assert_eq!(data_usage_info.buckets_count, 0);
|
||||||
|
assert!(data_usage_info.buckets_usage.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2000,6 +2042,7 @@ impl ScannerIOCycle for ECStore {
|
|||||||
let empty_usage = DataUsageInfo {
|
let empty_usage = DataUsageInfo {
|
||||||
last_update: Some(SystemTime::now()),
|
last_update: Some(SystemTime::now()),
|
||||||
scanner_cycle: Some(want_cycle),
|
scanner_cycle: Some(want_cycle),
|
||||||
|
usage_snapshot_complete: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
send_data_usage_update(&updates, empty_usage).await?;
|
send_data_usage_update(&updates, empty_usage).await?;
|
||||||
|
|||||||
@@ -57,7 +57,10 @@ pub(crate) use rustfs_ecstore::api::config::init as ecstore_config_init;
|
|||||||
pub(crate) use rustfs_ecstore::api::config::storageclass::{
|
pub(crate) use rustfs_ecstore::api::config::storageclass::{
|
||||||
RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD,
|
RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD,
|
||||||
};
|
};
|
||||||
pub(crate) use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info;
|
pub(crate) use rustfs_ecstore::api::data_usage::{
|
||||||
|
invalidate_data_usage_snapshot_cache as ecstore_invalidate_data_usage_snapshot_cache,
|
||||||
|
replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info,
|
||||||
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
|
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
|
||||||
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
|
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
|
||||||
@@ -107,11 +110,12 @@ pub(crate) mod owner {
|
|||||||
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi,
|
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi,
|
||||||
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
|
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
|
||||||
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr,
|
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr,
|
||||||
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure,
|
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config,
|
||||||
ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
|
ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
|
||||||
ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
|
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
|
||||||
ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle,
|
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
|
||||||
ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
|
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
|
||||||
|
scanner_replication_config_for_lifecycle_eval,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ fn snapshot(bucket: &str, last_update: SystemTime) -> DataUsageInfo {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
info.usage_snapshot_complete = true;
|
||||||
info.bucket_sizes.insert(bucket.to_string(), 42);
|
info.bucket_sizes.insert(bucket.to_string(), 42);
|
||||||
info.buckets_count = 1;
|
info.buckets_count = 1;
|
||||||
info.calculate_totals();
|
info.calculate_totals();
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ for later deletion.
|
|||||||
|
|
||||||
## Open Items
|
## Open Items
|
||||||
|
|
||||||
- `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while the authoritative `.usage.v2.json` pair is absent and continue removing deleted buckets from legacy copies that still exist. Remove the fallback and legacy cleanup after every supported direct-upgrade source version writes `.usage.v2.json`.
|
- `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while `.usage.v2.json` is absent and continue removing deleted buckets from legacy copies that still exist. The additive usage_snapshot_complete field in `.usage.v2.json` must remain optional while mixed-version clusters are supported; a missing field means the snapshot is not authoritative. Remove the legacy object fallback and cleanup only after every supported direct-upgrade source writes `.usage.v2.json`.
|
||||||
- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Current protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while the distributed scanner publishes usage only after every peer returns authenticated protocol v5 state. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, and remove the protocol-v4 activity codec after every supported peer implements protocol v5; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
|
- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Current protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while the distributed scanner publishes usage only after every peer returns authenticated protocol v5 state. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, and remove the protocol-v4 activity codec after every supported peer implements protocol v5; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
|
||||||
- `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
|
- `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
|
||||||
- `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
|
- `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ use hyper::{Method, StatusCode};
|
|||||||
use matchit::Params;
|
use matchit::Params;
|
||||||
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
|
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
|
||||||
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
||||||
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
|
use s3s::{Body, S3Error, S3Request, S3Response, S3Result, s3_error};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json;
|
use serde_json;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -126,6 +126,16 @@ fn compat_bucket_quota_response(quota: &BucketQuota) -> CompatibleBucketQuotaRes
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn map_quota_query_error(error: QuotaError, bucket: &str, operation: &str) -> S3Error {
|
||||||
|
match error {
|
||||||
|
QuotaError::ConfigNotFound { .. } => s3_error!(NoSuchBucket, "bucket not found: {}", bucket),
|
||||||
|
QuotaError::UsageUnavailable { .. } => {
|
||||||
|
s3_error!(ServiceUnavailable, "authoritative bucket usage is not available yet")
|
||||||
|
}
|
||||||
|
error => s3_error!(InternalError, "failed to {}: {}", operation, error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
pub struct BucketQuotaResponse {
|
pub struct BucketQuotaResponse {
|
||||||
pub bucket: String,
|
pub bucket: String,
|
||||||
@@ -376,27 +386,34 @@ impl Operation for GetBucketQuotaHandler {
|
|||||||
|
|
||||||
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
|
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
|
||||||
|
|
||||||
let (quota, current_usage) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e {
|
if is_compat_get_bucket_quota_path(req.uri.path()) {
|
||||||
QuotaError::ConfigNotFound { .. } => {
|
if !quota_checker.bucket_exists(&bucket).await {
|
||||||
s3_error!(NoSuchBucket, "bucket not found: {}", bucket)
|
return Err(s3_error!(NoSuchBucket, "bucket not found: {}", bucket));
|
||||||
}
|
}
|
||||||
_ => s3_error!(InternalError, "failed to get quota: {}", e),
|
let quota = quota_checker
|
||||||
})?;
|
.get_quota_config(&bucket)
|
||||||
|
.await
|
||||||
|
.map_err(|e| s3_error!(InternalError, "failed to get quota: {}", e))?;
|
||||||
|
let json = serde_json::to_string(&compat_bucket_quota_response("a))
|
||||||
|
.map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
|
||||||
|
return Ok(S3Response::new((StatusCode::OK, Body::from(json))));
|
||||||
|
}
|
||||||
|
|
||||||
let json = if is_compat_get_bucket_quota_path(req.uri.path()) {
|
let (quota, current_usage) = quota_checker
|
||||||
serde_json::to_string(&compat_bucket_quota_response("a))
|
.get_quota_stats(&bucket)
|
||||||
.map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
|
.await
|
||||||
} else {
|
.map_err(|error| map_quota_query_error(error, &bucket, "get quota"))?;
|
||||||
let response = BucketQuotaResponse {
|
|
||||||
bucket,
|
|
||||||
quota: quota.quota,
|
|
||||||
size: current_usage.unwrap_or(0),
|
|
||||||
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
|
|
||||||
};
|
|
||||||
|
|
||||||
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
|
let response = BucketQuotaResponse {
|
||||||
|
bucket,
|
||||||
|
quota: quota.quota,
|
||||||
|
size: current_usage.unwrap_or(0),
|
||||||
|
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let json =
|
||||||
|
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
|
||||||
|
|
||||||
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
|
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -550,12 +567,10 @@ impl Operation for GetBucketQuotaStatsHandler {
|
|||||||
|
|
||||||
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
|
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
|
||||||
|
|
||||||
let (quota, current_usage_opt) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e {
|
let (quota, current_usage_opt) = quota_checker
|
||||||
QuotaError::ConfigNotFound { .. } => {
|
.get_quota_stats(&bucket)
|
||||||
s3_error!(NoSuchBucket, "bucket not found: {}", bucket)
|
.await
|
||||||
}
|
.map_err(|error| map_quota_query_error(error, &bucket, "get quota stats"))?;
|
||||||
_ => s3_error!(InternalError, "failed to get quota stats: {}", e),
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let current_usage = current_usage_opt.unwrap_or(0);
|
let current_usage = current_usage_opt.unwrap_or(0);
|
||||||
let usage_percentage = quota.quota.and_then(|limit| {
|
let usage_percentage = quota.quota.and_then(|limit| {
|
||||||
@@ -659,7 +674,7 @@ impl Operation for CheckBucketQuotaHandler {
|
|||||||
let result = quota_checker
|
let result = quota_checker
|
||||||
.check_quota_with_usage_reporting(&bucket, operation, request.operation_size, true)
|
.check_quota_with_usage_reporting(&bucket, operation, request.operation_size, true)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| s3_error!(InternalError, "failed to check quota: {}", e))?;
|
.map_err(|error| map_quota_query_error(error, &bucket, "check quota"))?;
|
||||||
|
|
||||||
let response = CheckQuotaResponse {
|
let response = CheckQuotaResponse {
|
||||||
bucket,
|
bucket,
|
||||||
@@ -715,6 +730,19 @@ mod tests {
|
|||||||
assert!(json.contains("\"quotatype\":\"hard\""));
|
assert!(json.contains("\"quotatype\":\"hard\""));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_authoritative_usage_is_retryable() {
|
||||||
|
let error = map_quota_query_error(
|
||||||
|
QuotaError::UsageUnavailable {
|
||||||
|
bucket: "test-bucket".to_string(),
|
||||||
|
},
|
||||||
|
"test-bucket",
|
||||||
|
"check quota",
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(error.code(), &s3s::S3ErrorCode::ServiceUnavailable);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_quota_operation_parsing() {
|
fn test_quota_operation_parsing() {
|
||||||
let parse_operation = |operation: &str| match operation.to_uppercase().as_str() {
|
let parse_operation = |operation: &str| match operation.to_uppercase().as_str() {
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ use super::storage_api::admin_usecase::capacity::{
|
|||||||
PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||||
};
|
};
|
||||||
use super::storage_api::admin_usecase::contract::StorageAdminApi;
|
use super::storage_api::admin_usecase::contract::StorageAdminApi;
|
||||||
|
use super::storage_api::admin_usecase::contract::bucket::{BucketOperations as _, BucketOptions};
|
||||||
use super::storage_api::admin_usecase::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend_cached};
|
use super::storage_api::admin_usecase::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend_cached};
|
||||||
use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
|
use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
|
||||||
use crate::app::runtime_sources::{
|
use crate::app::runtime_sources::{
|
||||||
@@ -33,6 +34,7 @@ use crate::server::{DependencyReadiness, collect_dependency_readiness_report as
|
|||||||
use rustfs_data_usage::DataUsageInfo;
|
use rustfs_data_usage::DataUsageInfo;
|
||||||
use rustfs_madmin::{InfoMessage, StorageInfo};
|
use rustfs_madmin::{InfoMessage, StorageInfo};
|
||||||
use s3s::S3ErrorCode;
|
use s3s::S3ErrorCode;
|
||||||
|
use std::collections::HashSet;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
|
|
||||||
@@ -212,6 +214,16 @@ impl DefaultAdminUsecase {
|
|||||||
result.map_err(|_| Self::app_error(S3ErrorCode::InternalError, "load_data_usage_from_backend failed"))
|
result.map_err(|_| Self::app_error(S3ErrorCode::InternalError, "load_data_usage_from_backend failed"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn data_usage_snapshot_covers_namespace(info: &DataUsageInfo, buckets: impl IntoIterator<Item = String>) -> bool {
|
||||||
|
if !info.is_complete_bucket_usage_snapshot() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let current_buckets: HashSet<String> = buckets.into_iter().filter(|bucket| !bucket.starts_with('.')).collect();
|
||||||
|
current_buckets.len() == info.buckets_usage.len()
|
||||||
|
&& current_buckets.iter().all(|bucket| info.buckets_usage.contains_key(bucket))
|
||||||
|
}
|
||||||
|
|
||||||
async fn refresh_rebalance_status_snapshot(store: &ECStore) -> AdminUsecaseResult<()> {
|
async fn refresh_rebalance_status_snapshot(store: &ECStore) -> AdminUsecaseResult<()> {
|
||||||
store.refresh_rebalance_status_meta().await.map_err(|err| {
|
store.refresh_rebalance_status_meta().await.map_err(|err| {
|
||||||
error!("refresh rebalance metadata for pool status failed: {:?}", err);
|
error!("refresh rebalance metadata for pool status failed: {:?}", err);
|
||||||
@@ -255,6 +267,20 @@ impl DefaultAdminUsecase {
|
|||||||
pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
|
pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
|
||||||
let mut info = Self::map_data_usage_load_result(load_data_usage_from_backend_cached(store.clone()).await)?;
|
let mut info = Self::map_data_usage_load_result(load_data_usage_from_backend_cached(store.clone()).await)?;
|
||||||
apply_bucket_usage_memory_overlay(&mut info).await;
|
apply_bucket_usage_memory_overlay(&mut info).await;
|
||||||
|
let buckets = store
|
||||||
|
.list_bucket(&BucketOptions {
|
||||||
|
cached: true,
|
||||||
|
no_metadata: true,
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| Self::app_error(S3ErrorCode::InternalError, "list_bucket failed"))?;
|
||||||
|
if !Self::data_usage_snapshot_covers_namespace(&info, buckets.into_iter().map(|bucket| bucket.name)) {
|
||||||
|
return Err(Self::app_error(
|
||||||
|
S3ErrorCode::ServiceUnavailable,
|
||||||
|
"authoritative data usage snapshot is not available yet",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
|
let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
|
||||||
|
|
||||||
@@ -645,6 +671,32 @@ mod tests {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn data_usage_snapshot_must_cover_the_current_bucket_namespace() {
|
||||||
|
let mut info = DataUsageInfo {
|
||||||
|
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
|
||||||
|
usage_snapshot_complete: true,
|
||||||
|
buckets_count: 1,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
info.buckets_usage.insert("bucket-a".to_string(), Default::default());
|
||||||
|
|
||||||
|
assert!(DefaultAdminUsecase::data_usage_snapshot_covers_namespace(
|
||||||
|
&info,
|
||||||
|
["bucket-a".to_string(), ".rustfs.sys".to_string()]
|
||||||
|
));
|
||||||
|
assert!(!DefaultAdminUsecase::data_usage_snapshot_covers_namespace(
|
||||||
|
&info,
|
||||||
|
["bucket-a".to_string(), "bucket-b".to_string()]
|
||||||
|
));
|
||||||
|
|
||||||
|
info.usage_snapshot_complete = false;
|
||||||
|
assert!(!DefaultAdminUsecase::data_usage_snapshot_covers_namespace(
|
||||||
|
&info,
|
||||||
|
["bucket-a".to_string()]
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn execute_query_storage_info_returns_internal_error_when_store_uninitialized() {
|
async fn execute_query_storage_info_returns_internal_error_when_store_uninitialized() {
|
||||||
let usecase = DefaultAdminUsecase::without_context();
|
let usecase = DefaultAdminUsecase::without_context();
|
||||||
|
|||||||
@@ -25,23 +25,36 @@
|
|||||||
//! while the response carries the seeded numbers.
|
//! while the response carries the seeded numbers.
|
||||||
|
|
||||||
use super::gating_test_env::shared_gating_ecstore;
|
use super::gating_test_env::shared_gating_ecstore;
|
||||||
use super::storage_api::test::StoragePutObjReader as PutObjReader;
|
use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions, MakeBucketOptions};
|
||||||
use super::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions};
|
|
||||||
use super::storage_api::test::contract::object::ObjectIO as _;
|
use super::storage_api::test::contract::object::ObjectIO as _;
|
||||||
use super::storage_api::test::data_usage::{
|
use super::storage_api::test::data_usage::{
|
||||||
compute_bucket_usage, live_bucket_usage_computations, load_data_usage_from_backend_cached, record_bucket_object_write_memory,
|
compute_bucket_usage, live_bucket_usage_computations, load_data_usage_from_backend_cached, record_bucket_object_write_memory,
|
||||||
store_data_usage_in_backend,
|
store_data_usage_in_backend,
|
||||||
};
|
};
|
||||||
|
use super::storage_api::test::{ECStore, StoragePutObjReader as PutObjReader};
|
||||||
use crate::app::admin_usecase::DefaultAdminUsecase;
|
use crate::app::admin_usecase::DefaultAdminUsecase;
|
||||||
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
|
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, SystemTime};
|
use std::time::{Duration, SystemTime};
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
use tokio_util::sync::CancellationToken;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const SEEDED_BUCKET_SIZE: u64 = 123_456;
|
const SEEDED_BUCKET_SIZE: u64 = 123_456;
|
||||||
const SEEDED_BUCKET_OBJECTS: u64 = 42;
|
const SEEDED_BUCKET_OBJECTS: u64 = 42;
|
||||||
|
|
||||||
|
async fn next_snapshot_time(ecstore: Arc<ECStore>) -> SystemTime {
|
||||||
|
let now = SystemTime::now();
|
||||||
|
let latest = load_data_usage_from_backend_cached(ecstore)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.and_then(|info| info.last_update)
|
||||||
|
.unwrap_or(now);
|
||||||
|
latest.max(now) + Duration::from_nanos(1)
|
||||||
|
}
|
||||||
|
|
||||||
fn seeded_data_usage_info(bucket: &str, last_update: SystemTime) -> DataUsageInfo {
|
fn seeded_data_usage_info(bucket: &str, last_update: SystemTime) -> DataUsageInfo {
|
||||||
let usage = BucketUsageInfo {
|
let usage = BucketUsageInfo {
|
||||||
size: SEEDED_BUCKET_SIZE,
|
size: SEEDED_BUCKET_SIZE,
|
||||||
@@ -58,6 +71,7 @@ fn seeded_data_usage_info(bucket: &str, last_update: SystemTime) -> DataUsageInf
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
info.buckets_usage = HashMap::from([(bucket.to_string(), usage)]);
|
info.buckets_usage = HashMap::from([(bucket.to_string(), usage)]);
|
||||||
|
info.usage_snapshot_complete = true;
|
||||||
info.bucket_sizes = HashMap::from([(bucket.to_string(), SEEDED_BUCKET_SIZE)]);
|
info.bucket_sizes = HashMap::from([(bucket.to_string(), SEEDED_BUCKET_SIZE)]);
|
||||||
info
|
info
|
||||||
}
|
}
|
||||||
@@ -78,10 +92,18 @@ fn sized_data_usage_info(bucket: &str, size: u64, last_update: SystemTime) -> Da
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
info.buckets_usage = HashMap::from([(bucket.to_string(), usage)]);
|
info.buckets_usage = HashMap::from([(bucket.to_string(), usage)]);
|
||||||
|
info.usage_snapshot_complete = true;
|
||||||
info.bucket_sizes = HashMap::from([(bucket.to_string(), size)]);
|
info.bucket_sizes = HashMap::from([(bucket.to_string(), size)]);
|
||||||
info
|
info
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn persist_scanner_snapshot(ecstore: Arc<ECStore>, info: DataUsageInfo) {
|
||||||
|
let (sender, receiver) = mpsc::channel(1);
|
||||||
|
sender.send(info).await.expect("enqueue scanner snapshot");
|
||||||
|
drop(sender);
|
||||||
|
rustfs_scanner::scanner::store_data_usage_in_backend(CancellationToken::new(), ecstore, receiver).await;
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
|
async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
|
||||||
@@ -121,10 +143,34 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
|
|||||||
.make_bucket(&seeded_bucket, &MakeBucketOptions::default())
|
.make_bucket(&seeded_bucket, &MakeBucketOptions::default())
|
||||||
.await
|
.await
|
||||||
.expect("create seeded bucket");
|
.expect("create seeded bucket");
|
||||||
let seeded_at = SystemTime::now();
|
ecstore
|
||||||
store_data_usage_in_backend(seeded_data_usage_info(&seeded_bucket, seeded_at), ecstore.clone())
|
.make_bucket(&overlay_bucket, &MakeBucketOptions::default())
|
||||||
.await
|
.await
|
||||||
.expect("persist seeded data usage snapshot");
|
.expect("create overlay bucket");
|
||||||
|
let seeded_at = next_snapshot_time(ecstore.clone()).await;
|
||||||
|
let mut seeded_info = seeded_data_usage_info(&seeded_bucket, seeded_at);
|
||||||
|
for bucket in ecstore
|
||||||
|
.list_bucket(&BucketOptions {
|
||||||
|
no_metadata: true,
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("list the complete live bucket namespace")
|
||||||
|
.into_iter()
|
||||||
|
.filter(|bucket| !bucket.name.starts_with('.'))
|
||||||
|
{
|
||||||
|
seeded_info.buckets_usage.entry(bucket.name.clone()).or_default();
|
||||||
|
seeded_info.bucket_sizes.entry(bucket.name).or_default();
|
||||||
|
}
|
||||||
|
seeded_info.bucket_sizes.insert(live_bucket.clone(), control_usage.size);
|
||||||
|
seeded_info.buckets_usage.insert(live_bucket.clone(), control_usage);
|
||||||
|
seeded_info
|
||||||
|
.buckets_usage
|
||||||
|
.insert(overlay_bucket.clone(), BucketUsageInfo::default());
|
||||||
|
seeded_info.bucket_sizes.insert(overlay_bucket.clone(), 0);
|
||||||
|
seeded_info.buckets_count = u64::try_from(seeded_info.buckets_usage.len()).expect("test bucket count should fit into u64");
|
||||||
|
seeded_info.calculate_totals();
|
||||||
|
persist_scanner_snapshot(ecstore.clone(), seeded_info).await;
|
||||||
|
|
||||||
record_bucket_object_write_memory(&overlay_bucket, None, 512).await;
|
record_bucket_object_write_memory(&overlay_bucket, None, 512).await;
|
||||||
|
|
||||||
@@ -182,6 +228,138 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
|
|||||||
assert_eq!(overlay_usage.size, 512);
|
assert_eq!(overlay_usage.size, 512);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn data_usage_endpoint_treats_legacy_partial_stats_as_unknown_until_covered() {
|
||||||
|
let ecstore = shared_gating_ecstore().await;
|
||||||
|
let agent_bucket = format!("usage-legacy-agent-{}", Uuid::new_v4());
|
||||||
|
let small_bucket = format!("usage-legacy-small-{}", Uuid::new_v4());
|
||||||
|
ecstore
|
||||||
|
.make_bucket(&agent_bucket, &MakeBucketOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("create legacy partial snapshot bucket");
|
||||||
|
ecstore
|
||||||
|
.make_bucket(&small_bucket, &MakeBucketOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("create legacy zero snapshot bucket");
|
||||||
|
let mut reader = PutObjReader::from_vec(b"object exists".to_vec());
|
||||||
|
ecstore
|
||||||
|
.put_object(&agent_bucket, "object.bin", &mut reader, &Default::default())
|
||||||
|
.await
|
||||||
|
.expect("put object before persisting legacy partial snapshot");
|
||||||
|
let mut small_reader = PutObjReader::from_vec(vec![b'x'; 57_344]);
|
||||||
|
ecstore
|
||||||
|
.put_object(&small_bucket, "small.bin", &mut small_reader, &Default::default())
|
||||||
|
.await
|
||||||
|
.expect("put object before persisting legacy zero snapshot");
|
||||||
|
|
||||||
|
let persisted_at = next_snapshot_time(ecstore.clone()).await;
|
||||||
|
let mut legacy = DataUsageInfo {
|
||||||
|
last_update: Some(persisted_at),
|
||||||
|
buckets_count: 2,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
legacy.buckets_usage.insert(
|
||||||
|
agent_bucket.clone(),
|
||||||
|
BucketUsageInfo {
|
||||||
|
size: 196_870_144,
|
||||||
|
objects_count: 1_502,
|
||||||
|
versions_count: 1_502,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
legacy.buckets_usage.insert(small_bucket.clone(), BucketUsageInfo::default());
|
||||||
|
legacy.bucket_sizes.insert(agent_bucket.clone(), 196_870_144);
|
||||||
|
legacy.bucket_sizes.insert(small_bucket.clone(), 0);
|
||||||
|
store_data_usage_in_backend(legacy, ecstore.clone())
|
||||||
|
.await
|
||||||
|
.expect("persist legacy partial snapshot");
|
||||||
|
|
||||||
|
let before_endpoint = live_bucket_usage_computations();
|
||||||
|
let unknown = DefaultAdminUsecase::query_data_usage_info_with_store(ecstore.clone())
|
||||||
|
.await
|
||||||
|
.expect_err("an uncovered snapshot must be reported as temporarily unavailable");
|
||||||
|
|
||||||
|
assert_eq!(unknown.code, s3s::S3ErrorCode::ServiceUnavailable);
|
||||||
|
assert_eq!(
|
||||||
|
live_bucket_usage_computations(),
|
||||||
|
before_endpoint,
|
||||||
|
"uncovered snapshot handling must not restore request-time object listings"
|
||||||
|
);
|
||||||
|
|
||||||
|
let completed_at = persisted_at + Duration::from_nanos(1);
|
||||||
|
let mut completed = DataUsageInfo {
|
||||||
|
last_update: Some(completed_at),
|
||||||
|
usage_snapshot_complete: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
for bucket in ecstore
|
||||||
|
.list_bucket(&BucketOptions {
|
||||||
|
no_metadata: true,
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("list the complete live bucket namespace")
|
||||||
|
.into_iter()
|
||||||
|
.filter(|bucket| !bucket.name.starts_with('.'))
|
||||||
|
{
|
||||||
|
completed
|
||||||
|
.buckets_usage
|
||||||
|
.insert(bucket.name.clone(), BucketUsageInfo::default());
|
||||||
|
completed.bucket_sizes.insert(bucket.name, 0);
|
||||||
|
}
|
||||||
|
completed.buckets_usage.insert(
|
||||||
|
agent_bucket.clone(),
|
||||||
|
BucketUsageInfo {
|
||||||
|
size: 222_822_400,
|
||||||
|
objects_count: 1_700,
|
||||||
|
versions_count: 1_700,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
completed.bucket_sizes.insert(agent_bucket.clone(), 222_822_400);
|
||||||
|
completed.buckets_usage.insert(
|
||||||
|
small_bucket.clone(),
|
||||||
|
BucketUsageInfo {
|
||||||
|
size: 57_344,
|
||||||
|
objects_count: 1,
|
||||||
|
versions_count: 1,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
completed.bucket_sizes.insert(small_bucket.clone(), 57_344);
|
||||||
|
completed.buckets_count = u64::try_from(completed.buckets_usage.len()).expect("test bucket count should fit into u64");
|
||||||
|
completed.calculate_totals();
|
||||||
|
store_data_usage_in_backend(completed, ecstore.clone())
|
||||||
|
.await
|
||||||
|
.expect("persist complete covered snapshot");
|
||||||
|
|
||||||
|
let covered = DefaultAdminUsecase::query_data_usage_info_with_store(ecstore)
|
||||||
|
.await
|
||||||
|
.expect("query covered data usage info");
|
||||||
|
assert_eq!(covered.last_update, Some(completed_at));
|
||||||
|
assert!(covered.usage_snapshot_complete);
|
||||||
|
assert_eq!(
|
||||||
|
covered
|
||||||
|
.buckets_usage
|
||||||
|
.get(&agent_bucket)
|
||||||
|
.map(|usage| (usage.objects_count, usage.size)),
|
||||||
|
Some((1_700, 222_822_400))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
covered
|
||||||
|
.buckets_usage
|
||||||
|
.get(&small_bucket)
|
||||||
|
.map(|usage| (usage.objects_count, usage.size)),
|
||||||
|
Some((1, 57_344))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
live_bucket_usage_computations(),
|
||||||
|
before_endpoint,
|
||||||
|
"covered snapshot publication must not trigger request-time object listings"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// Revert detector for the snapshot-cache invalidation in
|
/// Revert detector for the snapshot-cache invalidation in
|
||||||
/// `save_data_usage_in_backend` (rustfs/backlog#1306): a fresh scanner save must
|
/// `save_data_usage_in_backend` (rustfs/backlog#1306): a fresh scanner save must
|
||||||
/// be visible to the very next cached read, not deferred until the 30s TTL
|
/// be visible to the very next cached read, not deferred until the 30s TTL
|
||||||
@@ -194,11 +372,11 @@ async fn save_data_usage_invalidates_snapshot_cache() {
|
|||||||
let ecstore = shared_gating_ecstore().await;
|
let ecstore = shared_gating_ecstore().await;
|
||||||
let bucket = format!("usage-invalidate-{}", Uuid::new_v4());
|
let bucket = format!("usage-invalidate-{}", Uuid::new_v4());
|
||||||
|
|
||||||
// Future-date both snapshots beyond the stale-guard's 5min tolerance so the
|
// Advance beyond the latest persisted timestamp by the smallest possible
|
||||||
// persists win deterministically over whatever snapshot a sibling serial
|
// increment so this save wins the monotonicity guard without masking a
|
||||||
// test may have left behind, regardless of test execution order.
|
// subsequent in-memory mutation as older than the snapshot.
|
||||||
let warm_at = SystemTime::now() + Duration::from_secs(600);
|
let warm_at = next_snapshot_time(ecstore.clone()).await;
|
||||||
let changed_at = warm_at + Duration::from_secs(1);
|
let changed_at = warm_at + Duration::from_nanos(1);
|
||||||
|
|
||||||
const WARM_SIZE: u64 = 111_000;
|
const WARM_SIZE: u64 = 111_000;
|
||||||
const CHANGED_SIZE: u64 = 222_000;
|
const CHANGED_SIZE: u64 = 222_000;
|
||||||
@@ -236,6 +414,38 @@ async fn save_data_usage_invalidates_snapshot_cache() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn scanner_save_invalidates_snapshot_cache() {
|
||||||
|
let ecstore = shared_gating_ecstore().await;
|
||||||
|
let bucket = format!("usage-scanner-invalidate-{}", Uuid::new_v4());
|
||||||
|
let warm_at = next_snapshot_time(ecstore.clone()).await;
|
||||||
|
let changed_at = warm_at + Duration::from_nanos(1);
|
||||||
|
|
||||||
|
const WARM_SIZE: u64 = 333_000;
|
||||||
|
const CHANGED_SIZE: u64 = 444_000;
|
||||||
|
|
||||||
|
store_data_usage_in_backend(sized_data_usage_info(&bucket, WARM_SIZE, warm_at), ecstore.clone())
|
||||||
|
.await
|
||||||
|
.expect("persist initial snapshot");
|
||||||
|
let warmed = load_data_usage_from_backend_cached(ecstore.clone())
|
||||||
|
.await
|
||||||
|
.expect("warm cached snapshot");
|
||||||
|
assert_eq!(warmed.buckets_usage.get(&bucket).map(|usage| usage.size), Some(WARM_SIZE));
|
||||||
|
|
||||||
|
persist_scanner_snapshot(ecstore.clone(), sized_data_usage_info(&bucket, CHANGED_SIZE, changed_at)).await;
|
||||||
|
|
||||||
|
let reloaded = load_data_usage_from_backend_cached(ecstore)
|
||||||
|
.await
|
||||||
|
.expect("reload cached snapshot after scanner save");
|
||||||
|
assert_eq!(
|
||||||
|
reloaded.buckets_usage.get(&bucket).map(|usage| usage.size),
|
||||||
|
Some(CHANGED_SIZE),
|
||||||
|
"scanner persistence must invalidate the process-local snapshot cache"
|
||||||
|
);
|
||||||
|
assert_eq!(reloaded.last_update, Some(changed_at));
|
||||||
|
}
|
||||||
|
|
||||||
/// Wire pin for the no-snapshot response shape (rustfs/backlog#1306): a
|
/// Wire pin for the no-snapshot response shape (rustfs/backlog#1306): a
|
||||||
/// default `DataUsageInfo` must keep serializing `last_update` as `null` with
|
/// default `DataUsageInfo` must keep serializing `last_update` as `null` with
|
||||||
/// empty bucket maps, so "no snapshot yet" stays distinguishable from real
|
/// empty bucket maps, so "no snapshot yet" stays distinguishable from real
|
||||||
|
|||||||
@@ -58,7 +58,9 @@ use crate::app::object_data_cache::{
|
|||||||
ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success,
|
ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success,
|
||||||
invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation,
|
invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation,
|
||||||
};
|
};
|
||||||
use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write};
|
use crate::app::object_usecase::{
|
||||||
|
build_put_like_object_lock_metadata, map_quota_check_outcome, validate_existing_object_lock_for_write,
|
||||||
|
};
|
||||||
use crate::app::runtime_sources::{
|
use crate::app::runtime_sources::{
|
||||||
AppContext, current_app_context, current_object_data_cache_for_context, current_object_store_handle_for_context,
|
AppContext, current_app_context, current_object_data_cache_for_context, current_object_store_handle_for_context,
|
||||||
};
|
};
|
||||||
@@ -516,6 +518,12 @@ impl DefaultMultipartUsecase {
|
|||||||
_ => None,
|
_ => None,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let quota_metadata_sys = self.bucket_metadata_sys();
|
||||||
|
if let Some(metadata_sys) = quota_metadata_sys.as_ref() {
|
||||||
|
let quota_checker = QuotaChecker::new(metadata_sys.clone());
|
||||||
|
map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?;
|
||||||
|
}
|
||||||
|
|
||||||
let obj_info = store
|
let obj_info = store
|
||||||
.clone()
|
.clone()
|
||||||
.complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, &opts)
|
.complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, &opts)
|
||||||
@@ -524,17 +532,18 @@ impl DefaultMultipartUsecase {
|
|||||||
let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await;
|
let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await;
|
||||||
record_capacity_write(Some(capacity_scope_token)).await;
|
record_capacity_write(Some(capacity_scope_token)).await;
|
||||||
|
|
||||||
// check quota after completing multipart upload
|
if let Some(metadata_sys) = quota_metadata_sys {
|
||||||
if let Some(metadata_sys) = self.bucket_metadata_sys() {
|
|
||||||
let quota_checker = QuotaChecker::new(metadata_sys);
|
let quota_checker = QuotaChecker::new(metadata_sys);
|
||||||
|
|
||||||
match quota_checker
|
match quota_checker
|
||||||
.check_quota(&bucket, QuotaOperation::PutObject, obj_info.size as u64)
|
.check_quota(&bucket, QuotaOperation::PutObject, obj_info.size.max(0) as u64)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(check_result) => {
|
Ok(check_result) => {
|
||||||
if !check_result.allowed {
|
if !check_result.allowed {
|
||||||
// Quota exceeded, delete the completed object
|
// Preserve the established compensation behavior for
|
||||||
|
// a known over-quota result. Unknown usage is rejected
|
||||||
|
// by the preflight check before the upload is committed.
|
||||||
let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await;
|
let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await;
|
||||||
let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await;
|
let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await;
|
||||||
return Err(S3Error::with_message(
|
return Err(S3Error::with_message(
|
||||||
@@ -546,18 +555,17 @@ impl DefaultMultipartUsecase {
|
|||||||
),
|
),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
// Update quota tracking after successful multipart upload
|
|
||||||
if versioned {
|
|
||||||
record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64)
|
|
||||||
.await;
|
|
||||||
} else {
|
|
||||||
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(err) => {
|
||||||
warn!("Quota check failed for bucket {}: {}, allowing operation", bucket, e);
|
warn!("Quota check failed for bucket {} after multipart completion: {}", bucket, err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if versioned {
|
||||||
|
record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
|
||||||
|
} else {
|
||||||
|
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
|
enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
|
||||||
|
|||||||
@@ -501,7 +501,7 @@ fn request_body_is_aws_chunked_framed(headers: &HeaderMap) -> bool {
|
|||||||
/// Map a bucket-quota checker outcome onto the S3 admission result.
|
/// Map a bucket-quota checker outcome onto the S3 admission result.
|
||||||
///
|
///
|
||||||
/// Hard is the only supported quota type, so a checker fault (bucket-config read, config parse, or usage lookup) must fail closed rather than admit the write: allowing it would silently bypass a configured hard quota. The no-quota happy path never reaches the error arm — `QuotaChecker::check_quota` returns `Ok(allowed)` via the zero-extra-I/O fast path when no quota is configured, so failing closed here cannot penalise buckets without a quota. A fault surfaces as a retryable `ServiceUnavailable` and is counted; the client-facing message stays generic so internal config/usage details are not leaked.
|
/// Hard is the only supported quota type, so a checker fault (bucket-config read, config parse, or usage lookup) must fail closed rather than admit the write: allowing it would silently bypass a configured hard quota. The no-quota happy path never reaches the error arm — `QuotaChecker::check_quota` returns `Ok(allowed)` via the zero-extra-I/O fast path when no quota is configured, so failing closed here cannot penalise buckets without a quota. A fault surfaces as a retryable `ServiceUnavailable` and is counted; the client-facing message stays generic so internal config/usage details are not leaked.
|
||||||
fn map_quota_check_outcome(bucket: &str, outcome: Result<QuotaCheckResult, QuotaError>) -> S3Result<()> {
|
pub(super) fn map_quota_check_outcome(bucket: &str, outcome: Result<QuotaCheckResult, QuotaError>) -> S3Result<QuotaCheckResult> {
|
||||||
match outcome {
|
match outcome {
|
||||||
Ok(result) if !result.allowed => Err(S3Error::with_message(
|
Ok(result) if !result.allowed => Err(S3Error::with_message(
|
||||||
S3ErrorCode::InvalidRequest,
|
S3ErrorCode::InvalidRequest,
|
||||||
@@ -513,13 +513,17 @@ fn map_quota_check_outcome(bucket: &str, outcome: Result<QuotaCheckResult, Quota
|
|||||||
)),
|
)),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
counter!("rustfs_bucket_quota_check_failed_total").increment(1);
|
counter!("rustfs_bucket_quota_check_failed_total").increment(1);
|
||||||
warn!(bucket, error = %e, state = "checker_failed", "Bucket quota check failed closed");
|
if matches!(&e, QuotaError::UsageUnavailable { .. }) {
|
||||||
|
debug!(bucket, error = %e, state = "usage_pending", "Bucket quota check waiting for authoritative usage");
|
||||||
|
} else {
|
||||||
|
warn!(bucket, error = %e, state = "checker_failed", "Bucket quota check failed closed");
|
||||||
|
}
|
||||||
Err(S3Error::with_message(
|
Err(S3Error::with_message(
|
||||||
S3ErrorCode::ServiceUnavailable,
|
S3ErrorCode::ServiceUnavailable,
|
||||||
"Bucket quota check temporarily unavailable, please retry".to_string(),
|
"Bucket quota check temporarily unavailable, please retry".to_string(),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
_ => Ok(()),
|
Ok(result) => Ok(result),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3263,7 +3267,7 @@ impl DefaultObjectUsecase {
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
let quota_checker = QuotaChecker::new(metadata_sys);
|
let quota_checker = QuotaChecker::new(metadata_sys);
|
||||||
map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await)
|
map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(|_| ())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_memory_bytes_blob(
|
fn build_memory_bytes_blob(
|
||||||
@@ -7983,25 +7987,9 @@ impl DefaultObjectUsecase {
|
|||||||
let extract_limits = put_object_extract_limits();
|
let extract_limits = put_object_extract_limits();
|
||||||
let extract_quota_snapshot = if let Some(metadata_sys) = self.bucket_metadata_sys() {
|
let extract_quota_snapshot = if let Some(metadata_sys) = self.bucket_metadata_sys() {
|
||||||
let quota_checker = QuotaChecker::new(metadata_sys);
|
let quota_checker = QuotaChecker::new(metadata_sys);
|
||||||
match quota_checker.get_quota_config(&bucket).await {
|
let check_result =
|
||||||
Ok(quota) => {
|
map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?;
|
||||||
if let Some(limit) = quota.quota {
|
check_result.current_usage.zip(check_result.quota_limit)
|
||||||
match quota_checker.get_real_time_usage(&bucket).await {
|
|
||||||
Ok(current_usage) => Some((current_usage, limit)),
|
|
||||||
Err(err) => {
|
|
||||||
warn!(bucket, error = %err, state = "extract_usage_snapshot_failed", "Bucket quota snapshot degraded to allow");
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
warn!(bucket, error = %err, state = "extract_quota_snapshot_failed", "Bucket quota snapshot degraded to allow");
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
@@ -8191,8 +8179,11 @@ impl DefaultObjectUsecase {
|
|||||||
let cache_adapter = self.object_data_cache();
|
let cache_adapter = self.object_data_cache();
|
||||||
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &fpath).await;
|
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &fpath).await;
|
||||||
|
|
||||||
let obj_info = match store.put_object(&bucket, &fpath, &mut reader, &opts).await {
|
let (obj_info, backfilled_old_current_size) = match store
|
||||||
Ok(info) => info,
|
.put_object_with_old_current_size(&bucket, &fpath, &mut reader, &opts)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(result) => result,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
if extract_options.ignore_errors {
|
if extract_options.ignore_errors {
|
||||||
warn!(error = %e, "Archive object write skipped due to ignore-errors");
|
warn!(error = %e, "Archive object write skipped due to ignore-errors");
|
||||||
@@ -8201,6 +8192,21 @@ impl DefaultObjectUsecase {
|
|||||||
return Err(ApiError::from(e).into());
|
return Err(ApiError::from(e).into());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let extract_versioned = BucketVersioningSys::prefix_enabled(&bucket, &fpath).await;
|
||||||
|
match previous_current_size_from_backfill(backfilled_old_current_size) {
|
||||||
|
Some(previous_current_size) => {
|
||||||
|
if extract_versioned {
|
||||||
|
record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64)
|
||||||
|
.await;
|
||||||
|
} else {
|
||||||
|
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, extract_versioned)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &fpath).await;
|
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &fpath).await;
|
||||||
if !wrote_any_entry {
|
if !wrote_any_entry {
|
||||||
rustfs_scanner::record_dirty_usage_bucket(&bucket);
|
rustfs_scanner::record_dirty_usage_bucket(&bucket);
|
||||||
@@ -14064,7 +14070,12 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn quota_admission_allows_within_limit() {
|
fn quota_admission_allows_within_limit() {
|
||||||
map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write");
|
let result = map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write");
|
||||||
|
|
||||||
|
assert_eq!(result.current_usage, Some(1024));
|
||||||
|
assert_eq!(result.quota_limit, Some(2048));
|
||||||
|
assert_eq!(result.operation_size, 512);
|
||||||
|
assert_eq!(result.remaining, Some(512));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -14085,4 +14096,16 @@ mod tests {
|
|||||||
.expect_err("a checker fault must fail closed");
|
.expect_err("a checker fault must fail closed");
|
||||||
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
|
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn quota_admission_fails_closed_on_unknown_authoritative_usage() {
|
||||||
|
let err = map_quota_check_outcome(
|
||||||
|
"bucket",
|
||||||
|
Err(QuotaError::UsageUnavailable {
|
||||||
|
bucket: "bucket".to_string(),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.expect_err("unknown authoritative usage must not admit a quota-controlled write");
|
||||||
|
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -931,6 +931,10 @@ pub(crate) mod s3_api {
|
|||||||
|
|
||||||
pub(crate) mod admin_usecase {
|
pub(crate) mod admin_usecase {
|
||||||
pub(crate) mod contract {
|
pub(crate) mod contract {
|
||||||
|
pub(crate) mod bucket {
|
||||||
|
pub(crate) use super::super::super::storage_contracts::{BucketOperations, BucketOptions};
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) use super::super::storage_contracts::StorageAdminApi;
|
pub(crate) use super::super::storage_contracts::StorageAdminApi;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -358,6 +358,7 @@ impl From<QuotaError> for ApiError {
|
|||||||
let code = match &err {
|
let code = match &err {
|
||||||
QuotaError::QuotaExceeded { .. } => S3ErrorCode::InvalidRequest,
|
QuotaError::QuotaExceeded { .. } => S3ErrorCode::InvalidRequest,
|
||||||
QuotaError::ConfigNotFound { .. } => S3ErrorCode::NoSuchBucket,
|
QuotaError::ConfigNotFound { .. } => S3ErrorCode::NoSuchBucket,
|
||||||
|
QuotaError::UsageUnavailable { .. } => S3ErrorCode::ServiceUnavailable,
|
||||||
QuotaError::InvalidConfig { .. } => S3ErrorCode::InvalidArgument,
|
QuotaError::InvalidConfig { .. } => S3ErrorCode::InvalidArgument,
|
||||||
QuotaError::StorageError(_) => S3ErrorCode::InternalError,
|
QuotaError::StorageError(_) => S3ErrorCode::InternalError,
|
||||||
};
|
};
|
||||||
@@ -486,6 +487,16 @@ mod tests {
|
|||||||
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
|
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_unknown_authoritative_quota_usage_maps_to_retryable_error() {
|
||||||
|
let api_error = ApiError::from(QuotaError::UsageUnavailable {
|
||||||
|
bucket: "bucket".to_string(),
|
||||||
|
});
|
||||||
|
|
||||||
|
assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
|
||||||
|
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_api_error_from_storage_error_mappings() {
|
fn test_api_error_from_storage_error_mappings() {
|
||||||
let test_cases = vec![
|
let test_cases = vec![
|
||||||
|
|||||||
Reference in New Issue
Block a user