gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-12 16:16:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): require authoritative usage snapshots (#5333)

Browse Source
This commit is contained in:
Henry Guo
2026-07-28 07:29:57 +08:00
committed by GitHub
parent 5a768d3a44
commit d5df66ac4f
25 changed files with 1829 additions and 374 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/metrics/collectors/bucket.rs
+40 -22
View File
@@ -33,10 +33,10 @@ use std::borrow::Cow;
pub struct BucketStats {
/// Name of the bucket
pub name: String,
/// Total size of all objects in the bucket (bytes)
pub size_bytes: u64,
/// Number of objects in the bucket
pub objects_count: u64,
/// Total size of all objects in the bucket (bytes), when authoritative.
pub size_bytes: Option<u64>,
/// Number of objects in the bucket, when authoritative.
pub objects_count: Option<u64>,
/// Quota limit for the bucket (bytes), 0 if no quota
pub quota_bytes: u64,
}
@@ -54,15 +54,19 @@ pub fn collect_bucket_metrics(buckets: &[BucketStats]) -> Vec<PrometheusMetric>
for bucket in buckets {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone());
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, bucket.size_bytes as f64)
.with_label("bucket", bucket_label.clone()),
);
if let Some(size_bytes) = bucket.size_bytes {
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_USAGE_BYTES_MD, size_bytes as f64)
.with_label("bucket", bucket_label.clone()),
);
}
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, bucket.objects_count as f64)
.with_label("bucket", bucket_label.clone()),
);
if let Some(objects_count) = bucket.objects_count {
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_OBJECTS_TOTAL_MD, objects_count as f64)
.with_label("bucket", bucket_label.clone()),
);
}
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_QUOTA_BYTES_MD, bucket.quota_bytes as f64)
@@ -83,14 +87,14 @@ mod tests {
let buckets = vec![
BucketStats {
name: "test-bucket".to_string(),
size_bytes: 1000,
objects_count: 100,
size_bytes: Some(1000),
objects_count: Some(100),
quota_bytes: 0,
},
BucketStats {
name: "another-bucket".to_string(),
size_bytes: 2000,
objects_count: 200,
size_bytes: Some(2000),
objects_count: Some(200),
quota_bytes: 0,
},
];
@@ -114,8 +118,8 @@ mod tests {
fn test_collect_bucket_metrics_with_quotas() {
let buckets = vec![BucketStats {
name: "quota-bucket".to_string(),
size_bytes: 500,
objects_count: 10,
size_bytes: Some(500),
objects_count: Some(10),
quota_bytes: 10000,
}];
@@ -146,8 +150,8 @@ mod tests {
fn test_collect_bucket_metrics_zero_quota_always_reported() {
let buckets = vec![BucketStats {
name: "no-quota-bucket".to_string(),
size_bytes: 100,
objects_count: 5,
size_bytes: Some(100),
objects_count: Some(5),
quota_bytes: 0,
}];
@@ -165,12 +169,26 @@ mod tests {
assert!(quota_metric.is_some());
}
#[test]
fn unknown_usage_only_reports_quota() {
let metrics = collect_bucket_metrics(&[BucketStats {
name: "unknown-usage".to_string(),
size_bytes: None,
objects_count: None,
quota_bytes: 4096,
}]);
assert_eq!(metrics.len(), 1);
assert_eq!(metrics[0].name, BUCKET_QUOTA_BYTES_MD.get_full_metric_name());
assert_eq!(metrics[0].value, 4096.0);
}
#[test]
fn test_bucket_stats_default() {
let stats = BucketStats::default();
assert!(stats.name.is_empty());
assert_eq!(stats.size_bytes, 0);
assert_eq!(stats.objects_count, 0);
assert_eq!(stats.size_bytes, None);
assert_eq!(stats.objects_count, None);
assert_eq!(stats.quota_bytes, 0);
}
}
crates/obs/src/metrics/collectors/cluster.rs
+36 -11
View File
@@ -43,9 +43,9 @@ pub struct ClusterStats {
/// Number of drives with no capacity observation
pub missing_capacity_drives: u64,
/// Total number of objects in the cluster
pub objects_count: u64,
pub objects_count: Option<u64>,
/// Total number of buckets in the cluster
pub buckets_count: u64,
pub buckets_count: Option<u64>,
}
/// Collects cluster-wide metrics from the provided cluster statistics.
@@ -53,16 +53,21 @@ pub struct ClusterStats {
/// Uses the metric descriptors from `metrics_type::cluster` module.
/// Returns a vector of Prometheus metrics for cluster statistics.
pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
vec![
let mut metrics = vec![
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD, stats.raw_capacity_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD, stats.usable_capacity_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USED_BYTES_MD, stats.used_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_FREE_BYTES_MD, stats.free_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_STALE_DRIVES_MD, stats.stale_capacity_drives as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_MISSING_DRIVES_MD, stats.missing_capacity_drives as f64),
PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, stats.objects_count as f64),
PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, stats.buckets_count as f64),
]
];
if let Some(objects_count) = stats.objects_count {
metrics.push(PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, objects_count as f64));
}
if let Some(buckets_count) = stats.buckets_count {
metrics.push(PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, buckets_count as f64));
}
metrics
}
#[cfg(test)]
@@ -79,8 +84,8 @@ mod tests {
free_bytes: 1300,
stale_capacity_drives: 1,
missing_capacity_drives: 0,
objects_count: 100,
buckets_count: 5,
objects_count: Some(100),
buckets_count: Some(5),
};
let metrics = collect_cluster_metrics(&stats);
@@ -116,7 +121,7 @@ mod tests {
let metrics = collect_cluster_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 8);
assert_eq!(metrics.len(), 6);
// All values should be zero
for metric in &metrics {
@@ -134,7 +139,27 @@ mod tests {
assert_eq!(stats.free_bytes, 0);
assert_eq!(stats.stale_capacity_drives, 0);
assert_eq!(stats.missing_capacity_drives, 0);
assert_eq!(stats.objects_count, 0);
assert_eq!(stats.buckets_count, 0);
assert_eq!(stats.objects_count, None);
assert_eq!(stats.buckets_count, None);
}
#[test]
fn unknown_cluster_cardinality_is_omitted() {
let metrics = collect_cluster_metrics(&ClusterStats {
objects_count: None,
buckets_count: None,
..Default::default()
});
assert!(
metrics
.iter()
.all(|metric| metric.name != CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name())
);
assert!(
metrics
.iter()
.all(|metric| metric.name != CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name())
);
}
}
crates/obs/src/metrics/scheduler.rs
+222 -17
View File
@@ -79,10 +79,13 @@ use crate::metrics::schema::audit::{AUDIT_FAILED_MESSAGES_MD, AUDIT_TARGET_QUEUE
use crate::metrics::schema::bucket_replication::{
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, TARGET_ARN_L,
};
use crate::metrics::schema::cluster::{CLUSTER_BUCKETS_TOTAL_MD, CLUSTER_OBJECTS_TOTAL_MD};
use crate::metrics::schema::cluster_usage::{
BUCKET_LABEL as USAGE_BUCKET_LABEL, RANGE_LABEL as USAGE_RANGE_LABEL, USAGE_BUCKET_DELETE_MARKERS_COUNT_MD,
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, USAGE_BUCKET_OBJECTS_TOTAL_MD,
USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, USAGE_BUCKET_TOTAL_BYTES_MD, USAGE_BUCKET_VERSIONS_COUNT_MD,
USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, USAGE_BUCKET_TOTAL_BYTES_MD, USAGE_BUCKET_VERSIONS_COUNT_MD, USAGE_BUCKETS_COUNT_MD,
USAGE_DELETE_MARKERS_COUNT_MD, USAGE_OBJECTS_COUNT_MD, USAGE_OBJECTS_DISTRIBUTION_MD, USAGE_SINCE_LAST_UPDATE_SECONDS_MD,
USAGE_TOTAL_BYTES_MD, USAGE_VERSIONS_COUNT_MD, USAGE_VERSIONS_DISTRIBUTION_MD,
};
use crate::metrics::schema::node_bucket::{BUCKET_OBJECTS_TOTAL_MD, BUCKET_QUOTA_BYTES_MD, BUCKET_USAGE_BYTES_MD};
use crate::metrics::schema::notification_target::{
@@ -659,6 +662,26 @@ fn bucket_live_keys(stats: &[crate::metrics::collectors::BucketStats]) -> HashSe
stats.iter().map(|stat| stat.name.clone()).collect()
}
fn bucket_observation_live_keys(stats: &[crate::metrics::collectors::BucketStats]) -> HashSet<BucketKey> {
stats
.iter()
.filter(|stat| stat.size_bytes.is_some() || stat.objects_count.is_some())
.map(|stat| stat.name.clone())
.collect()
}
fn bucket_observation_retire_keys(
previous_observations: &HashSet<BucketKey>,
current_buckets: &HashSet<BucketKey>,
current_observations: &HashSet<BucketKey>,
) -> Vec<BucketKey> {
previous_observations
.difference(current_observations)
.filter(|bucket| current_buckets.contains(*bucket))
.cloned()
.collect()
}
fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8>) -> Vec<PrometheusMetric> {
if zero_tombstones.is_empty() {
return Vec::new();
@@ -677,6 +700,50 @@ fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8
zero_metrics
}
#[derive(Default)]
struct BucketSeriesState {
has_seen_snapshot: bool,
live_keys: HashSet<BucketKey>,
observation_keys: HashSet<BucketKey>,
zero_tombstones: HashMap<BucketKey, u8>,
}
struct BucketSeriesUpdate {
metrics: Vec<PrometheusMetric>,
retire_observations: Vec<BucketKey>,
retire_buckets: Vec<BucketKey>,
}
impl BucketSeriesState {
fn observe(
&mut self,
stats: Option<&[crate::metrics::collectors::BucketStats]>,
tombstone_cycles: u8,
) -> Option<BucketSeriesUpdate> {
let stats = stats?;
let current_bucket_keys = bucket_live_keys(stats);
let current_observation_keys = bucket_observation_live_keys(stats);
let retire_observations =
bucket_observation_retire_keys(&self.observation_keys, &current_bucket_keys, &current_observation_keys);
self.observation_keys = current_observation_keys;
update_series_zero_tombstones(
&mut self.has_seen_snapshot,
&mut self.live_keys,
&mut self.zero_tombstones,
current_bucket_keys,
tombstone_cycles,
);
let mut metrics = collect_bucket_metrics(stats);
metrics.extend(collect_bucket_zero_tombstone_metrics(&self.zero_tombstones));
let retire_buckets = expire_series_zero_tombstones(&mut self.zero_tombstones);
Some(BucketSeriesUpdate {
metrics,
retire_observations,
retire_buckets,
})
}
}
fn retire_bucket_metric_series(bucket: &str) -> usize {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.to_string());
let labels = [("bucket", bucket_label.clone())];
@@ -685,6 +752,32 @@ fn retire_bucket_metric_series(bucket: &str) -> usize {
+ retire_metric_series(&BUCKET_QUOTA_BYTES_MD.get_full_metric_name(), &labels)
}
fn retire_bucket_observation_metric_series(bucket: &str) -> usize {
let labels = [("bucket", Cow::Owned(bucket.to_string()))];
retire_metric_series(&BUCKET_USAGE_BYTES_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels)
}
fn retire_cluster_usage_metric_series() -> usize {
let labels: [(&'static str, Cow<'static, str>); 0] = [];
[
USAGE_SINCE_LAST_UPDATE_SECONDS_MD.get_full_metric_name(),
USAGE_TOTAL_BYTES_MD.get_full_metric_name(),
USAGE_OBJECTS_COUNT_MD.get_full_metric_name(),
USAGE_VERSIONS_COUNT_MD.get_full_metric_name(),
USAGE_DELETE_MARKERS_COUNT_MD.get_full_metric_name(),
USAGE_BUCKETS_COUNT_MD.get_full_metric_name(),
]
.iter()
.map(|name| retire_metric_series(name, &labels))
.sum()
}
fn retire_cluster_usage_distribution_series(metric_name: String, range: &str) -> usize {
let labels = [(USAGE_RANGE_LABEL, Cow::Owned(range.to_string()))];
retire_metric_series(&metric_name, &labels)
}
fn bucket_usage_live_keys(stats: &[crate::metrics::collectors::BucketUsageStats]) -> HashSet<BucketKey> {
stats.iter().map(|stat| stat.bucket.clone()).collect()
}
@@ -973,11 +1066,23 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let token_clone = token.clone();
tokio::spawn(async move {
let mut interval = metrics_interval(cluster_interval, Duration::ZERO);
let mut objects_count_was_authoritative = false;
let mut buckets_count_was_authoritative = false;
loop {
tokio::select! {
_ = interval.tick() => {
run_metrics_collector_tick(health, MetricsCollectorTaskId::ClusterStats, "cluster_stats", async {
let (stats, cluster_health) = collect_cluster_and_health_stats().await;
if objects_count_was_authoritative && stats.objects_count.is_none() {
let labels: [(&'static str, Cow<'static, str>); 0] = [];
let _ = retire_metric_series(&CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels);
}
if buckets_count_was_authoritative && stats.buckets_count.is_none() {
let labels: [(&'static str, Cow<'static, str>); 0] = [];
let _ = retire_metric_series(&CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name(), &labels);
}
objects_count_was_authoritative = stats.objects_count.is_some();
buckets_count_was_authoritative = stats.buckets_count.is_some();
let mut metrics = collect_cluster_metrics(&stats);
metrics.extend(collect_cluster_health_metrics(&cluster_health));
report_metrics(&metrics);
@@ -1004,6 +1109,8 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let mut bucket_usage_object_size_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
let mut prev_bucket_usage_version_keys: HashSet<BucketRangeKey> = HashSet::new();
let mut bucket_usage_version_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
let mut prev_cluster_usage_object_size_keys: HashSet<String> = HashSet::new();
let mut prev_cluster_usage_version_keys: HashSet<String> = HashSet::new();
loop {
tokio::select! {
_ = interval.tick() => {
@@ -1028,6 +1135,30 @@ pub fn init_metrics_runtime(token: CancellationToken) {
}
if let Some((cluster_usage, bucket_usage)) = collect_cluster_usage_metric_stats().await {
let current_cluster_usage_object_size_keys = cluster_usage
.object_size_distribution
.iter()
.map(|(range, _)| range.clone())
.collect::<HashSet<_>>();
for range in prev_cluster_usage_object_size_keys.difference(&current_cluster_usage_object_size_keys) {
let _ = retire_cluster_usage_distribution_series(
USAGE_OBJECTS_DISTRIBUTION_MD.get_full_metric_name(),
range,
);
}
prev_cluster_usage_object_size_keys = current_cluster_usage_object_size_keys;
let current_cluster_usage_version_keys = cluster_usage
.versions_distribution
.iter()
.map(|(range, _)| range.clone())
.collect::<HashSet<_>>();
for range in prev_cluster_usage_version_keys.difference(&current_cluster_usage_version_keys) {
let _ = retire_cluster_usage_distribution_series(
USAGE_VERSIONS_DISTRIBUTION_MD.get_full_metric_name(),
range,
);
}
prev_cluster_usage_version_keys = current_cluster_usage_version_keys;
metrics.extend(collect_cluster_usage_metrics(&cluster_usage));
update_series_zero_tombstones(
&mut has_seen_bucket_usage_snapshot,
@@ -1073,6 +1204,41 @@ pub fn init_metrics_runtime(token: CancellationToken) {
&range,
);
}
} else if has_seen_bucket_usage_snapshot {
let _ = retire_cluster_usage_metric_series();
for range in prev_cluster_usage_object_size_keys.drain() {
let _ = retire_cluster_usage_distribution_series(
USAGE_OBJECTS_DISTRIBUTION_MD.get_full_metric_name(),
&range,
);
}
for range in prev_cluster_usage_version_keys.drain() {
let _ = retire_cluster_usage_distribution_series(
USAGE_VERSIONS_DISTRIBUTION_MD.get_full_metric_name(),
&range,
);
}
for bucket in prev_bucket_usage_keys.drain() {
let _ = retire_bucket_usage_metric_series(&bucket);
}
for (bucket, range) in prev_bucket_usage_object_size_keys.drain() {
let _ = retire_bucket_usage_distribution_series(
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD.get_full_metric_name(),
&bucket,
&range,
);
}
for (bucket, range) in prev_bucket_usage_version_keys.drain() {
let _ = retire_bucket_usage_distribution_series(
USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD.get_full_metric_name(),
&bucket,
&range,
);
}
bucket_usage_zero_tombstones.clear();
bucket_usage_object_size_zero_tombstones.clear();
bucket_usage_version_zero_tombstones.clear();
has_seen_bucket_usage_snapshot = false;
}
if !metrics.is_empty() {
@@ -1094,25 +1260,20 @@ pub fn init_metrics_runtime(token: CancellationToken) {
tokio::spawn(async move {
let mut interval = metrics_interval(bucket_interval, Duration::ZERO);
let tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
let mut has_seen_bucket_snapshot = false;
let mut prev_bucket_keys: HashSet<BucketKey> = HashSet::new();
let mut bucket_zero_tombstones: HashMap<BucketKey, u8> = HashMap::new();
let mut series_state = BucketSeriesState::default();
loop {
tokio::select! {
_ = interval.tick() => {
run_metrics_collector_tick(health, MetricsCollectorTaskId::BucketStats, "bucket_stats", async {
let stats = collect_bucket_stats().await;
update_series_zero_tombstones(
&mut has_seen_bucket_snapshot,
&mut prev_bucket_keys,
&mut bucket_zero_tombstones,
bucket_live_keys(&stats),
tombstone_cycles,
);
let mut metrics = collect_bucket_metrics(&stats);
metrics.extend(collect_bucket_zero_tombstone_metrics(&bucket_zero_tombstones));
report_metrics(&metrics);
for bucket in expire_series_zero_tombstones(&mut bucket_zero_tombstones) {
let Some(update) = series_state.observe(stats.as_deref(), tombstone_cycles) else {
return;
};
for bucket in update.retire_observations {
let _ = retire_bucket_observation_metric_series(&bucket);
}
report_metrics(&update.metrics);
for bucket in update.retire_buckets {
let _ = retire_bucket_metric_series(&bucket);
}
}).await;
@@ -1850,8 +2011,8 @@ mod tests {
let mut zero_tombstones = HashMap::new();
let live_stats = vec![crate::metrics::collectors::BucketStats {
name: "tmp".to_string(),
size_bytes: 1024,
objects_count: 8,
size_bytes: Some(1024),
objects_count: Some(8),
quota_bytes: 2048,
}];
@@ -1885,6 +2046,50 @@ mod tests {
assert!(zero_tombstones.is_empty());
}
#[test]
fn bucket_observation_retirement_distinguishes_unknown_usage_from_deletion() {
let previous = HashSet::from(["bucket".to_string()]);
let unknown_stats = vec![crate::metrics::collectors::BucketStats {
name: "bucket".to_string(),
size_bytes: None,
objects_count: None,
quota_bytes: 1024,
}];
let current_buckets = bucket_live_keys(&unknown_stats);
let current_observations = bucket_observation_live_keys(&unknown_stats);
assert_eq!(
bucket_observation_retire_keys(&previous, &current_buckets, &current_observations),
vec!["bucket".to_string()],
"an existing bucket with unknown usage must retire its previous usage observations"
);
assert!(
bucket_observation_retire_keys(&previous, &HashSet::new(), &HashSet::new()).is_empty(),
"a deleted bucket remains governed by the zero-tombstone lifecycle"
);
}
#[test]
fn unavailable_bucket_snapshot_preserves_metric_series_state() {
let mut state = BucketSeriesState::default();
let initial = [crate::metrics::collectors::BucketStats {
name: "bucket".to_string(),
size_bytes: Some(512),
objects_count: Some(2),
quota_bytes: 1024,
}];
assert!(state.observe(Some(&initial), 2).is_some());
let live_keys = state.live_keys.clone();
let observation_keys = state.observation_keys.clone();
let zero_tombstones = state.zero_tombstones.clone();
assert!(state.observe(None, 2).is_none());
assert_eq!(state.live_keys, live_keys);
assert_eq!(state.observation_keys, observation_keys);
assert_eq!(state.zero_tombstones, zero_tombstones);
}
#[test]
fn parse_system_metrics_interval_rounds_legacy_millis_up_to_one_second() {
temp_env::with_vars(
crates/obs/src/metrics/stats_collector.rs
+124 -29
View File
@@ -41,7 +41,11 @@ use rustfs_io_metrics::{
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_resource_and_system,
snapshot_process_resource_and_system_with,
};
use std::{collections::HashMap, sync::Arc, time::SystemTime};
use std::{
collections::{HashMap, HashSet},
sync::Arc,
time::SystemTime,
};
use sysinfo::{Networks, System};
use tracing::{instrument, warn};
@@ -52,8 +56,10 @@ const EVENT_METRICS_COLLECTOR_STATE: &str = "metrics_collector_state";
type ObsStorageInfo = <ObsStore as StorageAdminApi>::StorageInfo;
type ObsBackendInfo = <ObsStore as StorageAdminApi>::BackendInfo;
#[derive(Default)]
struct ObsDataUsageInfo {
last_update: Option<SystemTime>,
usage_snapshot_complete: bool,
buckets_count: u64,
objects_total_count: u64,
versions_total_count: u64,
@@ -62,6 +68,7 @@ struct ObsDataUsageInfo {
buckets_usage: HashMap<String, ObsBucketUsageInfo>,
}
#[derive(Default)]
struct ObsBucketUsageInfo {
size: u64,
objects_count: u64,
@@ -85,9 +92,11 @@ fn usize_to_u64_saturating(value: usize) -> u64 {
async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreResult<ObsDataUsageInfo> {
let data_usage = obs_load_data_usage_from_backend(store).await?;
let usage_snapshot_complete = data_usage.is_complete_bucket_usage_snapshot();
Ok(ObsDataUsageInfo {
last_update: data_usage.last_update,
usage_snapshot_complete,
buckets_count: data_usage.buckets_count,
objects_total_count: data_usage.objects_total_count,
versions_total_count: data_usage.versions_total_count,
@@ -113,6 +122,21 @@ async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreRes
})
}
fn bucket_usage_metric_values(data_usage: Option<&ObsDataUsageInfo>, bucket: &str) -> (Option<u64>, Option<u64>) {
data_usage
.filter(|usage| usage.usage_snapshot_complete)
.and_then(|usage| usage.buckets_usage.get(bucket))
.map(|usage| (Some(usage.size), Some(usage.objects_count)))
.unwrap_or((None, None))
}
fn data_usage_snapshot_covers_bucket_namespace(data_usage: &ObsDataUsageInfo, buckets: &HashSet<String>) -> bool {
data_usage.usage_snapshot_complete
&& u64::try_from(buckets.len()).ok() == Some(data_usage.buckets_count)
&& buckets.len() == data_usage.buckets_usage.len()
&& buckets.iter().all(|bucket| data_usage.buckets_usage.contains_key(bucket))
}
fn resolve_obs_object_store_handle() -> Option<Arc<ObsStore>> {
obs_resolve_object_store_handle()
}
@@ -365,25 +389,16 @@ pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthS
})
.count() as u64;
// Get bucket and object counts from data usage info.
let (buckets_count, objects_count) = match load_obs_data_usage_from_backend(store.clone()).await {
Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
Err(e) => {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "data_usage_load_failed", error = %e, "metrics collector state changed");
// Fall back to bucket list for buckets_count, objects_count stays 0.
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
.unwrap_or_else(|err| {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "bucket_list_failed", error = %err, "metrics collector state changed");
Vec::new()
});
(buckets.len() as u64, 0)
let data_usage = match load_obs_data_usage_from_backend(store).await {
Ok(data_usage) if data_usage.usage_snapshot_complete => Some(data_usage),
Ok(_) => None,
Err(error) => {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "cluster_stats", result = "data_usage_load_failed", error = %error, "metrics collector state changed");
None
}
};
let buckets_count = data_usage.as_ref().map(|usage| usage.buckets_count);
let objects_count = data_usage.as_ref().map(|usage| usage.objects_total_count);
let mut online = 0u64;
let mut offline = 0u64;
@@ -428,10 +443,12 @@ pub async fn collect_cluster_health_stats() -> ClusterHealthStats {
}
/// Collect bucket statistics from the storage layer.
pub async fn collect_bucket_stats() -> Vec<BucketStats> {
let Some(store) = resolve_obs_object_store_handle() else {
return Vec::new();
};
///
/// `None` means the bucket namespace could not be observed. Callers must keep
/// their prior metric-series state instead of treating that failure as an
/// authoritative empty namespace.
pub async fn collect_bucket_stats() -> Option<Vec<BucketStats>> {
let store = resolve_obs_object_store_handle()?;
// Load data usage info from backend to get bucket sizes and object counts
let data_usage = match load_obs_data_usage_from_backend(store.clone()).await {
@@ -453,7 +470,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
Ok(buckets) => buckets,
Err(e) => {
warn!(event = EVENT_METRICS_COLLECTOR_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_COLLECTOR, collector = "bucket_stats", result = "bucket_list_failed", error = %e, "metrics collector state changed");
return Vec::new();
return None;
}
};
@@ -465,11 +482,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
}
// Get size and objects_count from data usage info
let (size_bytes, objects_count) = data_usage
.as_ref()
.and_then(|du| du.buckets_usage.get(&bucket.name))
.map(|bui| (bui.size, bui.objects_count))
.unwrap_or((0, 0));
let (size_bytes, objects_count) = bucket_usage_metric_values(data_usage.as_ref(), &bucket.name);
// Get quota from bucket metadata
let quota_bytes = obs_bucket_quota_limit_bytes(&bucket.name).await;
@@ -482,7 +495,7 @@ pub async fn collect_bucket_stats() -> Vec<BucketStats> {
});
}
stats
Some(stats)
}
/// Collect bucket replication bandwidth stats from the global monitor.
@@ -935,6 +948,29 @@ pub async fn collect_iam_stats() -> Option<IamStats> {
pub async fn collect_cluster_usage_metric_stats() -> Option<(ClusterUsageStats, Vec<BucketUsageStats>)> {
let store = resolve_obs_object_store_handle()?;
let data_usage = load_obs_data_usage_from_backend(store.clone()).await.ok()?;
let bucket_namespace = store
.list_bucket(&BucketOptions {
cached: true,
no_metadata: true,
..Default::default()
})
.await
.ok()?
.into_iter()
.filter(|bucket| !bucket.name.starts_with('.'))
.map(|bucket| bucket.name)
.collect::<HashSet<_>>();
collect_cluster_usage_metric_stats_from_data_usage(data_usage, &bucket_namespace).await
}
async fn collect_cluster_usage_metric_stats_from_data_usage(
data_usage: ObsDataUsageInfo,
bucket_namespace: &HashSet<String>,
) -> Option<(ClusterUsageStats, Vec<BucketUsageStats>)> {
if !data_usage_snapshot_covers_bucket_namespace(&data_usage, bucket_namespace) {
return None;
}
let mut buckets = Vec::with_capacity(data_usage.buckets_usage.len());
for (bucket_name, usage) in &data_usage.buckets_usage {
@@ -1187,6 +1223,65 @@ mod tests {
info
}
#[test]
fn bucket_usage_metrics_distinguish_unknown_from_confirmed_zero() {
assert_eq!(bucket_usage_metric_values(None, "bucket"), (None, None));
let mut data_usage = ObsDataUsageInfo {
usage_snapshot_complete: true,
..Default::default()
};
data_usage
.buckets_usage
.insert("bucket".to_string(), ObsBucketUsageInfo::default());
assert_eq!(bucket_usage_metric_values(Some(&data_usage), "bucket"), (Some(0), Some(0)));
assert_eq!(bucket_usage_metric_values(Some(&data_usage), "missing"), (None, None));
}
#[tokio::test]
async fn cluster_usage_metrics_skip_incomplete_snapshot() {
assert!(
collect_cluster_usage_metric_stats_from_data_usage(ObsDataUsageInfo::default(), &HashSet::new())
.await
.is_none()
);
}
#[tokio::test]
async fn cluster_usage_metrics_publish_complete_empty_snapshot() {
let (cluster, buckets) = collect_cluster_usage_metric_stats_from_data_usage(
ObsDataUsageInfo {
usage_snapshot_complete: true,
..Default::default()
},
&HashSet::new(),
)
.await
.expect("complete empty usage should remain publishable");
assert_eq!(cluster.buckets_count, 0);
assert_eq!(cluster.objects_count, 0);
assert_eq!(cluster.total_bytes, 0);
assert!(buckets.is_empty());
}
#[tokio::test]
async fn cluster_usage_metrics_skip_snapshot_for_a_different_bucket_namespace() {
let data_usage = ObsDataUsageInfo {
usage_snapshot_complete: true,
buckets_count: 1,
buckets_usage: HashMap::from([("stale-bucket".to_string(), ObsBucketUsageInfo::default())]),
..Default::default()
};
assert!(
collect_cluster_usage_metric_stats_from_data_usage(data_usage, &HashSet::from(["live-bucket".to_string()]))
.await
.is_none()
);
}
#[test]
fn cluster_config_stats_accept_homogeneous_backend_parities() {
let stats = cluster_config_stats_from_backend_parities(Some(1), Some(2))