gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 03:46:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): harden runtime read-path quorum handling (#2872)

Browse Source
This commit is contained in:
houseme
2026-05-08 17:56:39 +08:00
committed by GitHub
parent 03045ff2e6
commit c90bfe2b23
21 changed files with 907 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/metrics/collectors/cluster.rs
+12 -2
View File
@@ -38,6 +38,10 @@ pub struct ClusterStats {
pub used_bytes: u64,
/// Available free storage in bytes
pub free_bytes: u64,
/// Number of drives backed by stale capacity snapshots
pub stale_capacity_drives: u64,
/// Number of drives with no capacity observation
pub missing_capacity_drives: u64,
/// Total number of objects in the cluster
pub objects_count: u64,
/// Total number of buckets in the cluster
@@ -54,6 +58,8 @@ pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD, stats.usable_capacity_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_USED_BYTES_MD, stats.used_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_FREE_BYTES_MD, stats.free_bytes as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_STALE_DRIVES_MD, stats.stale_capacity_drives as f64),
PrometheusMetric::from_descriptor(&CLUSTER_CAPACITY_MISSING_DRIVES_MD, stats.missing_capacity_drives as f64),
PrometheusMetric::from_descriptor(&CLUSTER_OBJECTS_TOTAL_MD, stats.objects_count as f64),
PrometheusMetric::from_descriptor(&CLUSTER_BUCKETS_TOTAL_MD, stats.buckets_count as f64),
]
@@ -71,6 +77,8 @@ mod tests {
usable_capacity_bytes: 2500,
used_bytes: 1200,
free_bytes: 1300,
stale_capacity_drives: 1,
missing_capacity_drives: 0,
objects_count: 100,
buckets_count: 5,
};
@@ -78,7 +86,7 @@ mod tests {
let metrics = collect_cluster_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 6);
assert_eq!(metrics.len(), 8);
// Verify raw capacity
let raw_capacity_name = CLUSTER_CAPACITY_RAW_TOTAL_BYTES_MD.get_full_metric_name();
@@ -108,7 +116,7 @@ mod tests {
let metrics = collect_cluster_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 6);
assert_eq!(metrics.len(), 8);
// All values should be zero
for metric in &metrics {
@@ -124,6 +132,8 @@ mod tests {
assert_eq!(stats.usable_capacity_bytes, 0);
assert_eq!(stats.used_bytes, 0);
assert_eq!(stats.free_bytes, 0);
assert_eq!(stats.stale_capacity_drives, 0);
assert_eq!(stats.missing_capacity_drives, 0);
assert_eq!(stats.objects_count, 0);
assert_eq!(stats.buckets_count, 0);
}
crates/obs/src/metrics/collectors/system_drive.rs
+26 -2
View File
@@ -40,6 +40,10 @@ pub struct DriveDetailedStats {
pub used_bytes: u64,
/// Free capacity in bytes
pub free_bytes: u64,
/// Capacity observation state: live, stale, or missing
pub capacity_observation_state: &'static str,
/// Age in seconds of the current capacity observation
pub capacity_observation_age_seconds: u64,
/// Used inodes
pub used_inodes: u64,
/// Free inodes
@@ -103,7 +107,7 @@ pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<Prome
);
}
let mut metrics = Vec::with_capacity(stats.len() * 19);
let mut metrics = Vec::with_capacity(stats.len() * 23);
for stat in stats {
let server_label = stat.server.as_str();
@@ -112,6 +116,24 @@ pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<Prome
push_drive_metric(&mut metrics, &DRIVE_TOTAL_BYTES_MD, stat.total_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_USED_BYTES_MD, stat.used_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_FREE_BYTES_MD, stat.free_bytes as f64, server_label, drive_label);
push_drive_metric(
&mut metrics,
&DRIVE_CAPACITY_OBSERVATION_AGE_SECONDS_MD,
stat.capacity_observation_age_seconds as f64,
server_label,
drive_label,
);
for state in ["live", "stale", "missing"] {
metrics.push(
PrometheusMetric::from_descriptor(
&DRIVE_CAPACITY_OBSERVATION_STATE_MD,
if state == stat.capacity_observation_state { 1.0 } else { 0.0 },
)
.with_label_owned(DRIVE_LABEL, drive_label.to_string())
.with_label_owned(SERVER_LABEL, server_label.to_string())
.with_label_owned("state", state.to_string()),
);
}
push_drive_metric(&mut metrics, &DRIVE_USED_INODES_MD, stat.used_inodes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_FREE_INODES_MD, stat.free_inodes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_TOTAL_INODES_MD, stat.total_inodes as f64, server_label, drive_label);
@@ -219,6 +241,8 @@ mod tests {
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
used_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
free_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
capacity_observation_state: "live",
capacity_observation_age_seconds: 0,
used_inodes: 100000,
free_inodes: 900000,
total_inodes: 1000000,
@@ -240,7 +264,7 @@ mod tests {
let metrics = collect_drive_detailed_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 19);
assert_eq!(metrics.len(), 23);
// Verify total bytes metric
let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name();
crates/obs/src/metrics/schema/cluster.rs
+20
View File
@@ -57,6 +57,26 @@ pub static CLUSTER_CAPACITY_FREE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock
)
});
/// Number of drives whose capacity is served from a stale snapshot.
pub static CLUSTER_CAPACITY_STALE_DRIVES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_stale_drives".to_string()),
"Count of drives whose capacity metrics are served from stale snapshots",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Number of drives with no capacity observation available.
pub static CLUSTER_CAPACITY_MISSING_DRIVES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_missing_drives".to_string()),
"Count of drives with missing capacity observations",
&[],
subsystems::CLUSTER_BASE_PATH,
)
});
/// Total number of objects in the cluster
pub static CLUSTER_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
crates/obs/src/metrics/schema/system_drive.rs
+18
View File
@@ -60,6 +60,24 @@ pub static DRIVE_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
)
});
pub static DRIVE_CAPACITY_OBSERVATION_STATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_observation_state".to_string()),
"Drive capacity observation state (1 for the active state label, 0 otherwise). States: live, stale, missing",
&[&ALL_DRIVE_LABELS[..], &["state"]].concat(),
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_CAPACITY_OBSERVATION_AGE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("capacity_observation_age_seconds".to_string()),
"Age in seconds of the drive capacity observation currently exported",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_USED_INODES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::DriveUsedInodes,
crates/obs/src/metrics/stats_collector.rs
+36
View File
@@ -47,6 +47,9 @@ const DRIVE_STATE_OK: &str = "ok";
const DRIVE_STATE_ONLINE: &str = "online";
const DRIVE_STATE_UNFORMATTED: &str = "unformatted";
const DRIVE_RUNTIME_STATE_RETURNING: &str = "returning";
const CAPACITY_OBSERVATION_LIVE: &str = "live";
const CAPACITY_OBSERVATION_STALE: &str = "stale";
const CAPACITY_OBSERVATION_MISSING: &str = "missing";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ErasureSetQuorumShape {
@@ -71,6 +74,15 @@ fn disk_is_online_for_metrics(state: &str, runtime_state: Option<&str>) -> bool
state_is_acceptable
}
fn disk_capacity_observation_state(source: Option<&str>, age_seconds: Option<u64>) -> (&'static str, u64) {
let age_seconds = age_seconds.unwrap_or(0);
match source {
Some("live_probe") => (CAPACITY_OBSERVATION_LIVE, age_seconds),
Some("snapshot") => (CAPACITY_OBSERVATION_STALE, age_seconds),
_ => (CAPACITY_OBSERVATION_MISSING, age_seconds),
}
}
fn derive_erasure_set_quorum_shape(set_drive_count: usize, parity: usize) -> ErasureSetQuorumShape {
let data_shards = set_drive_count.saturating_sub(parity);
let read_quorum = data_shards.max(1);
@@ -114,6 +126,22 @@ pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthS
let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
let stale_capacity_drives = storage_info
.disks
.iter()
.filter(|disk| {
disk_capacity_observation_state(disk.capacity_observation_source.as_deref(), disk.capacity_observation_age_seconds).0
== CAPACITY_OBSERVATION_STALE
})
.count() as u64;
let missing_capacity_drives = storage_info
.disks
.iter()
.filter(|disk| {
disk_capacity_observation_state(disk.capacity_observation_source.as_deref(), disk.capacity_observation_age_seconds).0
== CAPACITY_OBSERVATION_MISSING
})
.count() as u64;
// Get bucket and object counts from data usage info.
let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
@@ -151,6 +179,8 @@ pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthS
usable_capacity_bytes: usable_capacity,
used_bytes: used,
free_bytes: free,
stale_capacity_drives,
missing_capacity_drives,
objects_count,
buckets_count,
},
@@ -483,6 +513,10 @@ pub async fn collect_disk_and_system_drive_stats() -> (Vec<DiskStats>, Vec<Drive
.iter()
.map(|disk| {
let is_online = disk_is_online_for_metrics(disk.state.as_str(), disk.runtime_state.as_deref());
let (capacity_observation_state, capacity_observation_age_seconds) = disk_capacity_observation_state(
disk.capacity_observation_source.as_deref(),
disk.capacity_observation_age_seconds,
);
if is_online {
online_count += 1;
} else {
@@ -495,6 +529,8 @@ pub async fn collect_disk_and_system_drive_stats() -> (Vec<DiskStats>, Vec<Drive
total_bytes: disk.total_space,
used_bytes: disk.used_space,
free_bytes: disk.available_space,
capacity_observation_state,
capacity_observation_age_seconds,
used_inodes: 0,
free_inodes: 0,
total_inodes: 0,