mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-07 05:43:14 +00:00
fix(admin): stabilize cluster capacity usage reporting (#5053)
* fix(admin): stabilize cluster capacity usage reporting * test(admin): strengthen capacity usage regressions
This commit is contained in:
cxymds
@@ -87,50 +87,57 @@ mod capacity_dedup_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_four_disk_erasure_coding() {
|
||||
// 4-disk erasure coding: 2 data disks + 2 parity disks
|
||||
fn test_four_node_ec_2_2_reports_stable_usable_capacity() {
|
||||
const TIB: u64 = 1 << 40;
|
||||
const DISK_TOTAL: u64 = 2 * TIB;
|
||||
const DISK_FREE: u64 = TIB / 5;
|
||||
|
||||
let disks = vec![
|
||||
rustfs_madmin::Disk {
|
||||
endpoint: "node1".to_string(),
|
||||
drive_path: "/mnt/disk1".to_string(),
|
||||
drive_path: "/media/rustfs-01".to_string(),
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
disk_index: 0,
|
||||
total_space: 1_000_000_000_000, // 1TB
|
||||
available_space: 250_000_000_000,
|
||||
total_space: DISK_TOTAL,
|
||||
available_space: DISK_FREE,
|
||||
used_space: DISK_TOTAL - DISK_FREE,
|
||||
state: "ok".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
rustfs_madmin::Disk {
|
||||
endpoint: "node1".to_string(),
|
||||
drive_path: "/mnt/disk2".to_string(),
|
||||
endpoint: "node2".to_string(),
|
||||
drive_path: "/media/rustfs-01".to_string(),
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
disk_index: 1,
|
||||
total_space: 1_000_000_000_000,
|
||||
available_space: 250_000_000_000,
|
||||
total_space: DISK_TOTAL,
|
||||
available_space: DISK_FREE,
|
||||
used_space: DISK_TOTAL - DISK_FREE,
|
||||
state: "ok".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
rustfs_madmin::Disk {
|
||||
endpoint: "node1".to_string(),
|
||||
drive_path: "/mnt/disk3".to_string(),
|
||||
endpoint: "node3".to_string(),
|
||||
drive_path: "/media/rustfs-01".to_string(),
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
disk_index: 2,
|
||||
total_space: 1_000_000_000_000,
|
||||
available_space: 250_000_000_000,
|
||||
total_space: DISK_TOTAL,
|
||||
available_space: DISK_FREE,
|
||||
used_space: DISK_TOTAL - DISK_FREE,
|
||||
state: "ok".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
rustfs_madmin::Disk {
|
||||
endpoint: "node1".to_string(),
|
||||
drive_path: "/mnt/disk4".to_string(),
|
||||
endpoint: "node4".to_string(),
|
||||
drive_path: "/media/rustfs-01".to_string(),
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
disk_index: 3,
|
||||
total_space: 1_000_000_000_000,
|
||||
available_space: 250_000_000_000,
|
||||
total_space: DISK_TOTAL,
|
||||
available_space: DISK_FREE,
|
||||
used_space: DISK_TOTAL - DISK_FREE,
|
||||
state: "ok".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -146,9 +153,17 @@ mod capacity_dedup_tests {
|
||||
};
|
||||
|
||||
let total = get_total_usable_capacity(&disks, &info);
|
||||
let free = get_total_usable_capacity_free(&disks, &info);
|
||||
let used = total.saturating_sub(free);
|
||||
let expected_total = usize::try_from(4 * TIB).expect("4 TiB must fit the supported platform's usize");
|
||||
let expected_free = usize::try_from(2 * DISK_FREE).expect("usable free capacity must fit usize");
|
||||
let single_node_used = usize::try_from(DISK_TOTAL - DISK_FREE).expect("single-node used capacity must fit usize");
|
||||
|
||||
// Only count data disks (disk_index < 2)
|
||||
assert_eq!(total, 2_000_000_000_000, "Should count only data disks (2 × 1TB)");
|
||||
assert_eq!(total, expected_total, "usable total must count the two data disks");
|
||||
assert_eq!(free, expected_free, "usable free must count the two data disks");
|
||||
assert_eq!(used, 2 * single_node_used, "usable used must be approximately 3.6 TiB");
|
||||
assert_ne!(used, single_node_used, "used must not collapse to one node's approximately 1.8 TiB");
|
||||
assert_ne!(used, 4 * single_node_used, "used must not report the approximately 7.2 TiB raw aggregate");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -319,8 +319,8 @@ impl NotificationSys {
|
||||
if let Some(client) = client {
|
||||
let host = client.host.to_string();
|
||||
match timeout(peer_timeout, client.local_storage_info()).await {
|
||||
Ok(Ok(info)) => {
|
||||
update_storage_info_cache(cache, &host, &info);
|
||||
Ok(Ok(mut info)) => {
|
||||
normalize_and_cache_peer_storage_info(cache, &host, &mut info);
|
||||
Some(info)
|
||||
}
|
||||
Ok(Err(err)) => {
|
||||
@@ -1156,7 +1156,13 @@ fn handle_peer_failure(
|
||||
None
|
||||
}
|
||||
|
||||
fn update_storage_info_cache(cache: Option<&Mutex<PeerAdminCache>>, host: &str, info: &StorageInfo) {
|
||||
fn normalize_and_cache_peer_storage_info(cache: Option<&Mutex<PeerAdminCache>>, host: &str, info: &mut StorageInfo) {
|
||||
// `Disk::local` is relative to this aggregator, not to the peer that
|
||||
// produced the response.
|
||||
for disk in &mut info.disks {
|
||||
disk.local = false;
|
||||
}
|
||||
|
||||
let Some(cache) = cache else {
|
||||
return;
|
||||
};
|
||||
@@ -1760,6 +1766,53 @@ mod tests {
|
||||
assert_eq!(cache.lock().unwrap().storage_failures, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_and_cache_peer_storage_info_marks_disks_remote() {
|
||||
let cache = Mutex::new(PeerAdminCache::new());
|
||||
let mut info = StorageInfo {
|
||||
disks: vec![
|
||||
rustfs_madmin::Disk {
|
||||
endpoint: "http://node2:9000/media/rustfs-01".to_string(),
|
||||
drive_path: "/media/rustfs-01".to_string(),
|
||||
local: true,
|
||||
..Default::default()
|
||||
},
|
||||
rustfs_madmin::Disk {
|
||||
endpoint: "http://node3:9000/media/rustfs-01".to_string(),
|
||||
drive_path: "/media/rustfs-01".to_string(),
|
||||
local: true,
|
||||
..Default::default()
|
||||
},
|
||||
rustfs_madmin::Disk {
|
||||
endpoint: "http://node4:9000/media/rustfs-01".to_string(),
|
||||
drive_path: "/media/rustfs-01".to_string(),
|
||||
local: true,
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
normalize_and_cache_peer_storage_info(Some(&cache), "peer-1", &mut info);
|
||||
|
||||
assert!(info.disks.iter().all(|disk| !disk.local));
|
||||
let cached = cache.lock().expect("peer cache must remain available");
|
||||
assert!(
|
||||
cached
|
||||
.last_storage_info
|
||||
.as_ref()
|
||||
.expect("successful peer response must be cached")
|
||||
.disks
|
||||
.iter()
|
||||
.all(|disk| !disk.local)
|
||||
);
|
||||
drop(cached);
|
||||
|
||||
let degraded = handle_peer_failure(Some(&cache), "peer-1", &EndpointServerPools::default())
|
||||
.expect("first peer failure must return the cached snapshot");
|
||||
assert!(degraded.disks.iter().all(|disk| !disk.local));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handle_peer_failure_returns_offline_after_threshold_exceeded() {
|
||||
let cached_info = StorageInfo {
|
||||
@@ -2059,10 +2112,10 @@ mod tests {
|
||||
panic!("poison server cache mutex");
|
||||
});
|
||||
|
||||
update_storage_info_cache(
|
||||
normalize_and_cache_peer_storage_info(
|
||||
Some(&storage_cache),
|
||||
"peer-1",
|
||||
&StorageInfo {
|
||||
&mut StorageInfo {
|
||||
disks: vec![rustfs_madmin::Disk {
|
||||
endpoint: "disk-0".to_string(),
|
||||
state: "ok".to_string(),
|
||||
|
||||
Reference in New Issue
Block a user