mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
refactor: move ecstore pool-space builder (#3651)
This commit is contained in:
@@ -14,6 +14,11 @@
|
||||
|
||||
use std::slice::Iter;
|
||||
|
||||
use crate::bucket::utils::is_meta_bucketname;
|
||||
use crate::disk::DiskInfo;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::global::{DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, is_erasure_sd};
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct PoolAvailableSpace {
|
||||
pub index: usize,
|
||||
@@ -68,10 +73,118 @@ impl ServerPoolsAvailableSpace {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn has_space_for(dis: &[Option<DiskInfo>], size: i64) -> Result<bool> {
|
||||
let size = { if size < 0 { DISK_ASSUME_UNKNOWN_SIZE } else { size as u64 * 2 } };
|
||||
|
||||
let mut available = 0;
|
||||
let mut total = 0;
|
||||
let mut disks_num = 0;
|
||||
|
||||
for disk in dis.iter().flatten() {
|
||||
disks_num += 1;
|
||||
total += disk.total;
|
||||
available += disk.total - disk.used;
|
||||
}
|
||||
|
||||
if disks_num < dis.len() / 2 || disks_num == 0 {
|
||||
return Err(Error::other(format!(
|
||||
"not enough online disks to calculate the available space,need {}, found {}",
|
||||
(dis.len() / 2) + 1,
|
||||
disks_num,
|
||||
)));
|
||||
}
|
||||
|
||||
let per_disk = size / disks_num as u64;
|
||||
|
||||
for disk in dis.iter().flatten() {
|
||||
if !is_erasure_sd().await && disk.free_inodes < DISK_MIN_INODES && disk.used_inodes > 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if disk.free <= per_disk {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
|
||||
if available < size {
|
||||
return Ok(false);
|
||||
}
|
||||
available -= size;
|
||||
|
||||
let want = total as f64 * (1.0 - DISK_FILL_FRACTION);
|
||||
|
||||
Ok(available > want as u64)
|
||||
}
|
||||
|
||||
pub(crate) async fn build_server_pools_available_space(
|
||||
bucket: &str,
|
||||
size: i64,
|
||||
n_sets: &[usize],
|
||||
infos: &[Vec<Option<DiskInfo>>],
|
||||
) -> ServerPoolsAvailableSpace {
|
||||
let mut server_pools = vec![PoolAvailableSpace::default(); infos.len()];
|
||||
|
||||
for (i, zinfo) in infos.iter().enumerate() {
|
||||
if zinfo.is_empty() {
|
||||
server_pools[i] = PoolAvailableSpace {
|
||||
index: i,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if !is_meta_bucketname(bucket) && !has_space_for(zinfo, size).await.unwrap_or_default() {
|
||||
server_pools[i] = PoolAvailableSpace {
|
||||
index: i,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut available = 0;
|
||||
let mut max_used_pct = 0;
|
||||
for disk in zinfo.iter().flatten() {
|
||||
if disk.total == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
available += disk.total - disk.used;
|
||||
|
||||
let pct_used = disk.used * 100 / disk.total;
|
||||
|
||||
if pct_used > max_used_pct {
|
||||
max_used_pct = pct_used;
|
||||
}
|
||||
}
|
||||
|
||||
available *= n_sets[i] as u64;
|
||||
|
||||
server_pools[i] = PoolAvailableSpace {
|
||||
index: i,
|
||||
available,
|
||||
max_used_pct,
|
||||
}
|
||||
}
|
||||
|
||||
ServerPoolsAvailableSpace(server_pools)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn disk_info(total: u64, used: u64, free: u64) -> DiskInfo {
|
||||
DiskInfo {
|
||||
total,
|
||||
used,
|
||||
free,
|
||||
free_inodes: 1_024,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_available_space_creation_preserves_fields() {
|
||||
let space = PoolAvailableSpace {
|
||||
@@ -123,4 +236,37 @@ mod tests {
|
||||
|
||||
assert_eq!(indexes, vec![0]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_server_pools_available_space_returns_zero_for_empty_pool_info() {
|
||||
let spaces = build_server_pools_available_space("bucket-a", 64, &[1], &[Vec::new()]).await;
|
||||
|
||||
assert_eq!(spaces.0.len(), 1);
|
||||
assert_eq!(spaces.0[0].index, 0);
|
||||
assert_eq!(spaces.0[0].available, 0);
|
||||
assert_eq!(spaces.0[0].max_used_pct, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_server_pools_available_space_computes_available_capacity_and_max_used_pct() {
|
||||
let infos = vec![vec![Some(disk_info(1_000, 100, 900)), Some(disk_info(1_000, 200, 800))]];
|
||||
|
||||
let spaces = build_server_pools_available_space("bucket-a", 64, &[2], &infos).await;
|
||||
|
||||
assert_eq!(spaces.0.len(), 1);
|
||||
assert_eq!(spaces.0[0].index, 0);
|
||||
assert_eq!(spaces.0[0].available, 3_400);
|
||||
assert_eq!(spaces.0[0].max_used_pct, 20);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_server_pools_available_space_skips_capacity_guard_for_meta_bucket() {
|
||||
let infos = vec![vec![Some(disk_info(10, 9, 1)), Some(disk_info(10, 9, 1))]];
|
||||
|
||||
let spaces = build_server_pools_available_space(crate::disk::RUSTFS_META_BUCKET, 1_024, &[1], &infos).await;
|
||||
|
||||
assert_eq!(spaces.0.len(), 1);
|
||||
assert_eq!(spaces.0[0].available, 2);
|
||||
assert_eq!(spaces.0[0].max_used_pct, 90);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,10 +44,9 @@ use crate::error::{
|
||||
};
|
||||
use crate::event_notification::EventNotifier;
|
||||
use crate::global::{
|
||||
DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME,
|
||||
GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, TypeLocalDiskSetDrives, get_global_deployment_id, get_global_endpoints,
|
||||
get_global_region, get_global_tier_config_mgr, init_global_bucket_monitor, is_erasure_sd, set_global_deployment_id,
|
||||
set_object_layer,
|
||||
DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, TypeLocalDiskSetDrives,
|
||||
get_global_deployment_id, get_global_endpoints, get_global_region, get_global_tier_config_mgr, init_global_bucket_monitor,
|
||||
set_global_deployment_id, set_object_layer,
|
||||
};
|
||||
use crate::notification_sys::get_global_notification_sys;
|
||||
use crate::pools::PoolMeta;
|
||||
@@ -105,7 +104,7 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
pub use crate::layout::pool_space::{PoolAvailableSpace, ServerPoolsAvailableSpace};
|
||||
pub use crate::layout::pool_space::{PoolAvailableSpace, ServerPoolsAvailableSpace, has_space_for};
|
||||
|
||||
/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
|
||||
/// This is used to determine if a bucket is empty for deletion purposes.
|
||||
@@ -178,7 +177,7 @@ mod rebalance;
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
all_local_disk, all_local_disk_path, find_local_disk, find_local_disk_by_ref, get_disk_infos, get_disk_via_endpoint,
|
||||
has_space_for, init_local_disks, init_lock_clients, prewarm_local_disk_id_map,
|
||||
init_local_disks, init_lock_clients, prewarm_local_disk_id_map,
|
||||
};
|
||||
|
||||
pub struct ECStore {
|
||||
|
||||
@@ -239,47 +239,3 @@ pub async fn get_disk_infos(disks: &[Option<DiskStore>]) -> Vec<Option<DiskInfo>
|
||||
|
||||
res
|
||||
}
|
||||
|
||||
pub async fn has_space_for(dis: &[Option<DiskInfo>], size: i64) -> Result<bool> {
|
||||
let size = { if size < 0 { DISK_ASSUME_UNKNOWN_SIZE } else { size as u64 * 2 } };
|
||||
|
||||
let mut available = 0;
|
||||
let mut total = 0;
|
||||
let mut disks_num = 0;
|
||||
|
||||
for disk in dis.iter().flatten() {
|
||||
disks_num += 1;
|
||||
total += disk.total;
|
||||
available += disk.total - disk.used;
|
||||
}
|
||||
|
||||
if disks_num < dis.len() / 2 || disks_num == 0 {
|
||||
return Err(Error::other(format!(
|
||||
"not enough online disks to calculate the available space,need {}, found {}",
|
||||
(dis.len() / 2) + 1,
|
||||
disks_num,
|
||||
)));
|
||||
}
|
||||
|
||||
let per_disk = size / disks_num as u64;
|
||||
|
||||
for disk in dis.iter().flatten() {
|
||||
if !is_erasure_sd().await && disk.free_inodes < DISK_MIN_INODES && disk.used_inodes > 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if disk.free <= per_disk {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
|
||||
if available < size {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
available -= size;
|
||||
|
||||
let want = total as f64 * (1.0 - DISK_FILL_FRACTION);
|
||||
|
||||
Ok(available > want as u64)
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use super::*;
|
||||
use crate::config::get_global_storage_class;
|
||||
use crate::layout::pool_space::build_server_pools_available_space;
|
||||
use rustfs_storage_api::{NamespaceLocking as _, ObjectOperations as _, StorageAdminApi};
|
||||
pub(in crate::store) mod support;
|
||||
use support::{
|
||||
@@ -22,61 +23,6 @@ use support::{
|
||||
resolve_rebalance_delete_from_all_pools_results, resolve_store_rebalance_pool_meta_reload_result,
|
||||
};
|
||||
|
||||
async fn build_server_pools_available_space(
|
||||
bucket: &str,
|
||||
size: i64,
|
||||
n_sets: &[usize],
|
||||
infos: &[Vec<Option<DiskInfo>>],
|
||||
) -> ServerPoolsAvailableSpace {
|
||||
let mut server_pools = vec![PoolAvailableSpace::default(); infos.len()];
|
||||
|
||||
for (i, zinfo) in infos.iter().enumerate() {
|
||||
if zinfo.is_empty() {
|
||||
server_pools[i] = PoolAvailableSpace {
|
||||
index: i,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if !is_meta_bucketname(bucket) && !has_space_for(zinfo, size).await.unwrap_or_default() {
|
||||
server_pools[i] = PoolAvailableSpace {
|
||||
index: i,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut available = 0;
|
||||
let mut max_used_pct = 0;
|
||||
for disk in zinfo.iter().flatten() {
|
||||
if disk.total == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
available += disk.total - disk.used;
|
||||
|
||||
let pct_used = disk.used * 100 / disk.total;
|
||||
|
||||
if pct_used > max_used_pct {
|
||||
max_used_pct = pct_used;
|
||||
}
|
||||
}
|
||||
|
||||
available *= n_sets[i] as u64;
|
||||
|
||||
server_pools[i] = PoolAvailableSpace {
|
||||
index: i,
|
||||
available,
|
||||
max_used_pct,
|
||||
}
|
||||
}
|
||||
|
||||
ServerPoolsAvailableSpace(server_pools)
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
pub(super) async fn delete_all(&self, bucket: &str, prefix: &str) -> Result<()> {
|
||||
@@ -700,7 +646,6 @@ impl ECStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::disk::DiskInfo;
|
||||
|
||||
fn object_info_with_mod_time(unix_ts: i64, delete_marker: bool) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
@@ -710,16 +655,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn disk_info(total: u64, used: u64, free: u64) -> DiskInfo {
|
||||
DiskInfo {
|
||||
total,
|
||||
used,
|
||||
free,
|
||||
free_inodes: 1_024,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_returns_latest_delete_marker() {
|
||||
let candidates = vec![
|
||||
@@ -1008,37 +943,4 @@ mod tests {
|
||||
.contains("failed to resolve rebalance disk set: pool index 2, set index 7, pool count 3")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_server_pools_available_space_returns_zero_for_empty_pool_info() {
|
||||
let spaces = build_server_pools_available_space("bucket-a", 64, &[1], &[Vec::new()]).await;
|
||||
|
||||
assert_eq!(spaces.0.len(), 1);
|
||||
assert_eq!(spaces.0[0].index, 0);
|
||||
assert_eq!(spaces.0[0].available, 0);
|
||||
assert_eq!(spaces.0[0].max_used_pct, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_server_pools_available_space_computes_available_capacity_and_max_used_pct() {
|
||||
let infos = vec![vec![Some(disk_info(1_000, 100, 900)), Some(disk_info(1_000, 200, 800))]];
|
||||
|
||||
let spaces = build_server_pools_available_space("bucket-a", 64, &[2], &infos).await;
|
||||
|
||||
assert_eq!(spaces.0.len(), 1);
|
||||
assert_eq!(spaces.0[0].index, 0);
|
||||
assert_eq!(spaces.0[0].available, 3_400);
|
||||
assert_eq!(spaces.0[0].max_used_pct, 20);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_server_pools_available_space_skips_capacity_guard_for_meta_bucket() {
|
||||
let infos = vec![vec![Some(disk_info(10, 9, 1)), Some(disk_info(10, 9, 1))]];
|
||||
|
||||
let spaces = build_server_pools_available_space(crate::disk::RUSTFS_META_BUCKET, 1_024, &[1], &infos).await;
|
||||
|
||||
assert_eq!(spaces.0.len(), 1);
|
||||
assert_eq!(spaces.0[0].available, 2);
|
||||
assert_eq!(spaces.0[0].max_used_pct, 90);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,18 +5,18 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-ecstore-rebalance-helper-boundary`
|
||||
- Baseline: completed `E-005/E-LAYOUT-004`.
|
||||
- Branch: `overtrue/arch-ecstore-layout-pool-space-builder`
|
||||
- Baseline: completed `E-006/E-REBALANCE-001`.
|
||||
- Stacked on: merged ECStore layout foundation, format layout ownership, and
|
||||
endpoint layout move slices plus the set-format heal and pool-space layout
|
||||
helper slices.
|
||||
helper slices and the rebalance support helper boundary slice.
|
||||
- PR type for this branch: `pure-move`
|
||||
- Runtime behavior changes: none.
|
||||
- Rust code changes: move ECStore rebalance support helper types and pure
|
||||
result reducers into a local rebalance support boundary while preserving
|
||||
store orchestration.
|
||||
- Rust code changes: move ECStore capacity checks and pool-space builder logic
|
||||
into the layout pool-space owner while preserving the old `store` export path
|
||||
and rebalance orchestration.
|
||||
- CI/script changes: none.
|
||||
- Docs changes: record the ECStore rebalance support helper boundary slice.
|
||||
- Docs changes: record the ECStore pool-space builder layout slice.
|
||||
|
||||
## Phase 0 Tasks
|
||||
|
||||
@@ -2299,10 +2299,25 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
checks, migration/layer guards, formatting, diff hygiene, Rust risk scan,
|
||||
branch freshness check, pre-commit quality gate, and three-expert review.
|
||||
|
||||
- [x] `E-007/E-LAYOUT-005` Move ECStore pool-space builder helpers.
|
||||
- Do: move `has_space_for` and server-pool available-space construction into
|
||||
the ECStore layout pool-space owner while keeping `store::has_space_for`
|
||||
source-compatible through re-export.
|
||||
- Acceptance: `layout::pool_space` owns capacity checks, pool availability
|
||||
construction, filter helpers, and focused tests; rebalance only gathers
|
||||
runtime disk snapshots and calls the layout owner.
|
||||
- Must preserve: unknown-size handling, erasure fill-fraction math,
|
||||
inode/free-space guard behavior, meta-bucket capacity bypass, pool index
|
||||
ordering, available-space summation, and rebalance pool selection.
|
||||
- Verification: focused ECStore pool-space and rebalance tests,
|
||||
ECStore/RustFS/Heal compile checks, migration/layer guards, formatting,
|
||||
diff hygiene, Rust risk scan, branch freshness check, pre-commit quality
|
||||
gate, and three-expert review.
|
||||
|
||||
## Next PRs
|
||||
|
||||
1. `pure-move`: continue moving ECStore rebalance/layout support helpers once
|
||||
E-006/E-REBALANCE-001 lands.
|
||||
E-007/E-LAYOUT-005 lands.
|
||||
2. `pure-move`: continue pruning residual embedded startup-only orchestration
|
||||
once the lifecycle helpers are merged.
|
||||
|
||||
@@ -2310,9 +2325,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | passed | E-006/E-REBALANCE-001 extracts rebalance support helpers into a local boundary without moving async store orchestration. |
|
||||
| Migration preservation | passed | Latest-object tie-breaks, delete aggregation, pool lookup error classification, and sibling `store` visibility remain preserved. |
|
||||
| Testing/verification | passed | Focused rebalance tests, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. |
|
||||
| Quality/architecture | passed | E-007/E-LAYOUT-005 moves capacity and pool-space construction into the layout owner while leaving rebalance orchestration in place. |
|
||||
| Migration preservation | passed | Capacity math, inode/free-space guards, meta-bucket bypass, pool ordering, and old `store::has_space_for` imports remain preserved. |
|
||||
| Testing/verification | passed | Focused pool-space/rebalance tests, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. |
|
||||
|
||||
## Verification Notes
|
||||
|
||||
@@ -2437,6 +2452,19 @@ Passed before push:
|
||||
- `make pre-commit`: passed.
|
||||
- Three-expert review: passed.
|
||||
|
||||
- Issue #660 E-007/E-LAYOUT-005 current slice:
|
||||
- `cargo test -p rustfs-ecstore layout::pool_space -- --nocapture`: passed.
|
||||
- `cargo test -p rustfs-ecstore store::rebalance -- --nocapture`: passed.
|
||||
- `cargo check -p rustfs-ecstore -p rustfs -p rustfs-heal`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- Rust risk scan on changed Rust files: passed; added cast lines are moved
|
||||
capacity math from the existing implementation.
|
||||
- `make pre-commit`: passed.
|
||||
- Three-expert review: passed.
|
||||
|
||||
- Issue #660 X-012 current slice:
|
||||
- `cargo test -p rustfs-extension-schema`: passed.
|
||||
- `cargo check -p rustfs-extension-schema`: passed.
|
||||
|
||||
Reference in New Issue
Block a user