diff --git a/crates/ecstore/src/layout/pool_space.rs b/crates/ecstore/src/layout/pool_space.rs index 13fc92cc5..37d297a6b 100644 --- a/crates/ecstore/src/layout/pool_space.rs +++ b/crates/ecstore/src/layout/pool_space.rs @@ -14,6 +14,11 @@ use std::slice::Iter; +use crate::bucket::utils::is_meta_bucketname; +use crate::disk::DiskInfo; +use crate::error::{Error, Result}; +use crate::global::{DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, is_erasure_sd}; + #[derive(Debug, Default, Clone)] pub struct PoolAvailableSpace { pub index: usize, @@ -68,10 +73,118 @@ impl ServerPoolsAvailableSpace { } } +pub async fn has_space_for(dis: &[Option], size: i64) -> Result { + let size = { if size < 0 { DISK_ASSUME_UNKNOWN_SIZE } else { size as u64 * 2 } }; + + let mut available = 0; + let mut total = 0; + let mut disks_num = 0; + + for disk in dis.iter().flatten() { + disks_num += 1; + total += disk.total; + available += disk.total - disk.used; + } + + if disks_num < dis.len() / 2 || disks_num == 0 { + return Err(Error::other(format!( + "not enough online disks to calculate the available space,need {}, found {}", + (dis.len() / 2) + 1, + disks_num, + ))); + } + + let per_disk = size / disks_num as u64; + + for disk in dis.iter().flatten() { + if !is_erasure_sd().await && disk.free_inodes < DISK_MIN_INODES && disk.used_inodes > 0 { + return Ok(false); + } + + if disk.free <= per_disk { + return Ok(false); + } + } + + if available < size { + return Ok(false); + } + available -= size; + + let want = total as f64 * (1.0 - DISK_FILL_FRACTION); + + Ok(available > want as u64) +} + +pub(crate) async fn build_server_pools_available_space( + bucket: &str, + size: i64, + n_sets: &[usize], + infos: &[Vec>], +) -> ServerPoolsAvailableSpace { + let mut server_pools = vec![PoolAvailableSpace::default(); infos.len()]; + + for (i, zinfo) in infos.iter().enumerate() { + if zinfo.is_empty() { + server_pools[i] = PoolAvailableSpace { + index: i, + ..Default::default() + }; + + continue; + } + + if !is_meta_bucketname(bucket) && !has_space_for(zinfo, size).await.unwrap_or_default() { + server_pools[i] = PoolAvailableSpace { + index: i, + ..Default::default() + }; + + continue; + } + + let mut available = 0; + let mut max_used_pct = 0; + for disk in zinfo.iter().flatten() { + if disk.total == 0 { + continue; + } + + available += disk.total - disk.used; + + let pct_used = disk.used * 100 / disk.total; + + if pct_used > max_used_pct { + max_used_pct = pct_used; + } + } + + available *= n_sets[i] as u64; + + server_pools[i] = PoolAvailableSpace { + index: i, + available, + max_used_pct, + } + } + + ServerPoolsAvailableSpace(server_pools) +} + #[cfg(test)] mod tests { use super::*; + fn disk_info(total: u64, used: u64, free: u64) -> DiskInfo { + DiskInfo { + total, + used, + free, + free_inodes: 1_024, + ..Default::default() + } + } + #[test] fn pool_available_space_creation_preserves_fields() { let space = PoolAvailableSpace { @@ -123,4 +236,37 @@ mod tests { assert_eq!(indexes, vec![0]); } + + #[tokio::test] + async fn build_server_pools_available_space_returns_zero_for_empty_pool_info() { + let spaces = build_server_pools_available_space("bucket-a", 64, &[1], &[Vec::new()]).await; + + assert_eq!(spaces.0.len(), 1); + assert_eq!(spaces.0[0].index, 0); + assert_eq!(spaces.0[0].available, 0); + assert_eq!(spaces.0[0].max_used_pct, 0); + } + + #[tokio::test] + async fn build_server_pools_available_space_computes_available_capacity_and_max_used_pct() { + let infos = vec![vec![Some(disk_info(1_000, 100, 900)), Some(disk_info(1_000, 200, 800))]]; + + let spaces = build_server_pools_available_space("bucket-a", 64, &[2], &infos).await; + + assert_eq!(spaces.0.len(), 1); + assert_eq!(spaces.0[0].index, 0); + assert_eq!(spaces.0[0].available, 3_400); + assert_eq!(spaces.0[0].max_used_pct, 20); + } + + #[tokio::test] + async fn build_server_pools_available_space_skips_capacity_guard_for_meta_bucket() { + let infos = vec![vec![Some(disk_info(10, 9, 1)), Some(disk_info(10, 9, 1))]]; + + let spaces = build_server_pools_available_space(crate::disk::RUSTFS_META_BUCKET, 1_024, &[1], &infos).await; + + assert_eq!(spaces.0.len(), 1); + assert_eq!(spaces.0[0].available, 2); + assert_eq!(spaces.0[0].max_used_pct, 90); + } } diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index bde53b519..30b3f331a 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -44,10 +44,9 @@ use crate::error::{ }; use crate::event_notification::EventNotifier; use crate::global::{ - DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME, - GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, TypeLocalDiskSetDrives, get_global_deployment_id, get_global_endpoints, - get_global_region, get_global_tier_config_mgr, init_global_bucket_monitor, is_erasure_sd, set_global_deployment_id, - set_object_layer, + DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, TypeLocalDiskSetDrives, + get_global_deployment_id, get_global_endpoints, get_global_region, get_global_tier_config_mgr, init_global_bucket_monitor, + set_global_deployment_id, set_object_layer, }; use crate::notification_sys::get_global_notification_sys; use crate::pools::PoolMeta; @@ -105,7 +104,7 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo; type ObjectInfoOrErr = StorageObjectInfoOrErr; type WalkOptions = StorageWalkOptions bool>; -pub use crate::layout::pool_space::{PoolAvailableSpace, ServerPoolsAvailableSpace}; +pub use crate::layout::pool_space::{PoolAvailableSpace, ServerPoolsAvailableSpace, has_space_for}; /// Check if a directory contains any xl.meta files (indicating actual S3 objects) /// This is used to determine if a bucket is empty for deletion purposes. @@ -178,7 +177,7 @@ mod rebalance; use peer::init_local_peer; pub use peer::{ all_local_disk, all_local_disk_path, find_local_disk, find_local_disk_by_ref, get_disk_infos, get_disk_via_endpoint, - has_space_for, init_local_disks, init_lock_clients, prewarm_local_disk_id_map, + init_local_disks, init_lock_clients, prewarm_local_disk_id_map, }; pub struct ECStore { diff --git a/crates/ecstore/src/store/peer.rs b/crates/ecstore/src/store/peer.rs index fddde1a56..9e9378b9e 100644 --- a/crates/ecstore/src/store/peer.rs +++ b/crates/ecstore/src/store/peer.rs @@ -239,47 +239,3 @@ pub async fn get_disk_infos(disks: &[Option]) -> Vec res } - -pub async fn has_space_for(dis: &[Option], size: i64) -> Result { - let size = { if size < 0 { DISK_ASSUME_UNKNOWN_SIZE } else { size as u64 * 2 } }; - - let mut available = 0; - let mut total = 0; - let mut disks_num = 0; - - for disk in dis.iter().flatten() { - disks_num += 1; - total += disk.total; - available += disk.total - disk.used; - } - - if disks_num < dis.len() / 2 || disks_num == 0 { - return Err(Error::other(format!( - "not enough online disks to calculate the available space,need {}, found {}", - (dis.len() / 2) + 1, - disks_num, - ))); - } - - let per_disk = size / disks_num as u64; - - for disk in dis.iter().flatten() { - if !is_erasure_sd().await && disk.free_inodes < DISK_MIN_INODES && disk.used_inodes > 0 { - return Ok(false); - } - - if disk.free <= per_disk { - return Ok(false); - } - } - - if available < size { - return Ok(false); - } - - available -= size; - - let want = total as f64 * (1.0 - DISK_FILL_FRACTION); - - Ok(available > want as u64) -} diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index 7e977b19a..1917a5719 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -14,6 +14,7 @@ use super::*; use crate::config::get_global_storage_class; +use crate::layout::pool_space::build_server_pools_available_space; use rustfs_storage_api::{NamespaceLocking as _, ObjectOperations as _, StorageAdminApi}; pub(in crate::store) mod support; use support::{ @@ -22,61 +23,6 @@ use support::{ resolve_rebalance_delete_from_all_pools_results, resolve_store_rebalance_pool_meta_reload_result, }; -async fn build_server_pools_available_space( - bucket: &str, - size: i64, - n_sets: &[usize], - infos: &[Vec>], -) -> ServerPoolsAvailableSpace { - let mut server_pools = vec![PoolAvailableSpace::default(); infos.len()]; - - for (i, zinfo) in infos.iter().enumerate() { - if zinfo.is_empty() { - server_pools[i] = PoolAvailableSpace { - index: i, - ..Default::default() - }; - - continue; - } - - if !is_meta_bucketname(bucket) && !has_space_for(zinfo, size).await.unwrap_or_default() { - server_pools[i] = PoolAvailableSpace { - index: i, - ..Default::default() - }; - - continue; - } - - let mut available = 0; - let mut max_used_pct = 0; - for disk in zinfo.iter().flatten() { - if disk.total == 0 { - continue; - } - - available += disk.total - disk.used; - - let pct_used = disk.used * 100 / disk.total; - - if pct_used > max_used_pct { - max_used_pct = pct_used; - } - } - - available *= n_sets[i] as u64; - - server_pools[i] = PoolAvailableSpace { - index: i, - available, - max_used_pct, - } - } - - ServerPoolsAvailableSpace(server_pools) -} - impl ECStore { #[instrument(level = "debug", skip(self))] pub(super) async fn delete_all(&self, bucket: &str, prefix: &str) -> Result<()> { @@ -700,7 +646,6 @@ impl ECStore { #[cfg(test)] mod tests { use super::*; - use crate::disk::DiskInfo; fn object_info_with_mod_time(unix_ts: i64, delete_marker: bool) -> ObjectInfo { ObjectInfo { @@ -710,16 +655,6 @@ mod tests { } } - fn disk_info(total: u64, used: u64, free: u64) -> DiskInfo { - DiskInfo { - total, - used, - free, - free_inodes: 1_024, - ..Default::default() - } - } - #[test] fn resolve_latest_object_info_candidates_returns_latest_delete_marker() { let candidates = vec![ @@ -1008,37 +943,4 @@ mod tests { .contains("failed to resolve rebalance disk set: pool index 2, set index 7, pool count 3") ); } - - #[tokio::test] - async fn build_server_pools_available_space_returns_zero_for_empty_pool_info() { - let spaces = build_server_pools_available_space("bucket-a", 64, &[1], &[Vec::new()]).await; - - assert_eq!(spaces.0.len(), 1); - assert_eq!(spaces.0[0].index, 0); - assert_eq!(spaces.0[0].available, 0); - assert_eq!(spaces.0[0].max_used_pct, 0); - } - - #[tokio::test] - async fn build_server_pools_available_space_computes_available_capacity_and_max_used_pct() { - let infos = vec![vec![Some(disk_info(1_000, 100, 900)), Some(disk_info(1_000, 200, 800))]]; - - let spaces = build_server_pools_available_space("bucket-a", 64, &[2], &infos).await; - - assert_eq!(spaces.0.len(), 1); - assert_eq!(spaces.0[0].index, 0); - assert_eq!(spaces.0[0].available, 3_400); - assert_eq!(spaces.0[0].max_used_pct, 20); - } - - #[tokio::test] - async fn build_server_pools_available_space_skips_capacity_guard_for_meta_bucket() { - let infos = vec![vec![Some(disk_info(10, 9, 1)), Some(disk_info(10, 9, 1))]]; - - let spaces = build_server_pools_available_space(crate::disk::RUSTFS_META_BUCKET, 1_024, &[1], &infos).await; - - assert_eq!(spaces.0.len(), 1); - assert_eq!(spaces.0[0].available, 2); - assert_eq!(spaces.0[0].max_used_pct, 90); - } } diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index a97948cea..b89746f5b 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,18 +5,18 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-ecstore-rebalance-helper-boundary` -- Baseline: completed `E-005/E-LAYOUT-004`. +- Branch: `overtrue/arch-ecstore-layout-pool-space-builder` +- Baseline: completed `E-006/E-REBALANCE-001`. - Stacked on: merged ECStore layout foundation, format layout ownership, and endpoint layout move slices plus the set-format heal and pool-space layout - helper slices. + helper slices and the rebalance support helper boundary slice. - PR type for this branch: `pure-move` - Runtime behavior changes: none. -- Rust code changes: move ECStore rebalance support helper types and pure - result reducers into a local rebalance support boundary while preserving - store orchestration. +- Rust code changes: move ECStore capacity checks and pool-space builder logic + into the layout pool-space owner while preserving the old `store` export path + and rebalance orchestration. - CI/script changes: none. -- Docs changes: record the ECStore rebalance support helper boundary slice. +- Docs changes: record the ECStore pool-space builder layout slice. ## Phase 0 Tasks @@ -2299,10 +2299,25 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block checks, migration/layer guards, formatting, diff hygiene, Rust risk scan, branch freshness check, pre-commit quality gate, and three-expert review. +- [x] `E-007/E-LAYOUT-005` Move ECStore pool-space builder helpers. + - Do: move `has_space_for` and server-pool available-space construction into + the ECStore layout pool-space owner while keeping `store::has_space_for` + source-compatible through re-export. + - Acceptance: `layout::pool_space` owns capacity checks, pool availability + construction, filter helpers, and focused tests; rebalance only gathers + runtime disk snapshots and calls the layout owner. + - Must preserve: unknown-size handling, erasure fill-fraction math, + inode/free-space guard behavior, meta-bucket capacity bypass, pool index + ordering, available-space summation, and rebalance pool selection. + - Verification: focused ECStore pool-space and rebalance tests, + ECStore/RustFS/Heal compile checks, migration/layer guards, formatting, + diff hygiene, Rust risk scan, branch freshness check, pre-commit quality + gate, and three-expert review. + ## Next PRs 1. `pure-move`: continue moving ECStore rebalance/layout support helpers once - E-006/E-REBALANCE-001 lands. + E-007/E-LAYOUT-005 lands. 2. `pure-move`: continue pruning residual embedded startup-only orchestration once the lifecycle helpers are merged. @@ -2310,9 +2325,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | E-006/E-REBALANCE-001 extracts rebalance support helpers into a local boundary without moving async store orchestration. | -| Migration preservation | passed | Latest-object tie-breaks, delete aggregation, pool lookup error classification, and sibling `store` visibility remain preserved. | -| Testing/verification | passed | Focused rebalance tests, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. | +| Quality/architecture | passed | E-007/E-LAYOUT-005 moves capacity and pool-space construction into the layout owner while leaving rebalance orchestration in place. | +| Migration preservation | passed | Capacity math, inode/free-space guards, meta-bucket bypass, pool ordering, and old `store::has_space_for` imports remain preserved. | +| Testing/verification | passed | Focused pool-space/rebalance tests, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. | ## Verification Notes @@ -2437,6 +2452,19 @@ Passed before push: - `make pre-commit`: passed. - Three-expert review: passed. +- Issue #660 E-007/E-LAYOUT-005 current slice: + - `cargo test -p rustfs-ecstore layout::pool_space -- --nocapture`: passed. + - `cargo test -p rustfs-ecstore store::rebalance -- --nocapture`: passed. + - `cargo check -p rustfs-ecstore -p rustfs -p rustfs-heal`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - Rust risk scan on changed Rust files: passed; added cast lines are moved + capacity math from the existing implementation. + - `make pre-commit`: passed. + - Three-expert review: passed. + - Issue #660 X-012 current slice: - `cargo test -p rustfs-extension-schema`: passed. - `cargo check -p rustfs-extension-schema`: passed.