From 81f11df4dd1e4837974be923be0c0e76ba1a725d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Sun, 21 Jun 2026 11:25:44 +0800 Subject: [PATCH] refactor: add cluster status snapshots (#3682) --- crates/ecstore/src/api/mod.rs | 8 +- crates/ecstore/src/cluster/mod.rs | 233 +++++++++++++++++- docs/architecture/crate-boundaries.md | 3 + docs/architecture/migration-progress.md | 64 ++++- .../storage-control-data-plane.md | 4 + 5 files changed, 294 insertions(+), 18 deletions(-) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 6948f5e5e..b10fa4cb9 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -47,9 +47,11 @@ pub mod client { pub mod cluster { pub use crate::cluster::{ - ClusterControlPlane, ClusterControlPlaneSnapshot, ClusterDriveMembership, ClusterEndpointType, ClusterMembershipSnapshot, - ClusterNodeMembership, membership_snapshot_from_endpoint_pools, topology_snapshot_from_endpoint_pools, - topology_snapshot_from_endpoint_pools_with_capabilities, + ClusterControlPlane, ClusterControlPlaneSnapshot, ClusterDriveMembership, ClusterEndpointType, ClusterLocalNodeStorage, + ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot, ClusterNodeMembership, ClusterPeerHealth, + ClusterPeerHealthSnapshot, ClusterPoolState, ClusterPoolStateSnapshot, local_node_storage_snapshot_from_membership, + membership_snapshot_from_endpoint_pools, peer_health_snapshot_from_membership, pool_state_snapshot_from_endpoint_pools, + topology_snapshot_from_endpoint_pools, topology_snapshot_from_endpoint_pools_with_capabilities, }; } diff --git a/crates/ecstore/src/cluster/mod.rs b/crates/ecstore/src/cluster/mod.rs index f56d6d7ef..02d7ba6be 100644 --- a/crates/ecstore/src/cluster/mod.rs +++ b/crates/ecstore/src/cluster/mod.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use rustfs_storage_api::{ CapabilityStatus, DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet, @@ -31,6 +31,7 @@ const STORAGE_MEDIA_NOT_REPORTED: &str = "storage media not reported by endpoint const FAILURE_DOMAIN_NOT_REPORTED: &str = "failure domain labels not reported by endpoints"; const NUMA_NOT_WIRED: &str = "NUMA topology not wired into runtime"; const PROFILING_NOT_WIRED: &str = "profiling capability not wired into ECStore"; +const PEER_HEALTH_NOT_REPORTED: &str = "peer health not reported by endpoints"; #[derive(Debug, Clone)] pub struct ClusterControlPlane { @@ -50,10 +51,29 @@ impl ClusterControlPlane { membership_snapshot_from_endpoint_pools(&self.endpoint_pools) } + pub fn pool_state_snapshot(&self) -> ClusterPoolStateSnapshot { + pool_state_snapshot_from_endpoint_pools(&self.endpoint_pools) + } + + pub fn local_node_storage_snapshot(&self) -> ClusterLocalNodeStorageSnapshot { + let membership = self.membership_snapshot(); + local_node_storage_snapshot_from_membership(&membership) + } + + pub fn peer_health_snapshot(&self) -> ClusterPeerHealthSnapshot { + let membership = self.membership_snapshot(); + peer_health_snapshot_from_membership(&membership) + } + pub fn read_snapshot(&self) -> ClusterControlPlaneSnapshot { + let membership = self.membership_snapshot(); + ClusterControlPlaneSnapshot { topology: self.topology_snapshot(), - membership: self.membership_snapshot(), + pool_state: self.pool_state_snapshot(), + local_storage: local_node_storage_snapshot_from_membership(&membership), + peer_health: peer_health_snapshot_from_membership(&membership), + membership, } } } @@ -61,6 +81,9 @@ impl ClusterControlPlane { #[derive(Debug, Clone, PartialEq, Eq)] pub struct ClusterControlPlaneSnapshot { pub topology: TopologySnapshot, + pub pool_state: ClusterPoolStateSnapshot, + pub local_storage: ClusterLocalNodeStorageSnapshot, + pub peer_health: ClusterPeerHealthSnapshot, pub membership: ClusterMembershipSnapshot, } @@ -88,7 +111,50 @@ pub struct ClusterDriveMembership { pub endpoint_type: ClusterEndpointType, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ClusterPoolStateSnapshot { + pub pools: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClusterPoolState { + pub pool_index: usize, + pub set_count: usize, + pub drives_per_set: usize, + pub endpoint_count: usize, + pub local_drive_count: usize, + pub remote_drive_count: usize, + pub legacy: bool, + pub endpoint_types: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ClusterLocalNodeStorageSnapshot { + pub nodes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClusterLocalNodeStorage { + pub node_id: String, + pub pools: Vec, + pub drive_count: usize, + pub path_drive_count: usize, + pub url_drive_count: usize, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ClusterPeerHealthSnapshot { + pub peers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClusterPeerHealth { + pub node_id: String, + pub is_local: bool, + pub status: CapabilityStatus, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum ClusterEndpointType { Path, Url, @@ -171,6 +237,82 @@ pub fn membership_snapshot_from_endpoint_pools(endpoint_pools: &EndpointServerPo } } +pub fn pool_state_snapshot_from_endpoint_pools(endpoint_pools: &EndpointServerPools) -> ClusterPoolStateSnapshot { + ClusterPoolStateSnapshot { + pools: endpoint_pools + .as_ref() + .iter() + .enumerate() + .map(|(pool_index, pool)| { + let mut endpoint_types = BTreeSet::new(); + let mut local_drive_count = 0; + + for endpoint in pool.endpoints.as_ref() { + endpoint_types.insert(endpoint_type(endpoint)); + if endpoint.is_local { + local_drive_count += 1; + } + } + + let endpoint_count = pool.endpoints.as_ref().len(); + ClusterPoolState { + pool_index, + set_count: pool.set_count, + drives_per_set: pool.drives_per_set, + endpoint_count, + local_drive_count, + remote_drive_count: endpoint_count.saturating_sub(local_drive_count), + legacy: pool.legacy, + endpoint_types: endpoint_types.into_iter().collect(), + } + }) + .collect(), + } +} + +pub fn local_node_storage_snapshot_from_membership(membership: &ClusterMembershipSnapshot) -> ClusterLocalNodeStorageSnapshot { + ClusterLocalNodeStorageSnapshot { + nodes: membership + .nodes + .iter() + .filter(|node| node.is_local) + .map(|node| { + let mut path_drive_count = 0; + let mut url_drive_count = 0; + + for drive in membership.drives.iter().filter(|drive| drive.node_id == node.node_id) { + match drive.endpoint_type { + ClusterEndpointType::Path => path_drive_count += 1, + ClusterEndpointType::Url => url_drive_count += 1, + } + } + + ClusterLocalNodeStorage { + node_id: node.node_id.clone(), + pools: node.pools.clone(), + drive_count: path_drive_count + url_drive_count, + path_drive_count, + url_drive_count, + } + }) + .collect(), + } +} + +pub fn peer_health_snapshot_from_membership(membership: &ClusterMembershipSnapshot) -> ClusterPeerHealthSnapshot { + ClusterPeerHealthSnapshot { + peers: membership + .nodes + .iter() + .map(|node| ClusterPeerHealth { + node_id: node.node_id.clone(), + is_local: node.is_local, + status: CapabilityStatus::unknown().with_reason(PEER_HEALTH_NOT_REPORTED), + }) + .collect(), + } +} + fn topology_sets_from_endpoints( pool_index: usize, drives_per_set: usize, @@ -346,9 +488,55 @@ mod tests { assert_eq!(snapshot.drives[2].endpoint_type, ClusterEndpointType::Url); } + #[test] + fn pool_state_snapshot_counts_local_remote_drives_and_endpoint_types() { + let endpoint_pools = sample_mixed_endpoint_pools(); + let snapshot = pool_state_snapshot_from_endpoint_pools(&endpoint_pools); + + assert_eq!(snapshot.pools.len(), 1); + assert_eq!(snapshot.pools[0].set_count, 2); + assert_eq!(snapshot.pools[0].drives_per_set, 2); + assert_eq!(snapshot.pools[0].endpoint_count, 4); + assert_eq!(snapshot.pools[0].local_drive_count, 3); + assert_eq!(snapshot.pools[0].remote_drive_count, 1); + assert_eq!( + snapshot.pools[0].endpoint_types, + vec![ClusterEndpointType::Path, ClusterEndpointType::Url] + ); + } + + #[test] + fn local_node_storage_snapshot_keeps_only_local_drive_counts() { + let membership = membership_snapshot_from_endpoint_pools(&sample_mixed_endpoint_pools()); + let snapshot = local_node_storage_snapshot_from_membership(&membership); + + assert_eq!(snapshot.nodes.len(), 2); + assert_eq!(snapshot.nodes[0].node_id, LOCAL_NODE_ID); + assert_eq!(snapshot.nodes[0].drive_count, 2); + assert_eq!(snapshot.nodes[0].path_drive_count, 2); + assert_eq!(snapshot.nodes[0].url_drive_count, 0); + assert_eq!(snapshot.nodes[1].node_id, "node1.example:9000"); + assert_eq!(snapshot.nodes[1].drive_count, 1); + assert_eq!(snapshot.nodes[1].path_drive_count, 0); + assert_eq!(snapshot.nodes[1].url_drive_count, 1); + } + + #[test] + fn peer_health_snapshot_reports_static_unknown_status() { + let membership = membership_snapshot_from_endpoint_pools(&sample_url_endpoint_pools()); + let snapshot = peer_health_snapshot_from_membership(&membership); + + assert_eq!(snapshot.peers.len(), 2); + assert_eq!(snapshot.peers[0].node_id, "node1.example:9000"); + assert!(snapshot.peers[0].is_local); + assert_eq!(snapshot.peers[0].status.reason.as_deref(), Some(PEER_HEALTH_NOT_REPORTED)); + assert_eq!(snapshot.peers[1].node_id, "node2.example:9000"); + assert!(!snapshot.peers[1].is_local); + } + #[test] fn control_plane_read_snapshot_combines_topology_and_membership() { - let control_plane = ClusterControlPlane::new(sample_path_endpoint_pools()); + let control_plane = ClusterControlPlane::new(sample_mixed_endpoint_pools()); let snapshot = control_plane.read_snapshot(); let node_ids = snapshot .membership @@ -358,7 +546,10 @@ mod tests { .collect::>(); assert_eq!(snapshot.topology.pools[0].sets.len(), 2); - assert_eq!(node_ids, BTreeSet::from([LOCAL_NODE_ID])); + assert_eq!(snapshot.pool_state.pools[0].endpoint_count, 4); + assert_eq!(snapshot.local_storage.nodes.len(), 2); + assert_eq!(snapshot.peer_health.peers.len(), 3); + assert_eq!(node_ids, BTreeSet::from([LOCAL_NODE_ID, "node1.example:9000", "node2.example:9000"])); } fn sample_path_endpoint_pools() -> EndpointServerPools { @@ -406,4 +597,36 @@ mod tests { platform: "OS: test | Arch: test".to_owned(), }]) } + + fn sample_mixed_endpoint_pools() -> EndpointServerPools { + let mut endpoints = Vec::new(); + + for index in 0..2 { + let mut endpoint = + Endpoint::try_from(format!("/tmp/rustfs-cluster-control-plane-{index}").as_str()).expect("local endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(index); + endpoints.push(endpoint); + } + + for index in 2..4 { + let host = if index == 2 { "node1.example" } else { "node2.example" }; + let mut endpoint = Endpoint::try_from(format!("http://{host}:9000/export{index}").as_str()).expect("url endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(1); + endpoint.set_disk_index(index - 2); + endpoint.is_local = index == 2; + endpoints.push(endpoint); + } + + EndpointServerPools::from(vec![PoolEndpoints { + legacy: false, + set_count: 2, + drives_per_set: 2, + endpoints: Endpoints::from(endpoints), + cmd_line: "mixed-test-endpoints".to_owned(), + platform: "OS: test | Arch: test".to_owned(), + }]) + } } diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md index 419dbd759..78babb290 100644 --- a/docs/architecture/crate-boundaries.md +++ b/docs/architecture/crate-boundaries.md @@ -146,6 +146,9 @@ ECStore root `global` re-exports must also stay removed once consumers use ECStore ClusterControlPlane read models must stay owned by the crate-private `cluster` module. Public access goes through `rustfs_ecstore::api::cluster` so outer crates cannot depend on ECStore root control-plane internals. +Pool-state, local-node storage, and peer-health status projections are part of +the same facade boundary and must remain read-only until a later controller +slice explicitly wires dynamic health or membership behavior. RustFS startup internals must stay crate-private after the startup owner split. Only `startup_entrypoint` remains a public startup module for the binary diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 1f9d7cc99..db3a0013c 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,15 +5,15 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-cluster-control-plane-read-models` -- Baseline: completed `API-078`. -- Stacked on: ECStore root global facade prune. +- Branch: `overtrue/arch-cluster-control-plane-status-snapshots` +- Baseline: completed `C-001/C-002/C-003`. +- Stacked on: ECStore cluster control-plane read models. - PR type for this branch: `api-extraction` - Runtime behavior changes: none. -- Rust code changes: add ECStore ClusterControlPlane read models and route - RustFS runtime endpoint topology snapshots through the ECStore owner facade. -- CI/script changes: extend architecture rules for the cluster facade boundary. -- Docs changes: record the C-001/C-002/C-003 read-model slice. +- Rust code changes: add pool-state, local-node storage, and peer-health status + read models to the ECStore ClusterControlPlane snapshot. +- CI/script changes: none. +- Docs changes: record the C-004/C-005/C-006 status snapshot slice. ## Phase 0 Tasks @@ -539,6 +539,40 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block coverage, formatting, diff hygiene, risk scan, pre-commit quality gate, and three-expert review. +- [x] `C-004` Add pool state snapshot. + - Completed slice: add a static pool-state snapshot derived from existing + endpoint pools and expose it through `rustfs_ecstore::api::cluster`. + - Acceptance: pool state records pool index, set count, drives per set, + endpoint counts, local/remote drive counts, legacy flag, and endpoint type + coverage without reading disks or changing pool ownership. + - Must preserve: no placement change, no pool mutation, no command-line path + exposure, and no endpoint publication changes. + - Verification: ECStore pool-state tests, compile coverage, formatting, diff + hygiene, risk scan, pre-commit quality gate, and three-expert review. + +- [x] `C-005` Add local-node storage snapshot. + - Completed slice: add a read-only local-node storage projection from static + endpoint membership. + - Acceptance: local nodes include only local membership entries and report + aggregate path/url drive counts and pool coverage without exposing local + disk paths. + - Must preserve: no storage readiness, disk health, lock quorum, or object I/O + behavior changes. + - Verification: ECStore local-node storage tests, compile coverage, + formatting, diff hygiene, risk scan, pre-commit quality gate, and + three-expert review. + +- [x] `C-006` Add peer health snapshot. + - Completed slice: add a static peer-health read model that reports peer + identities from membership with unknown health status until real peer health + wiring lands. + - Acceptance: peer health is explicitly unknown and read-only; no background + health checks, RPC calls, timers, or failure-state mutation are introduced. + - Must preserve: no dynamic membership, no peer health loop, no control RPC, + no readiness impact, and no lock/object behavior changes. + - Verification: ECStore peer-health tests, compile coverage, formatting, diff + hygiene, risk scan, pre-commit quality gate, and three-expert review. + - [x] `TEST-PRTYPE-001` Check PR type enum consistency. - Acceptance: `./scripts/check_architecture_migration_rules.sh` parses the allowed PR types from [`crate-boundaries.md`](crate-boundaries.md) and fails @@ -3067,14 +3101,24 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | C-001/C-002/C-003 keeps ClusterControlPlane read models in ECStore with public access only through `api::cluster`. | -| Migration preservation | passed | Topology and static membership are read-only snapshots from existing endpoint pools; no placement, readiness, health, or lock behavior changes. | -| Testing/verification | passed | ECStore cluster tests, RustFS runtime capability tests, compile coverage, migration guard, formatting, diff hygiene, risk scan, and pre-commit gate passed. | +| Quality/architecture | passed | C-004/C-005/C-006 extends ClusterControlPlane with pool-state, local-node storage, and peer-health status read models behind `api::cluster`. | +| Migration preservation | passed | New status surfaces remain static read-only projections; no placement, readiness, health loop, lock, or object I/O behavior changes. | +| Testing/verification | passed | ECStore cluster tests, ECStore/RustFS compile coverage, migration guard, formatting, diff hygiene, risk scan, full pre-commit, and three-expert review passed. | ## Verification Notes Passed before push: +- Issue #660 C-004/C-005/C-006 current slice: + - `cargo test -p rustfs-ecstore cluster -- --nocapture`: passed, 7 tests. + - `cargo check -p rustfs-ecstore --all-targets`: passed. + - `cargo check -p rustfs --lib --bins`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - Rust added-line risk scan on changed Rust files: passed. + - `make pre-commit`: passed. + - Issue #660 C-001/C-002/C-003 current slice: - `cargo check -p rustfs-ecstore --all-targets`: passed. - `cargo check -p rustfs --lib --bins`: passed. diff --git a/docs/architecture/storage-control-data-plane.md b/docs/architecture/storage-control-data-plane.md index 71de6152d..44107e974 100644 --- a/docs/architecture/storage-control-data-plane.md +++ b/docs/architecture/storage-control-data-plane.md @@ -41,6 +41,10 @@ The first read-only implementation lives behind `rustfs_ecstore::api::cluster`. It maps existing endpoint pools into the shared storage-api topology contract and an ECStore-owned static membership snapshot. It must not expose local disk paths, start health checks, mutate endpoint ownership, or change placement/readiness. +The same facade also owns static pool-state, local-node storage, and peer-health +status projections. Peer health remains explicitly unknown until a later slice +wires real health signals; this document does not authorize background probes or +RPC-based health checks. Risk controls: