refactor: move ecstore owner layout modules (#3932)

crates/ecstore/src/cluster/control_plane.rs

@@ -0,0 +1,635 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{BTreeMap, BTreeSet};
+
+use crate::storage_api_contracts::topology::{
+    CapabilityStatus, DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet,
+    TopologySnapshot,
+};
+
+use crate::{
+    endpoints::EndpointServerPools,
+    layout::endpoint::{Endpoint, EndpointType},
+};
+
+const ENDPOINT_TYPE_LABEL: &str = "endpoint_type";
+const LOCAL_ENDPOINT_LABEL: &str = "local";
+const LOCAL_NODE_ID: &str = "local";
+const STORAGE_MEDIA_NOT_REPORTED: &str = "storage media not reported by endpoints";
+const FAILURE_DOMAIN_NOT_REPORTED: &str = "failure domain labels not reported by endpoints";
+const NUMA_NOT_WIRED: &str = "NUMA topology not wired into runtime";
+const PROFILING_NOT_WIRED: &str = "profiling capability not wired into ECStore";
+const PEER_HEALTH_NOT_REPORTED: &str = "peer health not reported by endpoints";
+
+#[derive(Debug, Clone)]
+pub struct ClusterControlPlane {
+    endpoint_pools: EndpointServerPools,
+}
+
+impl ClusterControlPlane {
+    pub fn new(endpoint_pools: EndpointServerPools) -> Self {
+        Self { endpoint_pools }
+    }
+
+    pub fn topology_snapshot(&self) -> TopologySnapshot {
+        topology_snapshot_from_endpoint_pools(&self.endpoint_pools)
+    }
+
+    pub fn membership_snapshot(&self) -> ClusterMembershipSnapshot {
+        membership_snapshot_from_endpoint_pools(&self.endpoint_pools)
+    }
+
+    pub fn pool_state_snapshot(&self) -> ClusterPoolStateSnapshot {
+        pool_state_snapshot_from_endpoint_pools(&self.endpoint_pools)
+    }
+
+    pub fn local_node_storage_snapshot(&self) -> ClusterLocalNodeStorageSnapshot {
+        let membership = self.membership_snapshot();
+        local_node_storage_snapshot_from_membership(&membership)
+    }
+
+    pub fn peer_health_snapshot(&self) -> ClusterPeerHealthSnapshot {
+        let membership = self.membership_snapshot();
+        peer_health_snapshot_from_membership(&membership)
+    }
+
+    pub fn read_snapshot(&self) -> ClusterControlPlaneSnapshot {
+        let membership = self.membership_snapshot();
+
+        ClusterControlPlaneSnapshot {
+            topology: self.topology_snapshot(),
+            pool_state: self.pool_state_snapshot(),
+            local_storage: local_node_storage_snapshot_from_membership(&membership),
+            peer_health: peer_health_snapshot_from_membership(&membership),
+            membership,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ClusterControlPlaneSnapshot {
+    pub topology: TopologySnapshot,
+    pub pool_state: ClusterPoolStateSnapshot,
+    pub local_storage: ClusterLocalNodeStorageSnapshot,
+    pub peer_health: ClusterPeerHealthSnapshot,
+    pub membership: ClusterMembershipSnapshot,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct ClusterMembershipSnapshot {
+    pub nodes: Vec<ClusterNodeMembership>,
+    pub drives: Vec<ClusterDriveMembership>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ClusterNodeMembership {
+    pub node_id: String,
+    pub grid_host: String,
+    pub is_local: bool,
+    pub pools: Vec<usize>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ClusterDriveMembership {
+    pub pool_index: usize,
+    pub set_index: usize,
+    pub disk_index: usize,
+    pub node_id: String,
+    pub is_local: bool,
+    pub endpoint_type: ClusterEndpointType,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct ClusterPoolStateSnapshot {
+    pub pools: Vec<ClusterPoolState>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ClusterPoolState {
+    pub pool_index: usize,
+    pub set_count: usize,
+    pub drives_per_set: usize,
+    pub endpoint_count: usize,
+    pub local_drive_count: usize,
+    pub remote_drive_count: usize,
+    pub legacy: bool,
+    pub endpoint_types: Vec<ClusterEndpointType>,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct ClusterLocalNodeStorageSnapshot {
+    pub nodes: Vec<ClusterLocalNodeStorage>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ClusterLocalNodeStorage {
+    pub node_id: String,
+    pub pools: Vec<usize>,
+    pub drive_count: usize,
+    pub path_drive_count: usize,
+    pub url_drive_count: usize,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct ClusterPeerHealthSnapshot {
+    pub peers: Vec<ClusterPeerHealth>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ClusterPeerHealth {
+    pub node_id: String,
+    pub is_local: bool,
+    pub status: CapabilityStatus,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum ClusterEndpointType {
+    Path,
+    Url,
+}
+
+pub fn topology_snapshot_from_endpoint_pools(endpoint_pools: &EndpointServerPools) -> TopologySnapshot {
+    topology_snapshot_from_endpoint_pools_with_capabilities(
+        endpoint_pools,
+        default_topology_capabilities(),
+        default_disk_capabilities(),
+    )
+}
+
+pub fn topology_snapshot_from_endpoint_pools_with_capabilities(
+    endpoint_pools: &EndpointServerPools,
+    capabilities: TopologyCapabilities,
+    disk_capabilities: DiskCapabilities,
+) -> TopologySnapshot {
+    TopologySnapshot {
+        pools: endpoint_pools
+            .as_ref()
+            .iter()
+            .enumerate()
+            .map(|(pool_index, pool)| {
+                let sets =
+                    topology_sets_from_endpoints(pool_index, pool.drives_per_set, pool.endpoints.as_ref(), &disk_capabilities);
+                TopologyPool {
+                    pool_index,
+                    pool_id: None,
+                    labels: TopologyLabels::default(),
+                    sets,
+                }
+            })
+            .collect(),
+        capabilities,
+    }
+}
+
+pub fn membership_snapshot_from_endpoint_pools(endpoint_pools: &EndpointServerPools) -> ClusterMembershipSnapshot {
+    let mut nodes = BTreeMap::<String, ClusterNodeMembership>::new();
+    let mut drives = Vec::new();
+
+    for (pool_index, pool) in endpoint_pools.as_ref().iter().enumerate() {
+        for (endpoint_index, endpoint) in pool.endpoints.as_ref().iter().enumerate() {
+            let (set_index, disk_index) = endpoint_indices(pool_index, endpoint_index, pool.drives_per_set, endpoint);
+            let node_id = endpoint_node_id(endpoint);
+
+            match nodes.entry(node_id.clone()) {
+                std::collections::btree_map::Entry::Vacant(entry) => {
+                    entry.insert(ClusterNodeMembership {
+                        node_id: node_id.clone(),
+                        grid_host: endpoint.grid_host(),
+                        is_local: endpoint.is_local,
+                        pools: vec![pool_index],
+                    });
+                }
+                std::collections::btree_map::Entry::Occupied(mut entry) => {
+                    let node = entry.get_mut();
+                    node.is_local |= endpoint.is_local;
+                    if !node.pools.contains(&pool_index) {
+                        node.pools.push(pool_index);
+                    }
+                }
+            }
+
+            drives.push(ClusterDriveMembership {
+                pool_index,
+                set_index,
+                disk_index,
+                node_id,
+                is_local: endpoint.is_local,
+                endpoint_type: endpoint_type(endpoint),
+            });
+        }
+    }
+
+    ClusterMembershipSnapshot {
+        nodes: nodes.into_values().collect(),
+        drives,
+    }
+}
+
+pub fn pool_state_snapshot_from_endpoint_pools(endpoint_pools: &EndpointServerPools) -> ClusterPoolStateSnapshot {
+    ClusterPoolStateSnapshot {
+        pools: endpoint_pools
+            .as_ref()
+            .iter()
+            .enumerate()
+            .map(|(pool_index, pool)| {
+                let mut endpoint_types = BTreeSet::new();
+                let mut local_drive_count = 0;
+
+                for endpoint in pool.endpoints.as_ref() {
+                    endpoint_types.insert(endpoint_type(endpoint));
+                    if endpoint.is_local {
+                        local_drive_count += 1;
+                    }
+                }
+
+                let endpoint_count = pool.endpoints.as_ref().len();
+                ClusterPoolState {
+                    pool_index,
+                    set_count: pool.set_count,
+                    drives_per_set: pool.drives_per_set,
+                    endpoint_count,
+                    local_drive_count,
+                    remote_drive_count: endpoint_count.saturating_sub(local_drive_count),
+                    legacy: pool.legacy,
+                    endpoint_types: endpoint_types.into_iter().collect(),
+                }
+            })
+            .collect(),
+    }
+}
+
+pub fn local_node_storage_snapshot_from_membership(membership: &ClusterMembershipSnapshot) -> ClusterLocalNodeStorageSnapshot {
+    ClusterLocalNodeStorageSnapshot {
+        nodes: membership
+            .nodes
+            .iter()
+            .filter(|node| node.is_local)
+            .map(|node| {
+                let mut path_drive_count = 0;
+                let mut url_drive_count = 0;
+
+                for drive in membership.drives.iter().filter(|drive| drive.node_id == node.node_id) {
+                    match drive.endpoint_type {
+                        ClusterEndpointType::Path => path_drive_count += 1,
+                        ClusterEndpointType::Url => url_drive_count += 1,
+                    }
+                }
+
+                ClusterLocalNodeStorage {
+                    node_id: node.node_id.clone(),
+                    pools: node.pools.clone(),
+                    drive_count: path_drive_count + url_drive_count,
+                    path_drive_count,
+                    url_drive_count,
+                }
+            })
+            .collect(),
+    }
+}
+
+pub fn peer_health_snapshot_from_membership(membership: &ClusterMembershipSnapshot) -> ClusterPeerHealthSnapshot {
+    ClusterPeerHealthSnapshot {
+        peers: membership
+            .nodes
+            .iter()
+            .map(|node| ClusterPeerHealth {
+                node_id: node.node_id.clone(),
+                is_local: node.is_local,
+                status: CapabilityStatus::unknown().with_reason(PEER_HEALTH_NOT_REPORTED),
+            })
+            .collect(),
+    }
+}
+
+fn topology_sets_from_endpoints(
+    pool_index: usize,
+    drives_per_set: usize,
+    endpoints: &[Endpoint],
+    disk_capabilities: &DiskCapabilities,
+) -> Vec<TopologySet> {
+    let mut sets = BTreeMap::<usize, Vec<TopologyDisk>>::new();
+
+    for (endpoint_index, endpoint) in endpoints.iter().enumerate() {
+        let (set_index, disk_index) = endpoint_indices(pool_index, endpoint_index, drives_per_set, endpoint);
+        sets.entry(set_index).or_default().push(topology_disk_from_endpoint(
+            pool_index,
+            set_index,
+            disk_index,
+            endpoint,
+            disk_capabilities,
+        ));
+    }
+
+    sets.into_iter()
+        .map(|(set_index, mut disks)| {
+            disks.sort_by_key(|disk| disk.disk_index);
+            TopologySet {
+                pool_index,
+                set_index,
+                set_id: None,
+                labels: TopologyLabels::default(),
+                disks,
+            }
+        })
+        .collect()
+}
+
+fn topology_disk_from_endpoint(
+    pool_index: usize,
+    set_index: usize,
+    disk_index: usize,
+    endpoint: &Endpoint,
+    disk_capabilities: &DiskCapabilities,
+) -> TopologyDisk {
+    TopologyDisk {
+        pool_index,
+        set_index,
+        disk_index,
+        disk_id: endpoint_disk_id(endpoint),
+        labels: endpoint_labels(endpoint),
+        capabilities: disk_capabilities.clone(),
+    }
+}
+
+fn endpoint_indices(pool_index: usize, endpoint_index: usize, drives_per_set: usize, endpoint: &Endpoint) -> (usize, usize) {
+    let safe_drives_per_set = drives_per_set.max(1);
+    let set_index = non_negative_index(endpoint.set_idx).unwrap_or(endpoint_index / safe_drives_per_set);
+    let disk_index = non_negative_index(endpoint.disk_idx).unwrap_or(endpoint_index % safe_drives_per_set);
+
+    debug_assert_eq!(non_negative_index(endpoint.pool_idx).unwrap_or(pool_index), pool_index);
+    (set_index, disk_index)
+}
+
+fn endpoint_disk_id(endpoint: &Endpoint) -> Option<String> {
+    let host_port = endpoint.host_port();
+    if host_port.is_empty() { None } else { Some(host_port) }
+}
+
+fn endpoint_node_id(endpoint: &Endpoint) -> String {
+    endpoint_disk_id(endpoint).unwrap_or_else(|| LOCAL_NODE_ID.to_owned())
+}
+
+fn endpoint_labels(endpoint: &Endpoint) -> TopologyLabels {
+    let mut additional = BTreeMap::new();
+    additional.insert(ENDPOINT_TYPE_LABEL.to_owned(), endpoint_type_label(endpoint).to_owned());
+    additional.insert(LOCAL_ENDPOINT_LABEL.to_owned(), endpoint.is_local.to_string());
+
+    TopologyLabels {
+        node: Some(endpoint_node_id(endpoint)),
+        additional,
+        ..TopologyLabels::default()
+    }
+}
+
+fn endpoint_type(endpoint: &Endpoint) -> ClusterEndpointType {
+    match endpoint.get_type() {
+        EndpointType::Path => ClusterEndpointType::Path,
+        EndpointType::Url => ClusterEndpointType::Url,
+    }
+}
+
+fn endpoint_type_label(endpoint: &Endpoint) -> &'static str {
+    match endpoint_type(endpoint) {
+        ClusterEndpointType::Path => "path",
+        ClusterEndpointType::Url => "url",
+    }
+}
+
+fn non_negative_index(index: i32) -> Option<usize> {
+    usize::try_from(index).ok()
+}
+
+fn default_topology_capabilities() -> TopologyCapabilities {
+    TopologyCapabilities {
+        profiling: CapabilityStatus::unknown().with_reason(PROFILING_NOT_WIRED),
+        numa: CapabilityStatus::unsupported().with_reason(NUMA_NOT_WIRED),
+        failure_domain_labels: CapabilityStatus::unknown().with_reason(FAILURE_DOMAIN_NOT_REPORTED),
+        media_labels: CapabilityStatus::unknown().with_reason(STORAGE_MEDIA_NOT_REPORTED),
+    }
+}
+
+fn default_disk_capabilities() -> DiskCapabilities {
+    DiskCapabilities {
+        media_type: CapabilityStatus::unknown().with_reason(STORAGE_MEDIA_NOT_REPORTED),
+        failure_domain: CapabilityStatus::unknown().with_reason(FAILURE_DOMAIN_NOT_REPORTED),
+        numa: CapabilityStatus::unsupported().with_reason(NUMA_NOT_WIRED),
+        profiling: CapabilityStatus::unknown().with_reason(PROFILING_NOT_WIRED),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::BTreeSet;
+
+    use crate::endpoints::{Endpoints, PoolEndpoints};
+
+    #[test]
+    fn topology_snapshot_maps_endpoint_sets_without_local_paths() {
+        let endpoint_pools = sample_path_endpoint_pools();
+        let snapshot = topology_snapshot_from_endpoint_pools(&endpoint_pools);
+
+        assert_eq!(snapshot.pools.len(), 1);
+        assert_eq!(snapshot.pools[0].sets.len(), 2);
+        assert_eq!(snapshot.pools[0].sets[0].disks.len(), 2);
+        assert_eq!(snapshot.pools[0].sets[1].disks.len(), 2);
+        assert_eq!(snapshot.pools[0].sets[1].disks[1].disk_index, 1);
+        assert_eq!(
+            snapshot.pools[0].sets[0].disks[0]
+                .labels
+                .additional
+                .get(ENDPOINT_TYPE_LABEL)
+                .map(String::as_str),
+            Some("path")
+        );
+        assert_eq!(snapshot.pools[0].sets[0].disks[0].labels.node.as_deref(), Some(LOCAL_NODE_ID));
+
+        let encoded = serde_json::to_string(&snapshot).expect("serialize topology snapshot");
+        assert!(!encoded.contains("/tmp/rustfs-cluster-control-plane"));
+    }
+
+    #[test]
+    fn topology_snapshot_uses_url_hosts_as_disk_ids() {
+        let endpoint_pools = sample_url_endpoint_pools();
+        let snapshot = topology_snapshot_from_endpoint_pools(&endpoint_pools);
+
+        assert_eq!(snapshot.pools[0].sets[0].disks[0].disk_id.as_deref(), Some("node1.example:9000"));
+        assert_eq!(snapshot.pools[0].sets[0].disks[0].labels.node.as_deref(), Some("node1.example:9000"));
+        assert_eq!(
+            snapshot.pools[0].sets[0].disks[0]
+                .labels
+                .additional
+                .get(ENDPOINT_TYPE_LABEL)
+                .map(String::as_str),
+            Some("url")
+        );
+    }
+
+    #[test]
+    fn membership_snapshot_groups_nodes_and_drives() {
+        let endpoint_pools = sample_url_endpoint_pools();
+        let snapshot = membership_snapshot_from_endpoint_pools(&endpoint_pools);
+
+        assert_eq!(snapshot.nodes.len(), 2);
+        assert_eq!(snapshot.drives.len(), 4);
+        assert_eq!(snapshot.nodes[0].node_id, "node1.example:9000");
+        assert_eq!(snapshot.nodes[0].pools, vec![0]);
+        assert_eq!(snapshot.drives[2].node_id, "node2.example:9000");
+        assert_eq!(snapshot.drives[2].set_index, 1);
+        assert_eq!(snapshot.drives[2].endpoint_type, ClusterEndpointType::Url);
+    }
+
+    #[test]
+    fn pool_state_snapshot_counts_local_remote_drives_and_endpoint_types() {
+        let endpoint_pools = sample_mixed_endpoint_pools();
+        let snapshot = pool_state_snapshot_from_endpoint_pools(&endpoint_pools);
+
+        assert_eq!(snapshot.pools.len(), 1);
+        assert_eq!(snapshot.pools[0].set_count, 2);
+        assert_eq!(snapshot.pools[0].drives_per_set, 2);
+        assert_eq!(snapshot.pools[0].endpoint_count, 4);
+        assert_eq!(snapshot.pools[0].local_drive_count, 3);
+        assert_eq!(snapshot.pools[0].remote_drive_count, 1);
+        assert_eq!(
+            snapshot.pools[0].endpoint_types,
+            vec![ClusterEndpointType::Path, ClusterEndpointType::Url]
+        );
+    }
+
+    #[test]
+    fn local_node_storage_snapshot_keeps_only_local_drive_counts() {
+        let membership = membership_snapshot_from_endpoint_pools(&sample_mixed_endpoint_pools());
+        let snapshot = local_node_storage_snapshot_from_membership(&membership);
+
+        assert_eq!(snapshot.nodes.len(), 2);
+        assert_eq!(snapshot.nodes[0].node_id, LOCAL_NODE_ID);
+        assert_eq!(snapshot.nodes[0].drive_count, 2);
+        assert_eq!(snapshot.nodes[0].path_drive_count, 2);
+        assert_eq!(snapshot.nodes[0].url_drive_count, 0);
+        assert_eq!(snapshot.nodes[1].node_id, "node1.example:9000");
+        assert_eq!(snapshot.nodes[1].drive_count, 1);
+        assert_eq!(snapshot.nodes[1].path_drive_count, 0);
+        assert_eq!(snapshot.nodes[1].url_drive_count, 1);
+    }
+
+    #[test]
+    fn peer_health_snapshot_reports_static_unknown_status() {
+        let membership = membership_snapshot_from_endpoint_pools(&sample_url_endpoint_pools());
+        let snapshot = peer_health_snapshot_from_membership(&membership);
+
+        assert_eq!(snapshot.peers.len(), 2);
+        assert_eq!(snapshot.peers[0].node_id, "node1.example:9000");
+        assert!(snapshot.peers[0].is_local);
+        assert_eq!(snapshot.peers[0].status.reason.as_deref(), Some(PEER_HEALTH_NOT_REPORTED));
+        assert_eq!(snapshot.peers[1].node_id, "node2.example:9000");
+        assert!(!snapshot.peers[1].is_local);
+    }
+
+    #[test]
+    fn control_plane_read_snapshot_combines_topology_and_membership() {
+        let control_plane = ClusterControlPlane::new(sample_mixed_endpoint_pools());
+        let snapshot = control_plane.read_snapshot();
+        let node_ids = snapshot
+            .membership
+            .nodes
+            .iter()
+            .map(|node| node.node_id.as_str())
+            .collect::<BTreeSet<_>>();
+
+        assert_eq!(snapshot.topology.pools[0].sets.len(), 2);
+        assert_eq!(snapshot.pool_state.pools[0].endpoint_count, 4);
+        assert_eq!(snapshot.local_storage.nodes.len(), 2);
+        assert_eq!(snapshot.peer_health.peers.len(), 3);
+        assert_eq!(node_ids, BTreeSet::from([LOCAL_NODE_ID, "node1.example:9000", "node2.example:9000"]));
+    }
+
+    fn sample_path_endpoint_pools() -> EndpointServerPools {
+        let endpoints = (0..4)
+            .map(|index| {
+                let mut endpoint =
+                    Endpoint::try_from(format!("/tmp/rustfs-cluster-control-plane-{index}").as_str()).expect("local endpoint");
+                endpoint.set_pool_index(0);
+                endpoint.set_set_index(index / 2);
+                endpoint.set_disk_index(index % 2);
+                endpoint
+            })
+            .collect::<Vec<_>>();
+
+        EndpointServerPools::from(vec![PoolEndpoints {
+            legacy: false,
+            set_count: 2,
+            drives_per_set: 2,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: "/tmp/rustfs-cluster-control-plane-{0...3}".to_owned(),
+            platform: "OS: test | Arch: test".to_owned(),
+        }])
+    }
+
+    fn sample_url_endpoint_pools() -> EndpointServerPools {
+        let endpoints = (0..4)
+            .map(|index| {
+                let host = if index < 2 { "node1.example" } else { "node2.example" };
+                let mut endpoint =
+                    Endpoint::try_from(format!("http://{host}:9000/export{index}").as_str()).expect("url endpoint");
+                endpoint.set_pool_index(0);
+                endpoint.set_set_index(index / 2);
+                endpoint.set_disk_index(index % 2);
+                endpoint.is_local = index < 2;
+                endpoint
+            })
+            .collect::<Vec<_>>();
+
+        EndpointServerPools::from(vec![PoolEndpoints {
+            legacy: false,
+            set_count: 2,
+            drives_per_set: 2,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: "http://node{1...2}.example:9000/export{0...3}".to_owned(),
+            platform: "OS: test | Arch: test".to_owned(),
+        }])
+    }
+
+    fn sample_mixed_endpoint_pools() -> EndpointServerPools {
+        let mut endpoints = Vec::new();
+
+        for index in 0..2 {
+            let mut endpoint =
+                Endpoint::try_from(format!("/tmp/rustfs-cluster-control-plane-{index}").as_str()).expect("local endpoint");
+            endpoint.set_pool_index(0);
+            endpoint.set_set_index(0);
+            endpoint.set_disk_index(index);
+            endpoints.push(endpoint);
+        }
+
+        for index in 2..4 {
+            let host = if index == 2 { "node1.example" } else { "node2.example" };
+            let mut endpoint = Endpoint::try_from(format!("http://{host}:9000/export{index}").as_str()).expect("url endpoint");
+            endpoint.set_pool_index(0);
+            endpoint.set_set_index(1);
+            endpoint.set_disk_index(index - 2);
+            endpoint.is_local = index == 2;
+            endpoints.push(endpoint);
+        }
+
+        EndpointServerPools::from(vec![PoolEndpoints {
+            legacy: false,
+            set_count: 2,
+            drives_per_set: 2,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: "mixed-test-endpoints".to_owned(),
+            platform: "OS: test | Arch: test".to_owned(),
+        }])
+    }
+}

crates/ecstore/src/cluster/mod.rs

@@ -12,624 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, BTreeSet};
+mod control_plane;
 
-use crate::storage_api_contracts::topology::{
-    CapabilityStatus, DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet,
-    TopologySnapshot,
+pub use control_plane::{
+    ClusterControlPlane, ClusterControlPlaneSnapshot, ClusterDriveMembership, ClusterEndpointType, ClusterLocalNodeStorage,
+    ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot, ClusterNodeMembership, ClusterPeerHealth,
+    ClusterPeerHealthSnapshot, ClusterPoolState, ClusterPoolStateSnapshot, local_node_storage_snapshot_from_membership,
+    membership_snapshot_from_endpoint_pools, peer_health_snapshot_from_membership, pool_state_snapshot_from_endpoint_pools,
+    topology_snapshot_from_endpoint_pools, topology_snapshot_from_endpoint_pools_with_capabilities,
 };
-
-use crate::{
-    endpoints::EndpointServerPools,
-    layout::endpoint::{Endpoint, EndpointType},
-};
-
-const ENDPOINT_TYPE_LABEL: &str = "endpoint_type";
-const LOCAL_ENDPOINT_LABEL: &str = "local";
-const LOCAL_NODE_ID: &str = "local";
-const STORAGE_MEDIA_NOT_REPORTED: &str = "storage media not reported by endpoints";
-const FAILURE_DOMAIN_NOT_REPORTED: &str = "failure domain labels not reported by endpoints";
-const NUMA_NOT_WIRED: &str = "NUMA topology not wired into runtime";
-const PROFILING_NOT_WIRED: &str = "profiling capability not wired into ECStore";
-const PEER_HEALTH_NOT_REPORTED: &str = "peer health not reported by endpoints";
-
-#[derive(Debug, Clone)]
-pub struct ClusterControlPlane {
-    endpoint_pools: EndpointServerPools,
-}
-
-impl ClusterControlPlane {
-    pub fn new(endpoint_pools: EndpointServerPools) -> Self {
-        Self { endpoint_pools }
-    }
-
-    pub fn topology_snapshot(&self) -> TopologySnapshot {
-        topology_snapshot_from_endpoint_pools(&self.endpoint_pools)
-    }
-
-    pub fn membership_snapshot(&self) -> ClusterMembershipSnapshot {
-        membership_snapshot_from_endpoint_pools(&self.endpoint_pools)
-    }
-
-    pub fn pool_state_snapshot(&self) -> ClusterPoolStateSnapshot {
-        pool_state_snapshot_from_endpoint_pools(&self.endpoint_pools)
-    }
-
-    pub fn local_node_storage_snapshot(&self) -> ClusterLocalNodeStorageSnapshot {
-        let membership = self.membership_snapshot();
-        local_node_storage_snapshot_from_membership(&membership)
-    }
-
-    pub fn peer_health_snapshot(&self) -> ClusterPeerHealthSnapshot {
-        let membership = self.membership_snapshot();
-        peer_health_snapshot_from_membership(&membership)
-    }
-
-    pub fn read_snapshot(&self) -> ClusterControlPlaneSnapshot {
-        let membership = self.membership_snapshot();
-
-        ClusterControlPlaneSnapshot {
-            topology: self.topology_snapshot(),
-            pool_state: self.pool_state_snapshot(),
-            local_storage: local_node_storage_snapshot_from_membership(&membership),
-            peer_health: peer_health_snapshot_from_membership(&membership),
-            membership,
-        }
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ClusterControlPlaneSnapshot {
-    pub topology: TopologySnapshot,
-    pub pool_state: ClusterPoolStateSnapshot,
-    pub local_storage: ClusterLocalNodeStorageSnapshot,
-    pub peer_health: ClusterPeerHealthSnapshot,
-    pub membership: ClusterMembershipSnapshot,
-}
-
-#[derive(Debug, Clone, Default, PartialEq, Eq)]
-pub struct ClusterMembershipSnapshot {
-    pub nodes: Vec<ClusterNodeMembership>,
-    pub drives: Vec<ClusterDriveMembership>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ClusterNodeMembership {
-    pub node_id: String,
-    pub grid_host: String,
-    pub is_local: bool,
-    pub pools: Vec<usize>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ClusterDriveMembership {
-    pub pool_index: usize,
-    pub set_index: usize,
-    pub disk_index: usize,
-    pub node_id: String,
-    pub is_local: bool,
-    pub endpoint_type: ClusterEndpointType,
-}
-
-#[derive(Debug, Clone, Default, PartialEq, Eq)]
-pub struct ClusterPoolStateSnapshot {
-    pub pools: Vec<ClusterPoolState>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ClusterPoolState {
-    pub pool_index: usize,
-    pub set_count: usize,
-    pub drives_per_set: usize,
-    pub endpoint_count: usize,
-    pub local_drive_count: usize,
-    pub remote_drive_count: usize,
-    pub legacy: bool,
-    pub endpoint_types: Vec<ClusterEndpointType>,
-}
-
-#[derive(Debug, Clone, Default, PartialEq, Eq)]
-pub struct ClusterLocalNodeStorageSnapshot {
-    pub nodes: Vec<ClusterLocalNodeStorage>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ClusterLocalNodeStorage {
-    pub node_id: String,
-    pub pools: Vec<usize>,
-    pub drive_count: usize,
-    pub path_drive_count: usize,
-    pub url_drive_count: usize,
-}
-
-#[derive(Debug, Clone, Default, PartialEq, Eq)]
-pub struct ClusterPeerHealthSnapshot {
-    pub peers: Vec<ClusterPeerHealth>,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ClusterPeerHealth {
-    pub node_id: String,
-    pub is_local: bool,
-    pub status: CapabilityStatus,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub enum ClusterEndpointType {
-    Path,
-    Url,
-}
-
-pub fn topology_snapshot_from_endpoint_pools(endpoint_pools: &EndpointServerPools) -> TopologySnapshot {
-    topology_snapshot_from_endpoint_pools_with_capabilities(
-        endpoint_pools,
-        default_topology_capabilities(),
-        default_disk_capabilities(),
-    )
-}
-
-pub fn topology_snapshot_from_endpoint_pools_with_capabilities(
-    endpoint_pools: &EndpointServerPools,
-    capabilities: TopologyCapabilities,
-    disk_capabilities: DiskCapabilities,
-) -> TopologySnapshot {
-    TopologySnapshot {
-        pools: endpoint_pools
-            .as_ref()
-            .iter()
-            .enumerate()
-            .map(|(pool_index, pool)| {
-                let sets =
-                    topology_sets_from_endpoints(pool_index, pool.drives_per_set, pool.endpoints.as_ref(), &disk_capabilities);
-                TopologyPool {
-                    pool_index,
-                    pool_id: None,
-                    labels: TopologyLabels::default(),
-                    sets,
-                }
-            })
-            .collect(),
-        capabilities,
-    }
-}
-
-pub fn membership_snapshot_from_endpoint_pools(endpoint_pools: &EndpointServerPools) -> ClusterMembershipSnapshot {
-    let mut nodes = BTreeMap::<String, ClusterNodeMembership>::new();
-    let mut drives = Vec::new();
-
-    for (pool_index, pool) in endpoint_pools.as_ref().iter().enumerate() {
-        for (endpoint_index, endpoint) in pool.endpoints.as_ref().iter().enumerate() {
-            let (set_index, disk_index) = endpoint_indices(pool_index, endpoint_index, pool.drives_per_set, endpoint);
-            let node_id = endpoint_node_id(endpoint);
-
-            match nodes.entry(node_id.clone()) {
-                std::collections::btree_map::Entry::Vacant(entry) => {
-                    entry.insert(ClusterNodeMembership {
-                        node_id: node_id.clone(),
-                        grid_host: endpoint.grid_host(),
-                        is_local: endpoint.is_local,
-                        pools: vec![pool_index],
-                    });
-                }
-                std::collections::btree_map::Entry::Occupied(mut entry) => {
-                    let node = entry.get_mut();
-                    node.is_local |= endpoint.is_local;
-                    if !node.pools.contains(&pool_index) {
-                        node.pools.push(pool_index);
-                    }
-                }
-            }
-
-            drives.push(ClusterDriveMembership {
-                pool_index,
-                set_index,
-                disk_index,
-                node_id,
-                is_local: endpoint.is_local,
-                endpoint_type: endpoint_type(endpoint),
-            });
-        }
-    }
-
-    ClusterMembershipSnapshot {
-        nodes: nodes.into_values().collect(),
-        drives,
-    }
-}
-
-pub fn pool_state_snapshot_from_endpoint_pools(endpoint_pools: &EndpointServerPools) -> ClusterPoolStateSnapshot {
-    ClusterPoolStateSnapshot {
-        pools: endpoint_pools
-            .as_ref()
-            .iter()
-            .enumerate()
-            .map(|(pool_index, pool)| {
-                let mut endpoint_types = BTreeSet::new();
-                let mut local_drive_count = 0;
-
-                for endpoint in pool.endpoints.as_ref() {
-                    endpoint_types.insert(endpoint_type(endpoint));
-                    if endpoint.is_local {
-                        local_drive_count += 1;
-                    }
-                }
-
-                let endpoint_count = pool.endpoints.as_ref().len();
-                ClusterPoolState {
-                    pool_index,
-                    set_count: pool.set_count,
-                    drives_per_set: pool.drives_per_set,
-                    endpoint_count,
-                    local_drive_count,
-                    remote_drive_count: endpoint_count.saturating_sub(local_drive_count),
-                    legacy: pool.legacy,
-                    endpoint_types: endpoint_types.into_iter().collect(),
-                }
-            })
-            .collect(),
-    }
-}
-
-pub fn local_node_storage_snapshot_from_membership(membership: &ClusterMembershipSnapshot) -> ClusterLocalNodeStorageSnapshot {
-    ClusterLocalNodeStorageSnapshot {
-        nodes: membership
-            .nodes
-            .iter()
-            .filter(|node| node.is_local)
-            .map(|node| {
-                let mut path_drive_count = 0;
-                let mut url_drive_count = 0;
-
-                for drive in membership.drives.iter().filter(|drive| drive.node_id == node.node_id) {
-                    match drive.endpoint_type {
-                        ClusterEndpointType::Path => path_drive_count += 1,
-                        ClusterEndpointType::Url => url_drive_count += 1,
-                    }
-                }
-
-                ClusterLocalNodeStorage {
-                    node_id: node.node_id.clone(),
-                    pools: node.pools.clone(),
-                    drive_count: path_drive_count + url_drive_count,
-                    path_drive_count,
-                    url_drive_count,
-                }
-            })
-            .collect(),
-    }
-}
-
-pub fn peer_health_snapshot_from_membership(membership: &ClusterMembershipSnapshot) -> ClusterPeerHealthSnapshot {
-    ClusterPeerHealthSnapshot {
-        peers: membership
-            .nodes
-            .iter()
-            .map(|node| ClusterPeerHealth {
-                node_id: node.node_id.clone(),
-                is_local: node.is_local,
-                status: CapabilityStatus::unknown().with_reason(PEER_HEALTH_NOT_REPORTED),
-            })
-            .collect(),
-    }
-}
-
-fn topology_sets_from_endpoints(
-    pool_index: usize,
-    drives_per_set: usize,
-    endpoints: &[Endpoint],
-    disk_capabilities: &DiskCapabilities,
-) -> Vec<TopologySet> {
-    let mut sets = BTreeMap::<usize, Vec<TopologyDisk>>::new();
-
-    for (endpoint_index, endpoint) in endpoints.iter().enumerate() {
-        let (set_index, disk_index) = endpoint_indices(pool_index, endpoint_index, drives_per_set, endpoint);
-        sets.entry(set_index).or_default().push(topology_disk_from_endpoint(
-            pool_index,
-            set_index,
-            disk_index,
-            endpoint,
-            disk_capabilities,
-        ));
-    }
-
-    sets.into_iter()
-        .map(|(set_index, mut disks)| {
-            disks.sort_by_key(|disk| disk.disk_index);
-            TopologySet {
-                pool_index,
-                set_index,
-                set_id: None,
-                labels: TopologyLabels::default(),
-                disks,
-            }
-        })
-        .collect()
-}
-
-fn topology_disk_from_endpoint(
-    pool_index: usize,
-    set_index: usize,
-    disk_index: usize,
-    endpoint: &Endpoint,
-    disk_capabilities: &DiskCapabilities,
-) -> TopologyDisk {
-    TopologyDisk {
-        pool_index,
-        set_index,
-        disk_index,
-        disk_id: endpoint_disk_id(endpoint),
-        labels: endpoint_labels(endpoint),
-        capabilities: disk_capabilities.clone(),
-    }
-}
-
-fn endpoint_indices(pool_index: usize, endpoint_index: usize, drives_per_set: usize, endpoint: &Endpoint) -> (usize, usize) {
-    let safe_drives_per_set = drives_per_set.max(1);
-    let set_index = non_negative_index(endpoint.set_idx).unwrap_or(endpoint_index / safe_drives_per_set);
-    let disk_index = non_negative_index(endpoint.disk_idx).unwrap_or(endpoint_index % safe_drives_per_set);
-
-    debug_assert_eq!(non_negative_index(endpoint.pool_idx).unwrap_or(pool_index), pool_index);
-    (set_index, disk_index)
-}
-
-fn endpoint_disk_id(endpoint: &Endpoint) -> Option<String> {
-    let host_port = endpoint.host_port();
-    if host_port.is_empty() { None } else { Some(host_port) }
-}
-
-fn endpoint_node_id(endpoint: &Endpoint) -> String {
-    endpoint_disk_id(endpoint).unwrap_or_else(|| LOCAL_NODE_ID.to_owned())
-}
-
-fn endpoint_labels(endpoint: &Endpoint) -> TopologyLabels {
-    let mut additional = BTreeMap::new();
-    additional.insert(ENDPOINT_TYPE_LABEL.to_owned(), endpoint_type_label(endpoint).to_owned());
-    additional.insert(LOCAL_ENDPOINT_LABEL.to_owned(), endpoint.is_local.to_string());
-
-    TopologyLabels {
-        node: Some(endpoint_node_id(endpoint)),
-        additional,
-        ..TopologyLabels::default()
-    }
-}
-
-fn endpoint_type(endpoint: &Endpoint) -> ClusterEndpointType {
-    match endpoint.get_type() {
-        EndpointType::Path => ClusterEndpointType::Path,
-        EndpointType::Url => ClusterEndpointType::Url,
-    }
-}
-
-fn endpoint_type_label(endpoint: &Endpoint) -> &'static str {
-    match endpoint_type(endpoint) {
-        ClusterEndpointType::Path => "path",
-        ClusterEndpointType::Url => "url",
-    }
-}
-
-fn non_negative_index(index: i32) -> Option<usize> {
-    usize::try_from(index).ok()
-}
-
-fn default_topology_capabilities() -> TopologyCapabilities {
-    TopologyCapabilities {
-        profiling: CapabilityStatus::unknown().with_reason(PROFILING_NOT_WIRED),
-        numa: CapabilityStatus::unsupported().with_reason(NUMA_NOT_WIRED),
-        failure_domain_labels: CapabilityStatus::unknown().with_reason(FAILURE_DOMAIN_NOT_REPORTED),
-        media_labels: CapabilityStatus::unknown().with_reason(STORAGE_MEDIA_NOT_REPORTED),
-    }
-}
-
-fn default_disk_capabilities() -> DiskCapabilities {
-    DiskCapabilities {
-        media_type: CapabilityStatus::unknown().with_reason(STORAGE_MEDIA_NOT_REPORTED),
-        failure_domain: CapabilityStatus::unknown().with_reason(FAILURE_DOMAIN_NOT_REPORTED),
-        numa: CapabilityStatus::unsupported().with_reason(NUMA_NOT_WIRED),
-        profiling: CapabilityStatus::unknown().with_reason(PROFILING_NOT_WIRED),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::collections::BTreeSet;
-
-    use crate::endpoints::{Endpoints, PoolEndpoints};
-
-    #[test]
-    fn topology_snapshot_maps_endpoint_sets_without_local_paths() {
-        let endpoint_pools = sample_path_endpoint_pools();
-        let snapshot = topology_snapshot_from_endpoint_pools(&endpoint_pools);
-
-        assert_eq!(snapshot.pools.len(), 1);
-        assert_eq!(snapshot.pools[0].sets.len(), 2);
-        assert_eq!(snapshot.pools[0].sets[0].disks.len(), 2);
-        assert_eq!(snapshot.pools[0].sets[1].disks.len(), 2);
-        assert_eq!(snapshot.pools[0].sets[1].disks[1].disk_index, 1);
-        assert_eq!(
-            snapshot.pools[0].sets[0].disks[0]
-                .labels
-                .additional
-                .get(ENDPOINT_TYPE_LABEL)
-                .map(String::as_str),
-            Some("path")
-        );
-        assert_eq!(snapshot.pools[0].sets[0].disks[0].labels.node.as_deref(), Some(LOCAL_NODE_ID));
-
-        let encoded = serde_json::to_string(&snapshot).expect("serialize topology snapshot");
-        assert!(!encoded.contains("/tmp/rustfs-cluster-control-plane"));
-    }
-
-    #[test]
-    fn topology_snapshot_uses_url_hosts_as_disk_ids() {
-        let endpoint_pools = sample_url_endpoint_pools();
-        let snapshot = topology_snapshot_from_endpoint_pools(&endpoint_pools);
-
-        assert_eq!(snapshot.pools[0].sets[0].disks[0].disk_id.as_deref(), Some("node1.example:9000"));
-        assert_eq!(snapshot.pools[0].sets[0].disks[0].labels.node.as_deref(), Some("node1.example:9000"));
-        assert_eq!(
-            snapshot.pools[0].sets[0].disks[0]
-                .labels
-                .additional
-                .get(ENDPOINT_TYPE_LABEL)
-                .map(String::as_str),
-            Some("url")
-        );
-    }
-
-    #[test]
-    fn membership_snapshot_groups_nodes_and_drives() {
-        let endpoint_pools = sample_url_endpoint_pools();
-        let snapshot = membership_snapshot_from_endpoint_pools(&endpoint_pools);
-
-        assert_eq!(snapshot.nodes.len(), 2);
-        assert_eq!(snapshot.drives.len(), 4);
-        assert_eq!(snapshot.nodes[0].node_id, "node1.example:9000");
-        assert_eq!(snapshot.nodes[0].pools, vec![0]);
-        assert_eq!(snapshot.drives[2].node_id, "node2.example:9000");
-        assert_eq!(snapshot.drives[2].set_index, 1);
-        assert_eq!(snapshot.drives[2].endpoint_type, ClusterEndpointType::Url);
-    }
-
-    #[test]
-    fn pool_state_snapshot_counts_local_remote_drives_and_endpoint_types() {
-        let endpoint_pools = sample_mixed_endpoint_pools();
-        let snapshot = pool_state_snapshot_from_endpoint_pools(&endpoint_pools);
-
-        assert_eq!(snapshot.pools.len(), 1);
-        assert_eq!(snapshot.pools[0].set_count, 2);
-        assert_eq!(snapshot.pools[0].drives_per_set, 2);
-        assert_eq!(snapshot.pools[0].endpoint_count, 4);
-        assert_eq!(snapshot.pools[0].local_drive_count, 3);
-        assert_eq!(snapshot.pools[0].remote_drive_count, 1);
-        assert_eq!(
-            snapshot.pools[0].endpoint_types,
-            vec![ClusterEndpointType::Path, ClusterEndpointType::Url]
-        );
-    }
-
-    #[test]
-    fn local_node_storage_snapshot_keeps_only_local_drive_counts() {
-        let membership = membership_snapshot_from_endpoint_pools(&sample_mixed_endpoint_pools());
-        let snapshot = local_node_storage_snapshot_from_membership(&membership);
-
-        assert_eq!(snapshot.nodes.len(), 2);
-        assert_eq!(snapshot.nodes[0].node_id, LOCAL_NODE_ID);
-        assert_eq!(snapshot.nodes[0].drive_count, 2);
-        assert_eq!(snapshot.nodes[0].path_drive_count, 2);
-        assert_eq!(snapshot.nodes[0].url_drive_count, 0);
-        assert_eq!(snapshot.nodes[1].node_id, "node1.example:9000");
-        assert_eq!(snapshot.nodes[1].drive_count, 1);
-        assert_eq!(snapshot.nodes[1].path_drive_count, 0);
-        assert_eq!(snapshot.nodes[1].url_drive_count, 1);
-    }
-
-    #[test]
-    fn peer_health_snapshot_reports_static_unknown_status() {
-        let membership = membership_snapshot_from_endpoint_pools(&sample_url_endpoint_pools());
-        let snapshot = peer_health_snapshot_from_membership(&membership);
-
-        assert_eq!(snapshot.peers.len(), 2);
-        assert_eq!(snapshot.peers[0].node_id, "node1.example:9000");
-        assert!(snapshot.peers[0].is_local);
-        assert_eq!(snapshot.peers[0].status.reason.as_deref(), Some(PEER_HEALTH_NOT_REPORTED));
-        assert_eq!(snapshot.peers[1].node_id, "node2.example:9000");
-        assert!(!snapshot.peers[1].is_local);
-    }
-
-    #[test]
-    fn control_plane_read_snapshot_combines_topology_and_membership() {
-        let control_plane = ClusterControlPlane::new(sample_mixed_endpoint_pools());
-        let snapshot = control_plane.read_snapshot();
-        let node_ids = snapshot
-            .membership
-            .nodes
-            .iter()
-            .map(|node| node.node_id.as_str())
-            .collect::<BTreeSet<_>>();
-
-        assert_eq!(snapshot.topology.pools[0].sets.len(), 2);
-        assert_eq!(snapshot.pool_state.pools[0].endpoint_count, 4);
-        assert_eq!(snapshot.local_storage.nodes.len(), 2);
-        assert_eq!(snapshot.peer_health.peers.len(), 3);
-        assert_eq!(node_ids, BTreeSet::from([LOCAL_NODE_ID, "node1.example:9000", "node2.example:9000"]));
-    }
-
-    fn sample_path_endpoint_pools() -> EndpointServerPools {
-        let endpoints = (0..4)
-            .map(|index| {
-                let mut endpoint =
-                    Endpoint::try_from(format!("/tmp/rustfs-cluster-control-plane-{index}").as_str()).expect("local endpoint");
-                endpoint.set_pool_index(0);
-                endpoint.set_set_index(index / 2);
-                endpoint.set_disk_index(index % 2);
-                endpoint
-            })
-            .collect::<Vec<_>>();
-
-        EndpointServerPools::from(vec![PoolEndpoints {
-            legacy: false,
-            set_count: 2,
-            drives_per_set: 2,
-            endpoints: Endpoints::from(endpoints),
-            cmd_line: "/tmp/rustfs-cluster-control-plane-{0...3}".to_owned(),
-            platform: "OS: test | Arch: test".to_owned(),
-        }])
-    }
-
-    fn sample_url_endpoint_pools() -> EndpointServerPools {
-        let endpoints = (0..4)
-            .map(|index| {
-                let host = if index < 2 { "node1.example" } else { "node2.example" };
-                let mut endpoint =
-                    Endpoint::try_from(format!("http://{host}:9000/export{index}").as_str()).expect("url endpoint");
-                endpoint.set_pool_index(0);
-                endpoint.set_set_index(index / 2);
-                endpoint.set_disk_index(index % 2);
-                endpoint.is_local = index < 2;
-                endpoint
-            })
-            .collect::<Vec<_>>();
-
-        EndpointServerPools::from(vec![PoolEndpoints {
-            legacy: false,
-            set_count: 2,
-            drives_per_set: 2,
-            endpoints: Endpoints::from(endpoints),
-            cmd_line: "http://node{1...2}.example:9000/export{0...3}".to_owned(),
-            platform: "OS: test | Arch: test".to_owned(),
-        }])
-    }
-
-    fn sample_mixed_endpoint_pools() -> EndpointServerPools {
-        let mut endpoints = Vec::new();
-
-        for index in 0..2 {
-            let mut endpoint =
-                Endpoint::try_from(format!("/tmp/rustfs-cluster-control-plane-{index}").as_str()).expect("local endpoint");
-            endpoint.set_pool_index(0);
-            endpoint.set_set_index(0);
-            endpoint.set_disk_index(index);
-            endpoints.push(endpoint);
-        }
-
-        for index in 2..4 {
-            let host = if index == 2 { "node1.example" } else { "node2.example" };
-            let mut endpoint = Endpoint::try_from(format!("http://{host}:9000/export{index}").as_str()).expect("url endpoint");
-            endpoint.set_pool_index(0);
-            endpoint.set_set_index(1);
-            endpoint.set_disk_index(index - 2);
-            endpoint.is_local = index == 2;
-            endpoints.push(endpoint);
-        }
-
-        EndpointServerPools::from(vec![PoolEndpoints {
-            legacy: false,
-            set_count: 2,
-            drives_per_set: 2,
-            endpoints: Endpoints::from(endpoints),
-            cmd_line: "mixed-test-endpoints".to_owned(),
-            platform: "OS: test | Arch: test".to_owned(),
-        }])
-    }
-}

crates/ecstore/src/core/sets.rs

@@ -1090,9 +1090,36 @@ async fn init_storage_disks_with_errors(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::endpoints::SetupType;
     use crate::layout::endpoint::Endpoint;
     use crate::storage_api_contracts::heal::HealOperations as _;
     use crate::storage_api_contracts::list::ListOperations as _;
+    use rustfs_lock::client::local::LocalClient;
+    use serial_test::serial;
+
+    struct SetupTypeGuard {
+        previous: SetupType,
+    }
+
+    impl SetupTypeGuard {
+        async fn switch_to(next: SetupType) -> Self {
+            let previous = runtime_sources::current_setup_type().await;
+            runtime_sources::set_setup_type(next).await;
+            Self { previous }
+        }
+    }
+
+    impl Drop for SetupTypeGuard {
+        fn drop(&mut self) {
+            let previous = self.previous.clone();
+            let handle = tokio::runtime::Handle::current();
+            tokio::task::block_in_place(|| {
+                handle.block_on(async move {
+                    runtime_sources::set_setup_type(previous).await;
+                });
+            });
+        }
+    }
 
     #[test]
     fn test_apply_delete_objects_results_preserves_original_order_for_out_of_order_batches() {
@@ -1203,8 +1230,10 @@ mod tests {
         assert_eq!(result, (Some(3), Some(1), Some(0)));
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
+    #[serial]
     async fn sets_list_objects_v2_lists_objects_within_the_pool() {
+        let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await;
         let format = FormatV3::new(1, 2);
         let mut endpoints = Vec::new();
         let mut disks = Vec::new();
@@ -1247,7 +1276,7 @@ mod tests {
             0,
             endpoints.clone(),
             format.clone(),
-            Vec::new(),
+            vec![Arc::new(LocalClient::new()), Arc::new(LocalClient::new())],
         )
         .await;
 
@@ -1272,23 +1301,26 @@ mod tests {
             exit_signal: None,
         });
 
-        sets.make_bucket("bucket", &MakeBucketOptions::default())
+        let bucket = format!("bucket-{}", Uuid::new_v4().simple());
+        let object = format!("object-{}", Uuid::new_v4().simple());
+
+        sets.make_bucket(&bucket, &MakeBucketOptions::default())
             .await
             .expect("bucket should be created");
 
         let mut reader = PutObjReader::from_vec(b"hello".to_vec());
-        sets.put_object("bucket", "object", &mut reader, &ObjectOptions::default())
+        sets.put_object(&bucket, &object, &mut reader, &ObjectOptions::default())
             .await
             .expect("object should be written");
 
         let result = sets
             .clone()
-            .list_objects_v2("bucket", "", None, None, 1000, false, None, false)
+            .list_objects_v2(&bucket, "", None, None, 1000, false, None, false)
             .await
             .expect("pool-level listing should succeed");
 
         assert_eq!(result.objects.len(), 1);
-        assert_eq!(result.objects[0].name, "object");
+        assert_eq!(result.objects[0].name, object);
     }
 
     #[tokio::test]

crates/ecstore/src/lib.rs

@@ -15,36 +15,51 @@
 
 extern crate core;
 
+#[path = "diagnostics/admin_server_info.rs"]
 mod admin_server_info;
 pub mod api;
+#[path = "services/batch_processor.rs"]
 mod batch_processor;
+#[path = "io_support/bitrot.rs"]
 mod bitrot;
 mod bucket;
 mod cache_value;
 mod cluster;
+#[path = "io_support/compress.rs"]
 mod compress;
 mod config;
 mod data_movement;
+#[path = "data_movement/backpressure.rs"]
 mod data_movement_backpressure;
 mod data_usage;
 mod disk;
+#[path = "layout/disks_layout_facade.rs"]
 mod disks_layout;
+#[path = "layout/endpoints_facade.rs"]
 mod endpoints;
 mod erasure_codec;
 mod erasure_coding;
 mod error;
+#[path = "diagnostics/get.rs"]
 mod get_diagnostics;
+#[path = "runtime/global.rs"]
 mod global;
 pub(crate) mod layout;
+#[path = "services/metrics_realtime.rs"]
 mod metrics_realtime;
+#[path = "services/notification_sys.rs"]
 mod notification_sys;
 mod object_api;
+#[path = "core/pools.rs"]
 mod pools;
 mod rebalance;
+#[path = "io_support/rio.rs"]
 mod rio;
 mod rpc;
+#[path = "runtime/sources.rs"]
 mod runtime_sources;
 mod set_disk;
+#[path = "core/sets.rs"]
 mod sets;
 mod storage_api_contracts;
 mod store;
@@ -58,10 +73,13 @@ mod store_utils;
 // pub mod checksum;
 mod client;
 mod event;
+#[path = "services/event_notification.rs"]
 mod event_notification;
 #[cfg(test)]
+#[path = "core/pools_test.rs"]
 mod pools_test;
 #[cfg(test)]
+#[path = "core/store_test.rs"]
 mod store_test;
 mod tier;