mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 11:56:38 +00:00
feat(admin): expose cluster snapshot view (#3803)
This commit is contained in:
@@ -13,6 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::admin::handlers::health::{HealthProbe, build_health_response_parts, collect_dependency_readiness};
|
use crate::admin::handlers::health::{HealthProbe, build_health_response_parts, collect_dependency_readiness};
|
||||||
|
use crate::app::admin_usecase::DefaultAdminUsecase;
|
||||||
use crate::app::context::resolve_oidc_handle;
|
use crate::app::context::resolve_oidc_handle;
|
||||||
use crate::license::has_valid_license;
|
use crate::license::has_valid_license;
|
||||||
use crate::server::has_path_prefix;
|
use crate::server::has_path_prefix;
|
||||||
@@ -151,6 +152,7 @@ impl Config {
|
|||||||
port,
|
port,
|
||||||
api: Api {
|
api: Api {
|
||||||
base_url: build_console_api_base_url(&format!("{http_prefix}{local_ip}:{port}")),
|
base_url: build_console_api_base_url(&format!("{http_prefix}{local_ip}:{port}")),
|
||||||
|
discovery: console_api_discovery(),
|
||||||
},
|
},
|
||||||
s3: S3 {
|
s3: S3 {
|
||||||
endpoint: format!("{http_prefix}{local_ip}:{port}"),
|
endpoint: format!("{http_prefix}{local_ip}:{port}"),
|
||||||
@@ -208,6 +210,26 @@ fn build_console_api_base_url(base_url: &str) -> String {
|
|||||||
struct Api {
|
struct Api {
|
||||||
#[serde(rename = "baseURL")]
|
#[serde(rename = "baseURL")]
|
||||||
base_url: String,
|
base_url: String,
|
||||||
|
discovery: ApiDiscovery,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Clone)]
|
||||||
|
struct ApiDiscovery {
|
||||||
|
#[serde(rename = "runtimeCapabilities")]
|
||||||
|
runtime_capabilities: String,
|
||||||
|
#[serde(rename = "clusterSnapshot")]
|
||||||
|
cluster_snapshot: String,
|
||||||
|
#[serde(rename = "extensionsCatalog")]
|
||||||
|
extensions_catalog: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn console_api_discovery() -> ApiDiscovery {
|
||||||
|
let usecase = DefaultAdminUsecase::from_global();
|
||||||
|
ApiDiscovery {
|
||||||
|
runtime_capabilities: usecase.runtime_capabilities_route().to_string(),
|
||||||
|
cluster_snapshot: usecase.cluster_snapshot_route().to_string(),
|
||||||
|
extensions_catalog: usecase.extensions_catalog_route().to_string(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Clone)]
|
#[derive(Debug, Serialize, Clone)]
|
||||||
@@ -791,6 +813,33 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn console_config_exposes_admin_discovery_paths() {
|
||||||
|
let cfg = Config::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001, "test", "2026-03-16T00:00:00Z");
|
||||||
|
|
||||||
|
assert_eq!(cfg.api.discovery.runtime_capabilities, "/rustfs/admin/v4/runtime/capabilities");
|
||||||
|
assert_eq!(cfg.api.discovery.cluster_snapshot, "/rustfs/admin/v4/cluster/snapshot");
|
||||||
|
assert_eq!(cfg.api.discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn console_config_handler_serializes_admin_discovery_paths() {
|
||||||
|
init_console_cfg(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001);
|
||||||
|
|
||||||
|
let response = config_handler(Uri::from_static("http://127.0.0.1:9001/rustfs/console/api/v1/config"), HeaderMap::new())
|
||||||
|
.await
|
||||||
|
.into_response();
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let body = response.into_body();
|
||||||
|
let bytes = body.collect().await.expect("collect console config body").to_bytes();
|
||||||
|
let value: serde_json::Value = serde_json::from_slice(&bytes).expect("console config JSON should deserialize");
|
||||||
|
|
||||||
|
assert_eq!(value["api"]["discovery"]["runtimeCapabilities"], "/rustfs/admin/v4/runtime/capabilities");
|
||||||
|
assert_eq!(value["api"]["discovery"]["clusterSnapshot"], "/rustfs/admin/v4/cluster/snapshot");
|
||||||
|
assert_eq!(value["api"]["discovery"]["extensionsCatalog"], "/rustfs/admin/v4/extensions/catalog");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn external_admin_paths_are_not_console_paths() {
|
fn external_admin_paths_are_not_console_paths() {
|
||||||
assert!(is_console_path("/rustfs/console/"));
|
assert!(is_console_path("/rustfs/console/"));
|
||||||
|
|||||||
@@ -0,0 +1,756 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use crate::admin::{
|
||||||
|
auth::validate_admin_request,
|
||||||
|
router::{AdminOperation, Operation, S3Router},
|
||||||
|
system,
|
||||||
|
};
|
||||||
|
use crate::app::admin_usecase::DefaultAdminUsecase;
|
||||||
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
|
use crate::cluster_snapshot::{
|
||||||
|
ClusterReadOnlySnapshot, ClusterRuntimeReadinessState, ClusterRuntimeStatusSnapshot, cluster_has_actionable_pressure,
|
||||||
|
};
|
||||||
|
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||||
|
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||||
|
use hyper::Method;
|
||||||
|
use matchit::Params;
|
||||||
|
use rustfs_concurrency::AdmissionState as WorkloadAdmissionState;
|
||||||
|
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadClass};
|
||||||
|
use rustfs_ecstore::api::cluster::{
|
||||||
|
ClusterDriveMembership, ClusterEndpointType, ClusterLocalNodeStorage, ClusterLocalNodeStorageSnapshot,
|
||||||
|
ClusterMembershipSnapshot, ClusterNodeMembership, ClusterPeerHealth, ClusterPeerHealthSnapshot, ClusterPoolState,
|
||||||
|
ClusterPoolStateSnapshot,
|
||||||
|
};
|
||||||
|
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||||
|
use rustfs_storage_api::{CapabilityState, CapabilityStatus, ObservabilitySnapshot, TopologySnapshot};
|
||||||
|
use s3s::header::CONTENT_TYPE;
|
||||||
|
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
pub fn register_cluster_snapshot_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
|
||||||
|
r.insert(
|
||||||
|
Method::GET,
|
||||||
|
format!("{}{}", ADMIN_PREFIX, "/v4/cluster/snapshot").as_str(),
|
||||||
|
AdminOperation(&GetClusterSnapshotHandler {}),
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterSnapshotResponse {
|
||||||
|
pub snapshot: Option<ClusterSnapshotView>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterSnapshotDiscoveryResponse {
|
||||||
|
pub path: String,
|
||||||
|
pub summary: Option<CapabilityStatus>,
|
||||||
|
pub topology: Option<CapabilityStatus>,
|
||||||
|
pub peer_health: Option<CapabilityStatus>,
|
||||||
|
pub workload_admission: Option<CapabilityStatus>,
|
||||||
|
pub runtime: Option<CapabilityStatus>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn authorize_cluster_snapshot_request(req: &S3Request<Body>) -> S3Result<()> {
|
||||||
|
let Some(input_cred) = &req.credentials else {
|
||||||
|
return Err(s3_error!(InvalidRequest, "authentication required"));
|
||||||
|
};
|
||||||
|
|
||||||
|
let (cred, owner) =
|
||||||
|
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
|
||||||
|
|
||||||
|
validate_admin_request(
|
||||||
|
&req.headers,
|
||||||
|
&cred,
|
||||||
|
owner,
|
||||||
|
false,
|
||||||
|
vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)],
|
||||||
|
req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_json_response(
|
||||||
|
status: StatusCode,
|
||||||
|
body: &impl Serialize,
|
||||||
|
request_id: Option<&HeaderValue>,
|
||||||
|
) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||||
|
let data = serde_json::to_vec(body).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
|
||||||
|
let mut header = HeaderMap::new();
|
||||||
|
header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||||
|
if let Some(value) = request_id {
|
||||||
|
header.insert("x-request-id", value.clone());
|
||||||
|
}
|
||||||
|
Ok(S3Response::with_headers((status, Body::from(data)), header))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct GetClusterSnapshotHandler {}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl Operation for GetClusterSnapshotHandler {
|
||||||
|
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||||
|
authorize_cluster_snapshot_request(&req).await?;
|
||||||
|
let snapshot = DefaultAdminUsecase::from_global()
|
||||||
|
.execute_collect_cluster_read_only_snapshot()
|
||||||
|
.await
|
||||||
|
.map(ClusterSnapshotView::from);
|
||||||
|
build_json_response(StatusCode::OK, &ClusterSnapshotResponse { snapshot }, req.headers.get("x-request-id"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn build_cluster_snapshot_discovery_response() -> ClusterSnapshotDiscoveryResponse {
|
||||||
|
let usecase = DefaultAdminUsecase::from_global();
|
||||||
|
let path = usecase.cluster_snapshot_route().to_string();
|
||||||
|
let snapshot = usecase.execute_collect_cluster_read_only_snapshot().await;
|
||||||
|
|
||||||
|
match snapshot {
|
||||||
|
Some(snapshot) => {
|
||||||
|
let summary = ClusterSnapshotSummary::from(&snapshot);
|
||||||
|
ClusterSnapshotDiscoveryResponse {
|
||||||
|
path,
|
||||||
|
summary: Some(summary.actionable_pressure.clone()),
|
||||||
|
topology: Some(summary.topology),
|
||||||
|
peer_health: Some(summary.peer_health),
|
||||||
|
workload_admission: Some(summary.workload_admission),
|
||||||
|
runtime: Some(summary.runtime),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => ClusterSnapshotDiscoveryResponse {
|
||||||
|
path,
|
||||||
|
summary: None,
|
||||||
|
topology: None,
|
||||||
|
peer_health: None,
|
||||||
|
workload_admission: None,
|
||||||
|
runtime: None,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterSnapshotView {
|
||||||
|
pub summary: ClusterSnapshotSummary,
|
||||||
|
pub runtime_capabilities_path: String,
|
||||||
|
pub extensions_catalog_path: String,
|
||||||
|
pub topology: TopologySnapshot,
|
||||||
|
pub membership: ClusterMembershipView,
|
||||||
|
pub pool_state: ClusterPoolStateView,
|
||||||
|
pub local_storage: ClusterLocalStorageView,
|
||||||
|
pub peer_health: ClusterPeerHealthView,
|
||||||
|
pub observability: ObservabilitySnapshot,
|
||||||
|
pub workload_admission: Vec<WorkloadAdmissionView>,
|
||||||
|
pub runtime_status: ClusterRuntimeStatusView,
|
||||||
|
pub actionable_pressure: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ClusterReadOnlySnapshot> for ClusterSnapshotView {
|
||||||
|
fn from(snapshot: ClusterReadOnlySnapshot) -> Self {
|
||||||
|
let actionable_pressure = cluster_has_actionable_pressure(&snapshot);
|
||||||
|
Self {
|
||||||
|
summary: ClusterSnapshotSummary::from(&snapshot),
|
||||||
|
runtime_capabilities_path: format!("{}{}", ADMIN_PREFIX, system::RUNTIME_CAPABILITIES_ROUTE_SUFFIX),
|
||||||
|
extensions_catalog_path: format!("{}{}", ADMIN_PREFIX, "/v4/extensions/catalog"),
|
||||||
|
topology: snapshot.topology,
|
||||||
|
membership: ClusterMembershipView::from(snapshot.membership),
|
||||||
|
pool_state: ClusterPoolStateView::from(snapshot.pool_state),
|
||||||
|
local_storage: ClusterLocalStorageView::from(snapshot.local_storage),
|
||||||
|
peer_health: ClusterPeerHealthView::from(snapshot.peer_health),
|
||||||
|
observability: snapshot.observability,
|
||||||
|
workload_admission: workload_admission_views(snapshot.workload_admission),
|
||||||
|
runtime_status: ClusterRuntimeStatusView::from(snapshot.runtime_status),
|
||||||
|
actionable_pressure,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterSnapshotSummary {
|
||||||
|
pub runtime: CapabilityStatus,
|
||||||
|
pub topology: CapabilityStatus,
|
||||||
|
pub membership: CapabilityStatus,
|
||||||
|
pub peer_health: CapabilityStatus,
|
||||||
|
pub observability: CapabilityStatus,
|
||||||
|
pub workload_admission: CapabilityStatus,
|
||||||
|
pub actionable_pressure: CapabilityStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&ClusterReadOnlySnapshot> for ClusterSnapshotSummary {
|
||||||
|
fn from(snapshot: &ClusterReadOnlySnapshot) -> Self {
|
||||||
|
let topology = summarize_topology(snapshot);
|
||||||
|
let membership = summarize_membership(snapshot);
|
||||||
|
let peer_health = summarize_peer_health(snapshot);
|
||||||
|
let observability = summarize_observability(snapshot);
|
||||||
|
let workload_admission = summarize_workload_admission(snapshot);
|
||||||
|
let actionable_pressure = if cluster_has_actionable_pressure(snapshot) {
|
||||||
|
CapabilityStatus::supported().with_reason("cluster snapshot reports degraded runtime or non-open admission")
|
||||||
|
} else {
|
||||||
|
CapabilityStatus::disabled().with_reason("cluster snapshot reports no actionable pressure")
|
||||||
|
};
|
||||||
|
|
||||||
|
Self {
|
||||||
|
runtime: summarize_runtime(snapshot),
|
||||||
|
topology,
|
||||||
|
membership,
|
||||||
|
peer_health,
|
||||||
|
observability,
|
||||||
|
workload_admission,
|
||||||
|
actionable_pressure,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterMembershipView {
|
||||||
|
pub nodes: Vec<ClusterNodeMembershipView>,
|
||||||
|
pub drives: Vec<ClusterDriveMembershipView>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ClusterMembershipSnapshot> for ClusterMembershipView {
|
||||||
|
fn from(snapshot: ClusterMembershipSnapshot) -> Self {
|
||||||
|
Self {
|
||||||
|
nodes: snapshot.nodes.into_iter().map(ClusterNodeMembershipView::from).collect(),
|
||||||
|
drives: snapshot.drives.into_iter().map(ClusterDriveMembershipView::from).collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterNodeMembershipView {
|
||||||
|
pub node_id: String,
|
||||||
|
pub grid_host: String,
|
||||||
|
pub is_local: bool,
|
||||||
|
pub pools: Vec<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ClusterNodeMembership> for ClusterNodeMembershipView {
|
||||||
|
fn from(node: ClusterNodeMembership) -> Self {
|
||||||
|
Self {
|
||||||
|
node_id: node.node_id,
|
||||||
|
grid_host: node.grid_host,
|
||||||
|
is_local: node.is_local,
|
||||||
|
pools: node.pools,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterDriveMembershipView {
|
||||||
|
pub pool_index: usize,
|
||||||
|
pub set_index: usize,
|
||||||
|
pub disk_index: usize,
|
||||||
|
pub node_id: String,
|
||||||
|
pub is_local: bool,
|
||||||
|
pub endpoint_type: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ClusterDriveMembership> for ClusterDriveMembershipView {
|
||||||
|
fn from(drive: ClusterDriveMembership) -> Self {
|
||||||
|
Self {
|
||||||
|
pool_index: drive.pool_index,
|
||||||
|
set_index: drive.set_index,
|
||||||
|
disk_index: drive.disk_index,
|
||||||
|
node_id: drive.node_id,
|
||||||
|
is_local: drive.is_local,
|
||||||
|
endpoint_type: endpoint_type_label(drive.endpoint_type),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterPoolStateView {
|
||||||
|
pub pools: Vec<ClusterPoolStateItemView>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ClusterPoolStateSnapshot> for ClusterPoolStateView {
|
||||||
|
fn from(snapshot: ClusterPoolStateSnapshot) -> Self {
|
||||||
|
Self {
|
||||||
|
pools: snapshot.pools.into_iter().map(ClusterPoolStateItemView::from).collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterPoolStateItemView {
|
||||||
|
pub pool_index: usize,
|
||||||
|
pub set_count: usize,
|
||||||
|
pub drives_per_set: usize,
|
||||||
|
pub endpoint_count: usize,
|
||||||
|
pub local_drive_count: usize,
|
||||||
|
pub remote_drive_count: usize,
|
||||||
|
pub legacy: bool,
|
||||||
|
pub endpoint_types: Vec<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ClusterPoolState> for ClusterPoolStateItemView {
|
||||||
|
fn from(pool: ClusterPoolState) -> Self {
|
||||||
|
Self {
|
||||||
|
pool_index: pool.pool_index,
|
||||||
|
set_count: pool.set_count,
|
||||||
|
drives_per_set: pool.drives_per_set,
|
||||||
|
endpoint_count: pool.endpoint_count,
|
||||||
|
local_drive_count: pool.local_drive_count,
|
||||||
|
remote_drive_count: pool.remote_drive_count,
|
||||||
|
legacy: pool.legacy,
|
||||||
|
endpoint_types: pool.endpoint_types.into_iter().map(endpoint_type_label).collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterLocalStorageView {
|
||||||
|
pub nodes: Vec<ClusterLocalNodeStorageView>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ClusterLocalNodeStorageSnapshot> for ClusterLocalStorageView {
|
||||||
|
fn from(snapshot: ClusterLocalNodeStorageSnapshot) -> Self {
|
||||||
|
Self {
|
||||||
|
nodes: snapshot.nodes.into_iter().map(ClusterLocalNodeStorageView::from).collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterLocalNodeStorageView {
|
||||||
|
pub node_id: String,
|
||||||
|
pub pools: Vec<usize>,
|
||||||
|
pub drive_count: usize,
|
||||||
|
pub path_drive_count: usize,
|
||||||
|
pub url_drive_count: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ClusterLocalNodeStorage> for ClusterLocalNodeStorageView {
|
||||||
|
fn from(node: ClusterLocalNodeStorage) -> Self {
|
||||||
|
Self {
|
||||||
|
node_id: node.node_id,
|
||||||
|
pools: node.pools,
|
||||||
|
drive_count: node.drive_count,
|
||||||
|
path_drive_count: node.path_drive_count,
|
||||||
|
url_drive_count: node.url_drive_count,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterPeerHealthView {
|
||||||
|
pub peers: Vec<ClusterPeerHealthItemView>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ClusterPeerHealthSnapshot> for ClusterPeerHealthView {
|
||||||
|
fn from(snapshot: ClusterPeerHealthSnapshot) -> Self {
|
||||||
|
Self {
|
||||||
|
peers: snapshot.peers.into_iter().map(ClusterPeerHealthItemView::from).collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterPeerHealthItemView {
|
||||||
|
pub node_id: String,
|
||||||
|
pub is_local: bool,
|
||||||
|
pub status: CapabilityStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ClusterPeerHealth> for ClusterPeerHealthItemView {
|
||||||
|
fn from(peer: ClusterPeerHealth) -> Self {
|
||||||
|
Self {
|
||||||
|
node_id: peer.node_id,
|
||||||
|
is_local: peer.is_local,
|
||||||
|
status: peer.status,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct WorkloadAdmissionView {
|
||||||
|
pub class: &'static str,
|
||||||
|
pub state: &'static str,
|
||||||
|
pub active: Option<usize>,
|
||||||
|
pub queued: Option<usize>,
|
||||||
|
pub limit: Option<usize>,
|
||||||
|
pub reason: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct ClusterRuntimeStatusView {
|
||||||
|
pub state: &'static str,
|
||||||
|
pub storage_ready: bool,
|
||||||
|
pub iam_ready: bool,
|
||||||
|
pub lock_quorum_ready: bool,
|
||||||
|
pub degraded_reasons: Vec<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ClusterRuntimeStatusSnapshot> for ClusterRuntimeStatusView {
|
||||||
|
fn from(runtime: ClusterRuntimeStatusSnapshot) -> Self {
|
||||||
|
Self {
|
||||||
|
state: runtime_readiness_state_label(runtime.state),
|
||||||
|
storage_ready: runtime.readiness.storage_ready,
|
||||||
|
iam_ready: runtime.readiness.iam_ready,
|
||||||
|
lock_quorum_ready: runtime.readiness.lock_quorum_ready,
|
||||||
|
degraded_reasons: runtime.degraded_reasons.into_iter().map(|reason| reason.as_str()).collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn workload_admission_views(snapshot: WorkloadAdmissionRegistrySnapshot) -> Vec<WorkloadAdmissionView> {
|
||||||
|
snapshot.entries().iter().cloned().map(WorkloadAdmissionView::from).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<WorkloadAdmissionSnapshot> for WorkloadAdmissionView {
|
||||||
|
fn from(snapshot: WorkloadAdmissionSnapshot) -> Self {
|
||||||
|
Self {
|
||||||
|
class: workload_class_label(snapshot.class),
|
||||||
|
state: admission_state_label(snapshot.state),
|
||||||
|
active: snapshot.active,
|
||||||
|
queued: snapshot.queued,
|
||||||
|
limit: snapshot.limit,
|
||||||
|
reason: snapshot.reason,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn endpoint_type_label(endpoint_type: ClusterEndpointType) -> &'static str {
|
||||||
|
match endpoint_type {
|
||||||
|
ClusterEndpointType::Path => "path",
|
||||||
|
ClusterEndpointType::Url => "url",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn workload_class_label(class: WorkloadClass) -> &'static str {
|
||||||
|
class.as_str()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn admission_state_label(state: AdmissionState) -> &'static str {
|
||||||
|
match state {
|
||||||
|
AdmissionState::Open => "open",
|
||||||
|
AdmissionState::Throttled => "throttled",
|
||||||
|
AdmissionState::Saturated => "saturated",
|
||||||
|
AdmissionState::Disabled => "disabled",
|
||||||
|
AdmissionState::Unknown => "unknown",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn runtime_readiness_state_label(state: ClusterRuntimeReadinessState) -> &'static str {
|
||||||
|
match state {
|
||||||
|
ClusterRuntimeReadinessState::Ready => "ready",
|
||||||
|
ClusterRuntimeReadinessState::Degraded => "degraded",
|
||||||
|
ClusterRuntimeReadinessState::Unknown => "unknown",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summarize_runtime(snapshot: &ClusterReadOnlySnapshot) -> CapabilityStatus {
|
||||||
|
match snapshot.runtime_status.state {
|
||||||
|
ClusterRuntimeReadinessState::Ready => CapabilityStatus::supported().with_reason("runtime readiness reports ready"),
|
||||||
|
ClusterRuntimeReadinessState::Degraded => {
|
||||||
|
let reasons = snapshot
|
||||||
|
.runtime_status
|
||||||
|
.degraded_reasons
|
||||||
|
.iter()
|
||||||
|
.map(|reason| reason.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ");
|
||||||
|
CapabilityStatus::unknown().with_reason(format!("runtime readiness degraded: {reasons}"))
|
||||||
|
}
|
||||||
|
ClusterRuntimeReadinessState::Unknown => CapabilityStatus::unknown().with_reason("runtime readiness status unknown"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summarize_topology(snapshot: &ClusterReadOnlySnapshot) -> CapabilityStatus {
|
||||||
|
summarize_named_capability_statuses(
|
||||||
|
[
|
||||||
|
("profiling", &snapshot.topology.capabilities.profiling),
|
||||||
|
("numa", &snapshot.topology.capabilities.numa),
|
||||||
|
("failure_domain_labels", &snapshot.topology.capabilities.failure_domain_labels),
|
||||||
|
("media_labels", &snapshot.topology.capabilities.media_labels),
|
||||||
|
],
|
||||||
|
"topology capability",
|
||||||
|
)
|
||||||
|
.with_reason("topology summary resolved from cluster snapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summarize_membership(snapshot: &ClusterReadOnlySnapshot) -> CapabilityStatus {
|
||||||
|
if snapshot.membership.nodes.is_empty() {
|
||||||
|
CapabilityStatus::unknown().with_reason("cluster membership has no nodes")
|
||||||
|
} else {
|
||||||
|
CapabilityStatus::supported().with_reason(format!(
|
||||||
|
"cluster membership reports {} nodes and {} drives",
|
||||||
|
snapshot.membership.nodes.len(),
|
||||||
|
snapshot.membership.drives.len()
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summarize_peer_health(snapshot: &ClusterReadOnlySnapshot) -> CapabilityStatus {
|
||||||
|
if snapshot.peer_health.peers.is_empty() {
|
||||||
|
return CapabilityStatus::unknown().with_reason("cluster peer health has no peers");
|
||||||
|
}
|
||||||
|
|
||||||
|
let unknown = snapshot
|
||||||
|
.peer_health
|
||||||
|
.peers
|
||||||
|
.iter()
|
||||||
|
.filter(|peer| peer.status.state == CapabilityState::Unknown)
|
||||||
|
.count();
|
||||||
|
if unknown > 0 {
|
||||||
|
CapabilityStatus::unknown().with_reason(format!("cluster peer health has {unknown} unresolved peers"))
|
||||||
|
} else {
|
||||||
|
CapabilityStatus::supported().with_reason("cluster peer health resolved for all peers")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summarize_workload_admission(snapshot: &ClusterReadOnlySnapshot) -> CapabilityStatus {
|
||||||
|
let entries = snapshot.workload_admission.entries();
|
||||||
|
if entries.is_empty() {
|
||||||
|
return CapabilityStatus::unknown().with_reason("workload admission snapshot is empty");
|
||||||
|
}
|
||||||
|
|
||||||
|
let non_open = entries
|
||||||
|
.iter()
|
||||||
|
.filter(|entry| entry.state != WorkloadAdmissionState::Open)
|
||||||
|
.map(|entry| entry.class.as_str())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if non_open.is_empty() {
|
||||||
|
CapabilityStatus::supported().with_reason("all workload admission classes are open")
|
||||||
|
} else {
|
||||||
|
CapabilityStatus::unknown().with_reason(format!("workload admission has non-open classes: {}", non_open.join(", ")))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summarize_observability(snapshot: &ClusterReadOnlySnapshot) -> CapabilityStatus {
|
||||||
|
let userspace_profiling = summarize_named_capability_statuses(
|
||||||
|
[
|
||||||
|
("cpu", &snapshot.observability.userspace_profiling.cpu),
|
||||||
|
("memory", &snapshot.observability.userspace_profiling.memory),
|
||||||
|
("continuous_cpu", &snapshot.observability.userspace_profiling.continuous_cpu),
|
||||||
|
("periodic_cpu", &snapshot.observability.userspace_profiling.periodic_cpu),
|
||||||
|
],
|
||||||
|
"userspace profiling",
|
||||||
|
);
|
||||||
|
let memory_sampling = summarize_named_capability_statuses(
|
||||||
|
[
|
||||||
|
("process", &snapshot.observability.memory_sampling.process),
|
||||||
|
("system", &snapshot.observability.memory_sampling.system),
|
||||||
|
("cgroup", &snapshot.observability.memory_sampling.cgroup),
|
||||||
|
],
|
||||||
|
"memory sampling",
|
||||||
|
);
|
||||||
|
let platform = summarize_named_capability_statuses(
|
||||||
|
[
|
||||||
|
("allocator", &snapshot.observability.platform.allocator),
|
||||||
|
("ebpf", &snapshot.observability.platform.ebpf),
|
||||||
|
("numa", &snapshot.observability.platform.numa),
|
||||||
|
],
|
||||||
|
"platform support",
|
||||||
|
);
|
||||||
|
|
||||||
|
if [userspace_profiling.state, memory_sampling.state, platform.state]
|
||||||
|
.into_iter()
|
||||||
|
.any(|state| state == CapabilityState::Unknown)
|
||||||
|
{
|
||||||
|
CapabilityStatus::unknown().with_reason("observability summary resolved from cluster snapshot")
|
||||||
|
} else {
|
||||||
|
CapabilityStatus::supported().with_reason("observability summary resolved from cluster snapshot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summarize_named_capability_statuses<const N: usize>(
|
||||||
|
statuses: [(&'static str, &CapabilityStatus); N],
|
||||||
|
subject: &'static str,
|
||||||
|
) -> CapabilityStatus {
|
||||||
|
let unknown = statuses
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(name, status)| (status.state == CapabilityState::Unknown).then_some(*name))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if !unknown.is_empty() {
|
||||||
|
return CapabilityStatus::unknown().with_reason(format!("{subject} unresolved fields: {}", unknown.join(", ")));
|
||||||
|
}
|
||||||
|
|
||||||
|
let supported_or_disabled = statuses
|
||||||
|
.iter()
|
||||||
|
.any(|(_, status)| matches!(status.state, CapabilityState::Supported | CapabilityState::Disabled));
|
||||||
|
if supported_or_disabled {
|
||||||
|
return CapabilityStatus::supported();
|
||||||
|
}
|
||||||
|
|
||||||
|
let unsupported = statuses.iter().map(|(name, _)| *name).collect::<Vec<_>>();
|
||||||
|
CapabilityStatus::unsupported().with_reason(format!("{subject} unsupported fields: {}", unsupported.join(", ")))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{ClusterSnapshotResponse, ClusterSnapshotSummary, ClusterSnapshotView};
|
||||||
|
use crate::cluster_snapshot::{ClusterReadOnlySnapshot, ClusterRuntimeReadinessState, ClusterRuntimeStatusSnapshot};
|
||||||
|
use crate::server::{DependencyReadiness, ReadinessDegradedReason};
|
||||||
|
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadClass};
|
||||||
|
use rustfs_ecstore::api::cluster::{
|
||||||
|
ClusterDriveMembership, ClusterEndpointType, ClusterLocalNodeStorage, ClusterLocalNodeStorageSnapshot,
|
||||||
|
ClusterMembershipSnapshot, ClusterNodeMembership, ClusterPeerHealth, ClusterPeerHealthSnapshot, ClusterPoolState,
|
||||||
|
ClusterPoolStateSnapshot,
|
||||||
|
};
|
||||||
|
use rustfs_storage_api::CapabilityState;
|
||||||
|
use rustfs_storage_api::{CapabilityStatus, ObservabilitySnapshot, TopologySnapshot};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cluster_snapshot_handler_requires_server_info_admin_permission() {
|
||||||
|
let src = include_str!("cluster_snapshot.rs");
|
||||||
|
let handler_block = extract_block_between_markers(src, "impl Operation for GetClusterSnapshotHandler", "#[cfg(test)]");
|
||||||
|
let auth_block =
|
||||||
|
extract_block_between_markers(src, "async fn authorize_cluster_snapshot_request", "fn build_json_response");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
handler_block.contains("authorize_cluster_snapshot_request(&req).await?;"),
|
||||||
|
"cluster snapshot handler should require admin authorization"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
auth_block.contains("AdminAction::ServerInfoAdminAction"),
|
||||||
|
"cluster snapshot should require server info admin permission"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cluster_snapshot_response_serializes_none_snapshot() {
|
||||||
|
let value = serde_json::to_value(ClusterSnapshotResponse { snapshot: None }).expect("serialize response");
|
||||||
|
assert_eq!(value, serde_json::json!({ "snapshot": null }));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cluster_snapshot_discovery_reports_path_without_snapshot() {
|
||||||
|
let response = super::build_cluster_snapshot_discovery_response().await;
|
||||||
|
|
||||||
|
assert_eq!(response.path, "/rustfs/admin/v4/cluster/snapshot");
|
||||||
|
assert_eq!(response.summary, None);
|
||||||
|
assert_eq!(response.topology, None);
|
||||||
|
assert_eq!(response.peer_health, None);
|
||||||
|
assert_eq!(response.workload_admission, None);
|
||||||
|
assert_eq!(response.runtime, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cluster_snapshot_view_serializes_machine_readable_sections() {
|
||||||
|
let snapshot = ClusterReadOnlySnapshot {
|
||||||
|
topology: TopologySnapshot::default(),
|
||||||
|
membership: ClusterMembershipSnapshot {
|
||||||
|
nodes: vec![ClusterNodeMembership {
|
||||||
|
node_id: "node-a".to_string(),
|
||||||
|
grid_host: "node-a:9000".to_string(),
|
||||||
|
is_local: true,
|
||||||
|
pools: vec![0],
|
||||||
|
}],
|
||||||
|
drives: vec![ClusterDriveMembership {
|
||||||
|
pool_index: 0,
|
||||||
|
set_index: 1,
|
||||||
|
disk_index: 2,
|
||||||
|
node_id: "node-a".to_string(),
|
||||||
|
is_local: true,
|
||||||
|
endpoint_type: ClusterEndpointType::Url,
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
pool_state: ClusterPoolStateSnapshot {
|
||||||
|
pools: vec![ClusterPoolState {
|
||||||
|
pool_index: 0,
|
||||||
|
set_count: 1,
|
||||||
|
drives_per_set: 4,
|
||||||
|
endpoint_count: 4,
|
||||||
|
local_drive_count: 2,
|
||||||
|
remote_drive_count: 2,
|
||||||
|
legacy: false,
|
||||||
|
endpoint_types: vec![ClusterEndpointType::Url],
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
local_storage: ClusterLocalNodeStorageSnapshot {
|
||||||
|
nodes: vec![ClusterLocalNodeStorage {
|
||||||
|
node_id: "node-a".to_string(),
|
||||||
|
pools: vec![0],
|
||||||
|
drive_count: 2,
|
||||||
|
path_drive_count: 0,
|
||||||
|
url_drive_count: 2,
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
peer_health: ClusterPeerHealthSnapshot {
|
||||||
|
peers: vec![ClusterPeerHealth {
|
||||||
|
node_id: "node-b".to_string(),
|
||||||
|
is_local: false,
|
||||||
|
status: CapabilityStatus::unknown().with_reason("peer state unavailable"),
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
observability: ObservabilitySnapshot::default(),
|
||||||
|
workload_admission: WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||||
|
WorkloadAdmissionSnapshot::new(WorkloadClass::Repair, AdmissionState::Unknown)
|
||||||
|
.with_counts(Some(1), Some(2), Some(8))
|
||||||
|
.with_reason("repair backlog not yet normalized"),
|
||||||
|
]),
|
||||||
|
runtime_status: ClusterRuntimeStatusSnapshot {
|
||||||
|
readiness: DependencyReadiness {
|
||||||
|
storage_ready: false,
|
||||||
|
iam_ready: true,
|
||||||
|
lock_quorum_ready: false,
|
||||||
|
},
|
||||||
|
state: ClusterRuntimeReadinessState::Degraded,
|
||||||
|
degraded_reasons: vec![ReadinessDegradedReason::StorageAndLockUnavailable],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
let value = serde_json::to_value(ClusterSnapshotView::from(snapshot)).expect("serialize view");
|
||||||
|
assert_eq!(value["runtime_capabilities_path"], "/rustfs/admin/v4/runtime/capabilities");
|
||||||
|
assert_eq!(value["extensions_catalog_path"], "/rustfs/admin/v4/extensions/catalog");
|
||||||
|
assert_eq!(value["membership"]["drives"][0]["endpoint_type"], "url");
|
||||||
|
assert_eq!(value["workload_admission"][0]["class"], "repair");
|
||||||
|
assert_eq!(value["workload_admission"][0]["state"], "unknown");
|
||||||
|
assert_eq!(value["runtime_status"]["state"], "degraded");
|
||||||
|
assert_eq!(value["summary"]["runtime"]["state"], "unknown");
|
||||||
|
assert_eq!(value["runtime_status"]["degraded_reasons"][0], "storage_and_lock_unavailable");
|
||||||
|
assert_eq!(value["actionable_pressure"], true);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cluster_snapshot_summary_reports_cross_surface_status() {
|
||||||
|
let snapshot = ClusterReadOnlySnapshot {
|
||||||
|
topology: TopologySnapshot::default(),
|
||||||
|
membership: ClusterMembershipSnapshot::default(),
|
||||||
|
pool_state: ClusterPoolStateSnapshot::default(),
|
||||||
|
local_storage: ClusterLocalNodeStorageSnapshot::default(),
|
||||||
|
peer_health: ClusterPeerHealthSnapshot::default(),
|
||||||
|
observability: ObservabilitySnapshot::default(),
|
||||||
|
workload_admission: WorkloadAdmissionRegistrySnapshot::new(vec![WorkloadAdmissionSnapshot::new(
|
||||||
|
WorkloadClass::ForegroundRead,
|
||||||
|
AdmissionState::Open,
|
||||||
|
)]),
|
||||||
|
runtime_status: ClusterRuntimeStatusSnapshot {
|
||||||
|
readiness: DependencyReadiness {
|
||||||
|
storage_ready: true,
|
||||||
|
iam_ready: true,
|
||||||
|
lock_quorum_ready: true,
|
||||||
|
},
|
||||||
|
state: ClusterRuntimeReadinessState::Ready,
|
||||||
|
degraded_reasons: Vec::new(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
let summary = ClusterSnapshotSummary::from(&snapshot);
|
||||||
|
assert_eq!(summary.runtime.state, CapabilityState::Supported);
|
||||||
|
assert_eq!(summary.membership.state, CapabilityState::Unknown);
|
||||||
|
assert_eq!(summary.peer_health.state, CapabilityState::Unknown);
|
||||||
|
assert_eq!(summary.workload_admission.state, CapabilityState::Supported);
|
||||||
|
assert_eq!(summary.actionable_pressure.state, CapabilityState::Disabled);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_block_between_markers(src: &str, start: &str, end: &str) -> String {
|
||||||
|
let start_index = src.find(start).expect("start marker should exist");
|
||||||
|
let end_index = src[start_index..]
|
||||||
|
.find(end)
|
||||||
|
.map(|index| start_index + index)
|
||||||
|
.unwrap_or(src.len());
|
||||||
|
src[start_index..end_index].to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
use crate::admin::{
|
use crate::admin::{
|
||||||
auth::validate_admin_request,
|
auth::validate_admin_request,
|
||||||
handlers::{plugins_instances, system},
|
handlers::{cluster_snapshot, plugins_instances, system},
|
||||||
plugin_contract::{
|
plugin_contract::{
|
||||||
PluginContractDomain, PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry,
|
PluginContractDomain, PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry,
|
||||||
PluginInstanceSource, PluginOperationalStateContract,
|
PluginInstanceSource, PluginOperationalStateContract,
|
||||||
@@ -62,6 +62,7 @@ pub fn register_extension_route(r: &mut S3Router<AdminOperation>) -> std::io::Re
|
|||||||
pub(crate) struct ExtensionCatalogResponse {
|
pub(crate) struct ExtensionCatalogResponse {
|
||||||
pub extensions: Vec<ExtensionSchema>,
|
pub extensions: Vec<ExtensionSchema>,
|
||||||
pub runtime_capabilities: ExtensionRuntimeCapabilitiesResponse,
|
pub runtime_capabilities: ExtensionRuntimeCapabilitiesResponse,
|
||||||
|
pub cluster_snapshot: cluster_snapshot::ClusterSnapshotDiscoveryResponse,
|
||||||
pub external_plugin_flow: TargetPluginExternalFlowGateStatus,
|
pub external_plugin_flow: TargetPluginExternalFlowGateStatus,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -123,10 +124,12 @@ async fn build_extension_catalog_response() -> Result<ExtensionCatalogResponse,
|
|||||||
extensions.push(target_marketplace_extension_schema(&example.manifest));
|
extensions.push(target_marketplace_extension_schema(&example.manifest));
|
||||||
extensions.sort_by(|a, b| a.extension_id.cmp(&b.extension_id));
|
extensions.sort_by(|a, b| a.extension_id.cmp(&b.extension_id));
|
||||||
let runtime_capabilities = system::build_runtime_capabilities_response().await?;
|
let runtime_capabilities = system::build_runtime_capabilities_response().await?;
|
||||||
|
let cluster_snapshot = cluster_snapshot::build_cluster_snapshot_discovery_response().await;
|
||||||
|
|
||||||
Ok(ExtensionCatalogResponse {
|
Ok(ExtensionCatalogResponse {
|
||||||
extensions,
|
extensions,
|
||||||
runtime_capabilities: build_extension_runtime_capabilities_response(&runtime_capabilities),
|
runtime_capabilities: build_extension_runtime_capabilities_response(&runtime_capabilities),
|
||||||
|
cluster_snapshot,
|
||||||
external_plugin_flow: TargetPluginExternalFlowGate::default().status(),
|
external_plugin_flow: TargetPluginExternalFlowGate::default().status(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -450,6 +453,8 @@ mod tests {
|
|||||||
Some(&runtime_capabilities.summary.userspace_profiling)
|
Some(&runtime_capabilities.summary.userspace_profiling)
|
||||||
);
|
);
|
||||||
assert!(validate_ops_profiler_contract(&response.runtime_capabilities.ops_profiler.contract).is_ok());
|
assert!(validate_ops_profiler_contract(&response.runtime_capabilities.ops_profiler.contract).is_ok());
|
||||||
|
|
||||||
|
assert_eq!(response.cluster_snapshot.path, format!("{}{}", ADMIN_PREFIX, "/v4/cluster/snapshot"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ pub mod account_info;
|
|||||||
pub mod audit;
|
pub mod audit;
|
||||||
mod audit_runtime_config;
|
mod audit_runtime_config;
|
||||||
pub mod bucket_meta;
|
pub mod bucket_meta;
|
||||||
|
pub mod cluster_snapshot;
|
||||||
pub mod config_admin;
|
pub mod config_admin;
|
||||||
pub mod event;
|
pub mod event;
|
||||||
pub mod extensions;
|
pub mod extensions;
|
||||||
@@ -73,6 +74,7 @@ mod tests {
|
|||||||
let _set_config_handler = config_admin::SetConfigHandler {};
|
let _set_config_handler = config_admin::SetConfigHandler {};
|
||||||
let _list_audit_targets = audit::ListAuditTargets {};
|
let _list_audit_targets = audit::ListAuditTargets {};
|
||||||
let _get_module_switches = module_switch::GetModuleSwitchesHandler {};
|
let _get_module_switches = module_switch::GetModuleSwitchesHandler {};
|
||||||
|
let _get_cluster_snapshot = cluster_snapshot::GetClusterSnapshotHandler {};
|
||||||
let _get_extension_catalog = extensions::GetExtensionCatalogHandler {};
|
let _get_extension_catalog = extensions::GetExtensionCatalogHandler {};
|
||||||
let _list_extension_instances = extensions::ListExtensionInstancesHandler {};
|
let _list_extension_instances = extensions::ListExtensionInstancesHandler {};
|
||||||
let _get_plugin_catalog = plugins_catalog::GetPluginCatalogHandler {};
|
let _get_plugin_catalog = plugins_catalog::GetPluginCatalogHandler {};
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use crate::admin::{
|
|||||||
auth::validate_admin_request,
|
auth::validate_admin_request,
|
||||||
router::{AdminOperation, Operation, S3Router},
|
router::{AdminOperation, Operation, S3Router},
|
||||||
};
|
};
|
||||||
|
use crate::app::admin_usecase::DefaultAdminUsecase;
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
use crate::server::{
|
use crate::server::{
|
||||||
ADMIN_PREFIX, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, RemoteAddr, current_module_switch_snapshot,
|
ADMIN_PREFIX, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, RemoteAddr, current_module_switch_snapshot,
|
||||||
@@ -62,10 +63,22 @@ struct ModuleSwitchesResponse {
|
|||||||
persisted_audit_enabled: bool,
|
persisted_audit_enabled: bool,
|
||||||
notify_source: ModuleSwitchSource,
|
notify_source: ModuleSwitchSource,
|
||||||
audit_source: ModuleSwitchSource,
|
audit_source: ModuleSwitchSource,
|
||||||
|
admin_discovery: ModuleSwitchDiscovery,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||||
|
struct ModuleSwitchDiscovery {
|
||||||
|
#[serde(rename = "runtimeCapabilities")]
|
||||||
|
runtime_capabilities: &'static str,
|
||||||
|
#[serde(rename = "clusterSnapshot")]
|
||||||
|
cluster_snapshot: &'static str,
|
||||||
|
#[serde(rename = "extensionsCatalog")]
|
||||||
|
extensions_catalog: &'static str,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<ModuleSwitchSnapshot> for ModuleSwitchesResponse {
|
impl From<ModuleSwitchSnapshot> for ModuleSwitchesResponse {
|
||||||
fn from(value: ModuleSwitchSnapshot) -> Self {
|
fn from(value: ModuleSwitchSnapshot) -> Self {
|
||||||
|
let usecase = DefaultAdminUsecase::from_global();
|
||||||
Self {
|
Self {
|
||||||
notify_enabled: value.notify_enabled,
|
notify_enabled: value.notify_enabled,
|
||||||
audit_enabled: value.audit_enabled,
|
audit_enabled: value.audit_enabled,
|
||||||
@@ -73,6 +86,11 @@ impl From<ModuleSwitchSnapshot> for ModuleSwitchesResponse {
|
|||||||
persisted_audit_enabled: value.persisted_audit_enabled,
|
persisted_audit_enabled: value.persisted_audit_enabled,
|
||||||
notify_source: value.notify_source,
|
notify_source: value.notify_source,
|
||||||
audit_source: value.audit_source,
|
audit_source: value.audit_source,
|
||||||
|
admin_discovery: ModuleSwitchDiscovery {
|
||||||
|
runtime_capabilities: usecase.runtime_capabilities_route(),
|
||||||
|
cluster_snapshot: usecase.cluster_snapshot_route(),
|
||||||
|
extensions_catalog: usecase.extensions_catalog_route(),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -195,6 +213,8 @@ impl Operation for UpdateModuleSwitchesHandler {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
use super::{ModuleSwitchDiscovery, ModuleSwitchSource, ModuleSwitchesResponse};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn module_switch_handlers_require_admin_authorization_contract() {
|
fn module_switch_handlers_require_admin_authorization_contract() {
|
||||||
let src = include_str!("module_switch.rs");
|
let src = include_str!("module_switch.rs");
|
||||||
@@ -215,6 +235,28 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn module_switch_response_exposes_admin_discovery_paths() {
|
||||||
|
let response = ModuleSwitchesResponse {
|
||||||
|
notify_enabled: true,
|
||||||
|
audit_enabled: false,
|
||||||
|
persisted_notify_enabled: true,
|
||||||
|
persisted_audit_enabled: false,
|
||||||
|
notify_source: ModuleSwitchSource::Console,
|
||||||
|
audit_source: ModuleSwitchSource::Console,
|
||||||
|
admin_discovery: ModuleSwitchDiscovery {
|
||||||
|
runtime_capabilities: "/rustfs/admin/v4/runtime/capabilities",
|
||||||
|
cluster_snapshot: "/rustfs/admin/v4/cluster/snapshot",
|
||||||
|
extensions_catalog: "/rustfs/admin/v4/extensions/catalog",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
let value = serde_json::to_value(response).expect("module switch response should serialize");
|
||||||
|
assert_eq!(value["admin_discovery"]["runtimeCapabilities"], "/rustfs/admin/v4/runtime/capabilities");
|
||||||
|
assert_eq!(value["admin_discovery"]["clusterSnapshot"], "/rustfs/admin/v4/cluster/snapshot");
|
||||||
|
assert_eq!(value["admin_discovery"]["extensionsCatalog"], "/rustfs/admin/v4/extensions/catalog");
|
||||||
|
}
|
||||||
|
|
||||||
fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str {
|
fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str {
|
||||||
let start = src
|
let start = src
|
||||||
.find(start_marker)
|
.find(start_marker)
|
||||||
|
|||||||
@@ -15,11 +15,12 @@
|
|||||||
use crate::admin::{
|
use crate::admin::{
|
||||||
auth::validate_admin_request,
|
auth::validate_admin_request,
|
||||||
plugin_contract::{
|
plugin_contract::{
|
||||||
PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse, PluginContractDomain, PluginContractEntrypointKind,
|
PluginCatalogAdminDiscovery, PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse, PluginContractDomain,
|
||||||
PluginContractPackaging, PluginDistributionContract, PluginRuntimeContract,
|
PluginContractEntrypointKind, PluginContractPackaging, PluginDistributionContract, PluginRuntimeContract,
|
||||||
},
|
},
|
||||||
router::{AdminOperation, Operation, S3Router},
|
router::{AdminOperation, Operation, S3Router},
|
||||||
};
|
};
|
||||||
|
use crate::app::admin_usecase::DefaultAdminUsecase;
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||||
@@ -57,6 +58,7 @@ fn target_domain_name_from_subsystem(subsystem: &str) -> PluginContractDomain {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn build_catalog_response() -> PluginCatalogResponse {
|
fn build_catalog_response() -> PluginCatalogResponse {
|
||||||
|
let usecase = DefaultAdminUsecase::from_global();
|
||||||
let mut plugins: HashMap<&'static str, PluginCatalogEntry> = HashMap::new();
|
let mut plugins: HashMap<&'static str, PluginCatalogEntry> = HashMap::new();
|
||||||
|
|
||||||
for descriptor in builtin_notify_target_admin_descriptors()
|
for descriptor in builtin_notify_target_admin_descriptors()
|
||||||
@@ -74,7 +76,14 @@ fn build_catalog_response() -> PluginCatalogResponse {
|
|||||||
plugin.domain_configs.sort_by_key(|a| a.domain);
|
plugin.domain_configs.sort_by_key(|a| a.domain);
|
||||||
}
|
}
|
||||||
|
|
||||||
PluginCatalogResponse { plugins }
|
PluginCatalogResponse {
|
||||||
|
plugins,
|
||||||
|
admin_discovery: PluginCatalogAdminDiscovery {
|
||||||
|
runtime_capabilities: usecase.runtime_capabilities_route().to_string(),
|
||||||
|
cluster_snapshot: usecase.cluster_snapshot_route().to_string(),
|
||||||
|
extensions_catalog: usecase.extensions_catalog_route().to_string(),
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn example_external_webhook_plugin_entry() -> PluginCatalogEntry {
|
fn example_external_webhook_plugin_entry() -> PluginCatalogEntry {
|
||||||
@@ -200,6 +209,10 @@ mod tests {
|
|||||||
fn plugin_catalog_contains_representative_builtin_targets() {
|
fn plugin_catalog_contains_representative_builtin_targets() {
|
||||||
let response = build_catalog_response();
|
let response = build_catalog_response();
|
||||||
|
|
||||||
|
assert_eq!(response.admin_discovery.runtime_capabilities, "/rustfs/admin/v4/runtime/capabilities");
|
||||||
|
assert_eq!(response.admin_discovery.cluster_snapshot, "/rustfs/admin/v4/cluster/snapshot");
|
||||||
|
assert_eq!(response.admin_discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog");
|
||||||
|
|
||||||
let webhook = response
|
let webhook = response
|
||||||
.plugins
|
.plugins
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -40,6 +40,22 @@ use hyper::Method;
|
|||||||
|
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize)]
|
||||||
|
struct PoolAdminDiscovery {
|
||||||
|
#[serde(rename = "runtimeCapabilities")]
|
||||||
|
runtime_capabilities: String,
|
||||||
|
#[serde(rename = "clusterSnapshot")]
|
||||||
|
cluster_snapshot: String,
|
||||||
|
#[serde(rename = "extensionsCatalog")]
|
||||||
|
extensions_catalog: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize)]
|
||||||
|
struct PoolStatusResponse {
|
||||||
|
pool: crate::app::admin_usecase::AdminPoolStatus,
|
||||||
|
admin_discovery: PoolAdminDiscovery,
|
||||||
|
}
|
||||||
|
|
||||||
const LOG_COMPONENT_ADMIN_API: &str = "admin_api";
|
const LOG_COMPONENT_ADMIN_API: &str = "admin_api";
|
||||||
const LOG_SUBSYSTEM_POOL_ADMIN: &str = "pool_admin";
|
const LOG_SUBSYSTEM_POOL_ADMIN: &str = "pool_admin";
|
||||||
const EVENT_ADMIN_REQUEST_STATE: &str = "admin_request_state";
|
const EVENT_ADMIN_REQUEST_STATE: &str = "admin_request_state";
|
||||||
@@ -396,6 +412,15 @@ fn parse_pool_idx_by_id(pool: &str, endpoint_count: usize) -> Option<usize> {
|
|||||||
(idx < endpoint_count).then_some(idx)
|
(idx < endpoint_count).then_some(idx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn pool_admin_discovery() -> PoolAdminDiscovery {
|
||||||
|
let usecase = DefaultAdminUsecase::from_global();
|
||||||
|
PoolAdminDiscovery {
|
||||||
|
runtime_capabilities: usecase.runtime_capabilities_route().to_string(),
|
||||||
|
cluster_snapshot: usecase.cluster_snapshot_route().to_string(),
|
||||||
|
extensions_catalog: usecase.extensions_catalog_route().to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn has_duplicate_indices(indices: &[usize]) -> bool {
|
fn has_duplicate_indices(indices: &[usize]) -> bool {
|
||||||
let mut seen = HashSet::with_capacity(indices.len());
|
let mut seen = HashSet::with_capacity(indices.len());
|
||||||
for idx in indices {
|
for idx in indices {
|
||||||
@@ -580,8 +605,12 @@ impl Operation for StatusPool {
|
|||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(S3Error::from)?;
|
.map_err(S3Error::from)?;
|
||||||
|
let response = PoolStatusResponse {
|
||||||
|
pool: pools_status,
|
||||||
|
admin_discovery: pool_admin_discovery(),
|
||||||
|
};
|
||||||
|
|
||||||
let data = serde_json::to_vec(&pools_status).map_err(|e| {
|
let data = serde_json::to_vec(&response).map_err(|e| {
|
||||||
log_pool_request_failed!("query_pool_status", "serialize_pool_status_failed", e);
|
log_pool_request_failed!("query_pool_status", "serialize_pool_status_failed", e);
|
||||||
S3Error::with_message(S3ErrorCode::InternalError, "parse accountInfo failed")
|
S3Error::with_message(S3ErrorCode::InternalError, "parse accountInfo failed")
|
||||||
})?;
|
})?;
|
||||||
@@ -1320,4 +1349,33 @@ mod pools_handler_tests {
|
|||||||
assert!(!has_duplicate_indices(&empty));
|
assert!(!has_duplicate_indices(&empty));
|
||||||
assert!(!has_duplicate_indices(&[0, 2, 1, 3]));
|
assert!(!has_duplicate_indices(&[0, 2, 1, 3]));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_pool_status_response_exposes_admin_discovery_paths() {
|
||||||
|
let response = super::PoolStatusResponse {
|
||||||
|
pool: crate::app::admin_usecase::AdminPoolStatus {
|
||||||
|
id: 0,
|
||||||
|
cmd_line: "pool-0".to_string(),
|
||||||
|
last_update: time::OffsetDateTime::UNIX_EPOCH,
|
||||||
|
total_size: 0,
|
||||||
|
current_size: 0,
|
||||||
|
used_size: 0,
|
||||||
|
used: 0.0,
|
||||||
|
status: "active".to_string(),
|
||||||
|
decommission_status: "none".to_string(),
|
||||||
|
rebalance_status: "none".to_string(),
|
||||||
|
decommission: None,
|
||||||
|
},
|
||||||
|
admin_discovery: super::PoolAdminDiscovery {
|
||||||
|
runtime_capabilities: "/rustfs/admin/v4/runtime/capabilities".to_string(),
|
||||||
|
cluster_snapshot: "/rustfs/admin/v4/cluster/snapshot".to_string(),
|
||||||
|
extensions_catalog: "/rustfs/admin/v4/extensions/catalog".to_string(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
let value = serde_json::to_value(response).expect("pool status response should serialize");
|
||||||
|
assert_eq!(value["admin_discovery"]["runtimeCapabilities"], "/rustfs/admin/v4/runtime/capabilities");
|
||||||
|
assert_eq!(value["admin_discovery"]["clusterSnapshot"], "/rustfs/admin/v4/cluster/snapshot");
|
||||||
|
assert_eq!(value["admin_discovery"]["extensionsCatalog"], "/rustfs/admin/v4/extensions/catalog");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::metrics;
|
use super::{cluster_snapshot, metrics};
|
||||||
use crate::admin::auth::validate_admin_request;
|
use crate::admin::auth::validate_admin_request;
|
||||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||||
use crate::app::admin_usecase::{DefaultAdminUsecase, QueryServerInfoRequest};
|
use crate::app::admin_usecase::{DefaultAdminUsecase, QueryServerInfoRequest};
|
||||||
@@ -25,6 +25,7 @@ use http::{HeaderMap, HeaderValue};
|
|||||||
use hyper::{Method, StatusCode};
|
use hyper::{Method, StatusCode};
|
||||||
use matchit::Params;
|
use matchit::Params;
|
||||||
use rustfs_concurrency::WorkloadAdmissionRegistrySnapshot;
|
use rustfs_concurrency::WorkloadAdmissionRegistrySnapshot;
|
||||||
|
use rustfs_madmin::{InfoMessage, StorageInfo};
|
||||||
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
||||||
use rustfs_storage_api::{
|
use rustfs_storage_api::{
|
||||||
CapabilityState, CapabilityStatus, ObservabilitySnapshotProvider, TopologySnapshot, TopologySnapshotProvider,
|
CapabilityState, CapabilityStatus, ObservabilitySnapshotProvider, TopologySnapshot, TopologySnapshotProvider,
|
||||||
@@ -149,6 +150,36 @@ impl Operation for ServiceHandle {
|
|||||||
|
|
||||||
pub struct ServerInfoHandler {}
|
pub struct ServerInfoHandler {}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
struct SystemAdminDiscovery {
|
||||||
|
#[serde(rename = "runtimeCapabilities")]
|
||||||
|
runtime_capabilities: String,
|
||||||
|
#[serde(rename = "clusterSnapshot")]
|
||||||
|
cluster_snapshot: String,
|
||||||
|
#[serde(rename = "extensionsCatalog")]
|
||||||
|
extensions_catalog: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct ServerInfoResponse {
|
||||||
|
info: InfoMessage,
|
||||||
|
admin_discovery: SystemAdminDiscovery,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct StorageInfoResponse {
|
||||||
|
info: StorageInfo,
|
||||||
|
admin_discovery: SystemAdminDiscovery,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn system_admin_discovery(usecase: &DefaultAdminUsecase) -> SystemAdminDiscovery {
|
||||||
|
SystemAdminDiscovery {
|
||||||
|
runtime_capabilities: usecase.runtime_capabilities_route().to_string(),
|
||||||
|
cluster_snapshot: usecase.cluster_snapshot_route().to_string(),
|
||||||
|
extensions_catalog: usecase.extensions_catalog_route().to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
impl Operation for ServerInfoHandler {
|
impl Operation for ServerInfoHandler {
|
||||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||||
@@ -177,8 +208,12 @@ impl Operation for ServerInfoHandler {
|
|||||||
.await
|
.await
|
||||||
.map_err(S3Error::from)?
|
.map_err(S3Error::from)?
|
||||||
.info;
|
.info;
|
||||||
|
let response = ServerInfoResponse {
|
||||||
|
info,
|
||||||
|
admin_discovery: system_admin_discovery(&usecase),
|
||||||
|
};
|
||||||
|
|
||||||
let data = serde_json::to_vec(&info).map_err(|e| {
|
let data = serde_json::to_vec(&response).map_err(|e| {
|
||||||
log_system_request_failed!("query_server_info", "serialize_server_info_failed", e);
|
log_system_request_failed!("query_server_info", "serialize_server_info_failed", e);
|
||||||
S3Error::with_message(S3ErrorCode::InternalError, "parse serverInfo failed")
|
S3Error::with_message(S3ErrorCode::InternalError, "parse serverInfo failed")
|
||||||
})?;
|
})?;
|
||||||
@@ -227,8 +262,12 @@ impl Operation for StorageInfoHandler {
|
|||||||
|
|
||||||
let usecase = DefaultAdminUsecase::from_global();
|
let usecase = DefaultAdminUsecase::from_global();
|
||||||
let info = usecase.execute_query_storage_info().await.map_err(S3Error::from)?;
|
let info = usecase.execute_query_storage_info().await.map_err(S3Error::from)?;
|
||||||
|
let response = StorageInfoResponse {
|
||||||
|
info,
|
||||||
|
admin_discovery: system_admin_discovery(&usecase),
|
||||||
|
};
|
||||||
|
|
||||||
let data = serde_json::to_vec(&info).map_err(|e| {
|
let data = serde_json::to_vec(&response).map_err(|e| {
|
||||||
log_system_request_failed!("query_storage_info", "serialize_storage_info_failed", e);
|
log_system_request_failed!("query_storage_info", "serialize_storage_info_failed", e);
|
||||||
S3Error::with_message(S3ErrorCode::InternalError, "failed to serialize storage info")
|
S3Error::with_message(S3ErrorCode::InternalError, "failed to serialize storage info")
|
||||||
})?;
|
})?;
|
||||||
@@ -250,11 +289,14 @@ pub struct RuntimeCapabilitiesSummary {
|
|||||||
pub memory_sampling: CapabilityStatus,
|
pub memory_sampling: CapabilityStatus,
|
||||||
pub platform: CapabilityStatus,
|
pub platform: CapabilityStatus,
|
||||||
pub topology: CapabilityStatus,
|
pub topology: CapabilityStatus,
|
||||||
|
pub cluster_snapshot: CapabilityStatus,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
pub struct RuntimeCapabilitiesResponse {
|
pub struct RuntimeCapabilitiesResponse {
|
||||||
pub summary: RuntimeCapabilitiesSummary,
|
pub summary: RuntimeCapabilitiesSummary,
|
||||||
|
pub cluster_snapshot_path: String,
|
||||||
|
pub cluster_snapshot_summary: Option<CapabilityStatus>,
|
||||||
pub observability: rustfs_storage_api::ObservabilitySnapshot,
|
pub observability: rustfs_storage_api::ObservabilitySnapshot,
|
||||||
pub workload_admission: WorkloadAdmissionRegistrySnapshot,
|
pub workload_admission: WorkloadAdmissionRegistrySnapshot,
|
||||||
pub topology: Option<TopologySnapshot>,
|
pub topology: Option<TopologySnapshot>,
|
||||||
@@ -265,9 +307,11 @@ pub struct RuntimeCapabilitiesHandler {}
|
|||||||
|
|
||||||
pub(crate) async fn build_runtime_capabilities_response()
|
pub(crate) async fn build_runtime_capabilities_response()
|
||||||
-> Result<RuntimeCapabilitiesResponse, rustfs_storage_api::CapabilitySnapshotError> {
|
-> Result<RuntimeCapabilitiesResponse, rustfs_storage_api::CapabilitySnapshotError> {
|
||||||
|
let usecase = DefaultAdminUsecase::from_global();
|
||||||
let observability_provider = RustFsObservabilitySnapshotProvider;
|
let observability_provider = RustFsObservabilitySnapshotProvider;
|
||||||
let observability = observability_provider.observability_snapshot().await?;
|
let observability = observability_provider.observability_snapshot().await?;
|
||||||
let workload_admission = workload_admission_registry_snapshot();
|
let workload_admission = workload_admission_registry_snapshot();
|
||||||
|
let cluster_snapshot_discovery = cluster_snapshot::build_cluster_snapshot_discovery_response().await;
|
||||||
|
|
||||||
let (topology, topology_status) = if let Some(endpoint_pools) = resolve_endpoints_handle() {
|
let (topology, topology_status) = if let Some(endpoint_pools) = resolve_endpoints_handle() {
|
||||||
let topology_provider = EndpointTopologySnapshotProvider::new(endpoint_pools);
|
let topology_provider = EndpointTopologySnapshotProvider::new(endpoint_pools);
|
||||||
@@ -276,10 +320,17 @@ pub(crate) async fn build_runtime_capabilities_response()
|
|||||||
} else {
|
} else {
|
||||||
(None, CapabilityStatus::unknown().with_reason(TOPOLOGY_SNAPSHOT_NOT_AVAILABLE))
|
(None, CapabilityStatus::unknown().with_reason(TOPOLOGY_SNAPSHOT_NOT_AVAILABLE))
|
||||||
};
|
};
|
||||||
let summary = build_runtime_capabilities_summary(&observability, topology.as_ref(), &topology_status);
|
let summary = build_runtime_capabilities_summary(
|
||||||
|
&observability,
|
||||||
|
topology.as_ref(),
|
||||||
|
&topology_status,
|
||||||
|
cluster_snapshot_discovery.summary.as_ref(),
|
||||||
|
);
|
||||||
|
|
||||||
Ok(RuntimeCapabilitiesResponse {
|
Ok(RuntimeCapabilitiesResponse {
|
||||||
summary,
|
summary,
|
||||||
|
cluster_snapshot_path: usecase.cluster_snapshot_route().to_string(),
|
||||||
|
cluster_snapshot_summary: cluster_snapshot_discovery.summary,
|
||||||
observability,
|
observability,
|
||||||
workload_admission,
|
workload_admission,
|
||||||
topology,
|
topology,
|
||||||
@@ -291,6 +342,7 @@ fn build_runtime_capabilities_summary(
|
|||||||
observability: &rustfs_storage_api::ObservabilitySnapshot,
|
observability: &rustfs_storage_api::ObservabilitySnapshot,
|
||||||
topology: Option<&TopologySnapshot>,
|
topology: Option<&TopologySnapshot>,
|
||||||
topology_status: &CapabilityStatus,
|
topology_status: &CapabilityStatus,
|
||||||
|
cluster_snapshot_summary: Option<&CapabilityStatus>,
|
||||||
) -> RuntimeCapabilitiesSummary {
|
) -> RuntimeCapabilitiesSummary {
|
||||||
let userspace_profiling = summarize_named_capability_statuses(
|
let userspace_profiling = summarize_named_capability_statuses(
|
||||||
[
|
[
|
||||||
@@ -347,6 +399,9 @@ fn build_runtime_capabilities_summary(
|
|||||||
memory_sampling,
|
memory_sampling,
|
||||||
platform,
|
platform,
|
||||||
topology: topology_summary,
|
topology: topology_summary,
|
||||||
|
cluster_snapshot: cluster_snapshot_summary.cloned().unwrap_or_else(|| {
|
||||||
|
CapabilityStatus::unknown().with_reason("cluster snapshot is not available before storage endpoint pools initialize")
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -457,10 +512,12 @@ impl Operation for DataUsageInfoHandler {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
OBSERVABILITY_SUMMARY_RESOLVED, TOPOLOGY_SNAPSHOT_NOT_AVAILABLE, TOPOLOGY_SUMMARY_RESOLVED,
|
OBSERVABILITY_SUMMARY_RESOLVED, ServerInfoResponse, TOPOLOGY_SNAPSHOT_NOT_AVAILABLE, TOPOLOGY_SUMMARY_RESOLVED,
|
||||||
build_runtime_capabilities_response, build_runtime_capabilities_summary,
|
build_runtime_capabilities_response, build_runtime_capabilities_summary, system_admin_discovery,
|
||||||
};
|
};
|
||||||
|
use crate::app::admin_usecase::DefaultAdminUsecase;
|
||||||
use rustfs_concurrency::{AdmissionState, WorkloadClass};
|
use rustfs_concurrency::{AdmissionState, WorkloadClass};
|
||||||
|
use rustfs_madmin::{InfoMessage, StorageInfo};
|
||||||
use rustfs_storage_api::{
|
use rustfs_storage_api::{
|
||||||
CapabilityState, CapabilityStatus, MemorySamplingState, ObservabilitySnapshot, PlatformSupport, TopologyCapabilities,
|
CapabilityState, CapabilityStatus, MemorySamplingState, ObservabilitySnapshot, PlatformSupport, TopologyCapabilities,
|
||||||
TopologySnapshot, UserspaceProfilingCapability,
|
TopologySnapshot, UserspaceProfilingCapability,
|
||||||
@@ -477,6 +534,9 @@ mod tests {
|
|||||||
assert_eq!(response.topology_status.reason.as_deref(), Some(TOPOLOGY_SNAPSHOT_NOT_AVAILABLE));
|
assert_eq!(response.topology_status.reason.as_deref(), Some(TOPOLOGY_SNAPSHOT_NOT_AVAILABLE));
|
||||||
assert_eq!(response.summary.topology.state, CapabilityState::Unknown);
|
assert_eq!(response.summary.topology.state, CapabilityState::Unknown);
|
||||||
assert_eq!(response.summary.topology.reason.as_deref(), Some(TOPOLOGY_SNAPSHOT_NOT_AVAILABLE));
|
assert_eq!(response.summary.topology.reason.as_deref(), Some(TOPOLOGY_SNAPSHOT_NOT_AVAILABLE));
|
||||||
|
assert_eq!(response.cluster_snapshot_path, "/rustfs/admin/v4/cluster/snapshot");
|
||||||
|
assert_eq!(response.cluster_snapshot_summary, None);
|
||||||
|
assert_eq!(response.summary.cluster_snapshot.state, CapabilityState::Unknown);
|
||||||
assert_eq!(response.observability.platform.os.as_deref(), Some(std::env::consts::OS));
|
assert_eq!(response.observability.platform.os.as_deref(), Some(std::env::consts::OS));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
response
|
response
|
||||||
@@ -495,6 +555,49 @@ mod tests {
|
|||||||
assert_eq!(response.workload_admission.entries().len(), WorkloadClass::REQUIRED.len());
|
assert_eq!(response.workload_admission.entries().len(), WorkloadClass::REQUIRED.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn server_info_response_exposes_admin_discovery_paths() {
|
||||||
|
let usecase = DefaultAdminUsecase::without_context();
|
||||||
|
let response = ServerInfoResponse {
|
||||||
|
info: InfoMessage {
|
||||||
|
mode: None,
|
||||||
|
domain: None,
|
||||||
|
region: None,
|
||||||
|
sqs_arn: None,
|
||||||
|
deployment_id: None,
|
||||||
|
buckets: None,
|
||||||
|
objects: None,
|
||||||
|
versions: None,
|
||||||
|
delete_markers: None,
|
||||||
|
usage: None,
|
||||||
|
services: None,
|
||||||
|
backend: None,
|
||||||
|
servers: None,
|
||||||
|
pools: None,
|
||||||
|
},
|
||||||
|
admin_discovery: system_admin_discovery(&usecase),
|
||||||
|
};
|
||||||
|
|
||||||
|
let value = serde_json::to_value(response).expect("server info response should serialize");
|
||||||
|
assert_eq!(value["admin_discovery"]["runtimeCapabilities"], "/rustfs/admin/v4/runtime/capabilities");
|
||||||
|
assert_eq!(value["admin_discovery"]["clusterSnapshot"], "/rustfs/admin/v4/cluster/snapshot");
|
||||||
|
assert_eq!(value["admin_discovery"]["extensionsCatalog"], "/rustfs/admin/v4/extensions/catalog");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn storage_info_response_exposes_admin_discovery_paths() {
|
||||||
|
let usecase = DefaultAdminUsecase::without_context();
|
||||||
|
let response = super::StorageInfoResponse {
|
||||||
|
info: StorageInfo::default(),
|
||||||
|
admin_discovery: system_admin_discovery(&usecase),
|
||||||
|
};
|
||||||
|
|
||||||
|
let value = serde_json::to_value(response).expect("storage info response should serialize");
|
||||||
|
assert_eq!(value["admin_discovery"]["runtimeCapabilities"], "/rustfs/admin/v4/runtime/capabilities");
|
||||||
|
assert_eq!(value["admin_discovery"]["clusterSnapshot"], "/rustfs/admin/v4/cluster/snapshot");
|
||||||
|
assert_eq!(value["admin_discovery"]["extensionsCatalog"], "/rustfs/admin/v4/extensions/catalog");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_capabilities_summary_marks_observability_supported_when_groups_are_resolved() {
|
fn runtime_capabilities_summary_marks_observability_supported_when_groups_are_resolved() {
|
||||||
let observability = ObservabilitySnapshot {
|
let observability = ObservabilitySnapshot {
|
||||||
@@ -529,7 +632,12 @@ mod tests {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
let summary = build_runtime_capabilities_summary(&observability, Some(&topology), &CapabilityStatus::supported());
|
let summary = build_runtime_capabilities_summary(
|
||||||
|
&observability,
|
||||||
|
Some(&topology),
|
||||||
|
&CapabilityStatus::supported(),
|
||||||
|
Some(&CapabilityStatus::supported().with_reason("cluster snapshot is available")),
|
||||||
|
);
|
||||||
|
|
||||||
assert_eq!(summary.observability.state, CapabilityState::Supported);
|
assert_eq!(summary.observability.state, CapabilityState::Supported);
|
||||||
assert_eq!(summary.observability.reason.as_deref(), Some(OBSERVABILITY_SUMMARY_RESOLVED));
|
assert_eq!(summary.observability.reason.as_deref(), Some(OBSERVABILITY_SUMMARY_RESOLVED));
|
||||||
@@ -538,6 +646,7 @@ mod tests {
|
|||||||
assert_eq!(summary.platform.state, CapabilityState::Supported);
|
assert_eq!(summary.platform.state, CapabilityState::Supported);
|
||||||
assert_eq!(summary.topology.state, CapabilityState::Supported);
|
assert_eq!(summary.topology.state, CapabilityState::Supported);
|
||||||
assert_eq!(summary.topology.reason.as_deref(), Some(TOPOLOGY_SUMMARY_RESOLVED));
|
assert_eq!(summary.topology.reason.as_deref(), Some(TOPOLOGY_SUMMARY_RESOLVED));
|
||||||
|
assert_eq!(summary.cluster_snapshot.state, CapabilityState::Supported);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -574,13 +683,19 @@ mod tests {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
let summary = build_runtime_capabilities_summary(&observability, Some(&topology), &CapabilityStatus::supported());
|
let summary = build_runtime_capabilities_summary(
|
||||||
|
&observability,
|
||||||
|
Some(&topology),
|
||||||
|
&CapabilityStatus::supported(),
|
||||||
|
Some(&CapabilityStatus::unknown().with_reason("cluster snapshot unresolved")),
|
||||||
|
);
|
||||||
|
|
||||||
assert_eq!(summary.observability.state, CapabilityState::Unknown);
|
assert_eq!(summary.observability.state, CapabilityState::Unknown);
|
||||||
assert_eq!(summary.userspace_profiling.state, CapabilityState::Unknown);
|
assert_eq!(summary.userspace_profiling.state, CapabilityState::Unknown);
|
||||||
assert_eq!(summary.memory_sampling.state, CapabilityState::Unknown);
|
assert_eq!(summary.memory_sampling.state, CapabilityState::Unknown);
|
||||||
assert_eq!(summary.platform.state, CapabilityState::Unknown);
|
assert_eq!(summary.platform.state, CapabilityState::Unknown);
|
||||||
assert_eq!(summary.topology.state, CapabilityState::Unknown);
|
assert_eq!(summary.topology.state, CapabilityState::Unknown);
|
||||||
|
assert_eq!(summary.cluster_snapshot.state, CapabilityState::Unknown);
|
||||||
assert!(
|
assert!(
|
||||||
summary
|
summary
|
||||||
.userspace_profiling
|
.userspace_profiling
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ use crate::admin::{
|
|||||||
auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object},
|
auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object},
|
||||||
router::{AdminOperation, Operation, S3Router},
|
router::{AdminOperation, Operation, S3Router},
|
||||||
};
|
};
|
||||||
|
use crate::app::admin_usecase::DefaultAdminUsecase;
|
||||||
use crate::app::context::{resolve_object_store_handle, resolve_token_signing_key};
|
use crate::app::context::{resolve_object_store_handle, resolve_token_signing_key};
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
use crate::server::{RemoteAddr, TABLE_CATALOG_COMPAT_PREFIX, TABLE_CATALOG_PREFIX};
|
use crate::server::{RemoteAddr, TABLE_CATALOG_COMPAT_PREFIX, TABLE_CATALOG_PREFIX};
|
||||||
@@ -190,6 +191,17 @@ struct CatalogConfigResponse {
|
|||||||
defaults: BTreeMap<&'static str, &'static str>,
|
defaults: BTreeMap<&'static str, &'static str>,
|
||||||
overrides: BTreeMap<&'static str, &'static str>,
|
overrides: BTreeMap<&'static str, &'static str>,
|
||||||
endpoints: Vec<&'static str>,
|
endpoints: Vec<&'static str>,
|
||||||
|
admin_discovery: CatalogAdminDiscovery,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct CatalogAdminDiscovery {
|
||||||
|
#[serde(rename = "runtimeCapabilities")]
|
||||||
|
runtime_capabilities: &'static str,
|
||||||
|
#[serde(rename = "clusterSnapshot")]
|
||||||
|
cluster_snapshot: &'static str,
|
||||||
|
#[serde(rename = "extensionsCatalog")]
|
||||||
|
extensions_catalog: &'static str,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
@@ -975,6 +987,7 @@ fn register_table_catalog_prefix_routes(r: &mut S3Router<AdminOperation>, prefix
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn catalog_config_response() -> CatalogConfigResponse {
|
fn catalog_config_response() -> CatalogConfigResponse {
|
||||||
|
let usecase = DefaultAdminUsecase::from_global();
|
||||||
CatalogConfigResponse {
|
CatalogConfigResponse {
|
||||||
defaults: BTreeMap::from([
|
defaults: BTreeMap::from([
|
||||||
(WAREHOUSE_PROPERTY, DEFAULT_WAREHOUSE_ID),
|
(WAREHOUSE_PROPERTY, DEFAULT_WAREHOUSE_ID),
|
||||||
@@ -983,6 +996,11 @@ fn catalog_config_response() -> CatalogConfigResponse {
|
|||||||
]),
|
]),
|
||||||
overrides: BTreeMap::new(),
|
overrides: BTreeMap::new(),
|
||||||
endpoints: TABLE_CATALOG_ENDPOINTS.to_vec(),
|
endpoints: TABLE_CATALOG_ENDPOINTS.to_vec(),
|
||||||
|
admin_discovery: CatalogAdminDiscovery {
|
||||||
|
runtime_capabilities: usecase.runtime_capabilities_route(),
|
||||||
|
cluster_snapshot: usecase.cluster_snapshot_route(),
|
||||||
|
extensions_catalog: usecase.extensions_catalog_route(),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5345,6 +5363,9 @@ mod tests {
|
|||||||
Some(&TABLE_CATALOG_COMPAT_PREFIX)
|
Some(&TABLE_CATALOG_COMPAT_PREFIX)
|
||||||
);
|
);
|
||||||
assert!(response.overrides.is_empty());
|
assert!(response.overrides.is_empty());
|
||||||
|
assert_eq!(response.admin_discovery.runtime_capabilities, "/rustfs/admin/v4/runtime/capabilities");
|
||||||
|
assert_eq!(response.admin_discovery.cluster_snapshot, "/rustfs/admin/v4/cluster/snapshot");
|
||||||
|
assert_eq!(response.admin_discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog");
|
||||||
assert!(response.endpoints.contains(&"GET /v1/{prefix}/namespaces"));
|
assert!(response.endpoints.contains(&"GET /v1/{prefix}/namespaces"));
|
||||||
assert!(response.endpoints.contains(&"HEAD /v1/{prefix}/namespaces/{namespace}"));
|
assert!(response.endpoints.contains(&"HEAD /v1/{prefix}/namespaces/{namespace}"));
|
||||||
assert!(
|
assert!(
|
||||||
|
|||||||
@@ -31,9 +31,9 @@ mod console_test;
|
|||||||
mod route_registration_test;
|
mod route_registration_test;
|
||||||
|
|
||||||
use handlers::{
|
use handlers::{
|
||||||
audit, bucket_meta, config_admin, extensions, heal, health, kms, module_switch, object_zip_download, oidc, plugins_catalog,
|
audit, bucket_meta, cluster_snapshot, config_admin, extensions, heal, health, kms, module_switch, object_zip_download, oidc,
|
||||||
plugins_instances, pools, profile_admin, quota as quota_handler, rebalance, replication as replication_handler, scanner,
|
plugins_catalog, plugins_instances, pools, profile_admin, quota as quota_handler, rebalance,
|
||||||
site_replication, sts, system, table_catalog, tier, tls_debug, user,
|
replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, user,
|
||||||
};
|
};
|
||||||
use router::{AdminOperation, S3Router};
|
use router::{AdminOperation, S3Router};
|
||||||
use s3s::route::S3Route;
|
use s3s::route::S3Route;
|
||||||
@@ -70,6 +70,7 @@ fn register_admin_routes(r: &mut S3Router<AdminOperation>) -> std::io::Result<()
|
|||||||
scanner::register_scanner_route(r)?;
|
scanner::register_scanner_route(r)?;
|
||||||
audit::register_audit_target_route(r)?;
|
audit::register_audit_target_route(r)?;
|
||||||
module_switch::register_module_switch_route(r)?;
|
module_switch::register_module_switch_route(r)?;
|
||||||
|
cluster_snapshot::register_cluster_snapshot_route(r)?;
|
||||||
extensions::register_extension_route(r)?;
|
extensions::register_extension_route(r)?;
|
||||||
object_zip_download::register_object_zip_download_route(r)?;
|
object_zip_download::register_object_zip_download_route(r)?;
|
||||||
plugins_catalog::register_plugin_catalog_route(r)?;
|
plugins_catalog::register_plugin_catalog_route(r)?;
|
||||||
|
|||||||
@@ -300,6 +300,17 @@ pub(crate) struct PluginCatalogEntry {
|
|||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
pub(crate) struct PluginCatalogResponse {
|
pub(crate) struct PluginCatalogResponse {
|
||||||
pub plugins: Vec<PluginCatalogEntry>,
|
pub plugins: Vec<PluginCatalogEntry>,
|
||||||
|
pub admin_discovery: PluginCatalogAdminDiscovery,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct PluginCatalogAdminDiscovery {
|
||||||
|
#[serde(rename = "runtimeCapabilities")]
|
||||||
|
pub runtime_capabilities: String,
|
||||||
|
#[serde(rename = "clusterSnapshot")]
|
||||||
|
pub cluster_snapshot: String,
|
||||||
|
#[serde(rename = "extensionsCatalog")]
|
||||||
|
pub extensions_catalog: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
@@ -366,10 +377,10 @@ pub(crate) struct PluginInstancesResponse {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
PluginArtifactContract, PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse, PluginContractDomain,
|
PluginArtifactContract, PluginCatalogAdminDiscovery, PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse,
|
||||||
PluginContractEntrypointKind, PluginContractPackaging, PluginDistributionContract, PluginInstanceDetail,
|
PluginContractDomain, PluginContractEntrypointKind, PluginContractPackaging, PluginDistributionContract,
|
||||||
PluginInstanceDiagnostic, PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry,
|
PluginInstanceDetail, PluginInstanceDiagnostic, PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount,
|
||||||
PluginInstanceSource, PluginInstancesResponse, PluginRuntimeContract, PluginRuntimeTransport,
|
PluginInstanceEntry, PluginInstanceSource, PluginInstancesResponse, PluginRuntimeContract, PluginRuntimeTransport,
|
||||||
};
|
};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@@ -400,6 +411,11 @@ mod tests {
|
|||||||
}],
|
}],
|
||||||
installation: None,
|
installation: None,
|
||||||
}],
|
}],
|
||||||
|
admin_discovery: PluginCatalogAdminDiscovery {
|
||||||
|
runtime_capabilities: "/rustfs/admin/v4/runtime/capabilities".to_string(),
|
||||||
|
cluster_snapshot: "/rustfs/admin/v4/cluster/snapshot".to_string(),
|
||||||
|
extensions_catalog: "/rustfs/admin/v4/extensions/catalog".to_string(),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
let value = serde_json::to_value(response).expect("catalog response should serialize");
|
let value = serde_json::to_value(response).expect("catalog response should serialize");
|
||||||
@@ -427,7 +443,12 @@ mod tests {
|
|||||||
"subsystem": "notify_webhook",
|
"subsystem": "notify_webhook",
|
||||||
"valid_fields": ["endpoint", "auth_token"]
|
"valid_fields": ["endpoint", "auth_token"]
|
||||||
}]
|
}]
|
||||||
}]
|
}],
|
||||||
|
"admin_discovery": {
|
||||||
|
"runtimeCapabilities": "/rustfs/admin/v4/runtime/capabilities",
|
||||||
|
"clusterSnapshot": "/rustfs/admin/v4/cluster/snapshot",
|
||||||
|
"extensionsCatalog": "/rustfs/admin/v4/extensions/catalog"
|
||||||
|
}
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -344,6 +344,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
|||||||
RouteRiskLevel::Sensitive,
|
RouteRiskLevel::Sensitive,
|
||||||
),
|
),
|
||||||
admin(HttpMethod::Put, "/rustfs/admin/v3/module-switches", CONFIG_UPDATE, RouteRiskLevel::High),
|
admin(HttpMethod::Put, "/rustfs/admin/v3/module-switches", CONFIG_UPDATE, RouteRiskLevel::High),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/rustfs/admin/v4/cluster/snapshot",
|
||||||
|
SERVER_INFO,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
admin(
|
admin(
|
||||||
HttpMethod::Get,
|
HttpMethod::Get,
|
||||||
"/rustfs/admin/v4/extensions/catalog",
|
"/rustfs/admin/v4/extensions/catalog",
|
||||||
|
|||||||
@@ -225,6 +225,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
|||||||
),
|
),
|
||||||
admin_route(Method::GET, "/v3/module-switches"),
|
admin_route(Method::GET, "/v3/module-switches"),
|
||||||
admin_route(Method::PUT, "/v3/module-switches"),
|
admin_route(Method::PUT, "/v3/module-switches"),
|
||||||
|
admin_route(Method::GET, "/v4/cluster/snapshot"),
|
||||||
admin_route(Method::GET, "/v4/extensions/catalog"),
|
admin_route(Method::GET, "/v4/extensions/catalog"),
|
||||||
admin_route(Method::GET, "/v4/extensions/instances"),
|
admin_route(Method::GET, "/v4/extensions/instances"),
|
||||||
admin_route(Method::GET, "/v4/runtime/capabilities"),
|
admin_route(Method::GET, "/v4/runtime/capabilities"),
|
||||||
@@ -724,6 +725,7 @@ fn test_register_routes_cover_representative_admin_paths() {
|
|||||||
assert_route(&router, Method::GET, &admin_path("/v3/audit/target/list"));
|
assert_route(&router, Method::GET, &admin_path("/v3/audit/target/list"));
|
||||||
assert_route(&router, Method::GET, &admin_path("/v3/module-switches"));
|
assert_route(&router, Method::GET, &admin_path("/v3/module-switches"));
|
||||||
assert_route(&router, Method::PUT, &admin_path("/v3/module-switches"));
|
assert_route(&router, Method::PUT, &admin_path("/v3/module-switches"));
|
||||||
|
assert_route(&router, Method::GET, &admin_path("/v4/cluster/snapshot"));
|
||||||
assert_route(&router, Method::GET, &admin_path("/v4/extensions/catalog"));
|
assert_route(&router, Method::GET, &admin_path("/v4/extensions/catalog"));
|
||||||
assert_route(&router, Method::GET, &admin_path("/v4/extensions/instances"));
|
assert_route(&router, Method::GET, &admin_path("/v4/extensions/instances"));
|
||||||
assert_route(&router, Method::GET, &admin_path("/v4/runtime/capabilities"));
|
assert_route(&router, Method::GET, &admin_path("/v4/runtime/capabilities"));
|
||||||
|
|||||||
@@ -19,8 +19,11 @@ use super::EndpointServerPools;
|
|||||||
use super::get_server_info;
|
use super::get_server_info;
|
||||||
use super::{PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free};
|
use super::{PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free};
|
||||||
use super::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
|
use super::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
|
||||||
use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
|
use crate::app::context::{
|
||||||
|
AppContext, get_global_app_context, resolve_endpoints_handle, resolve_object_store_handle_for_context,
|
||||||
|
};
|
||||||
use crate::capacity::resolve_admin_used_capacity;
|
use crate::capacity::resolve_admin_used_capacity;
|
||||||
|
use crate::cluster_snapshot::{ClusterReadOnlySnapshot, collect_cluster_read_only_snapshot};
|
||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
use crate::server::{DependencyReadiness, collect_dependency_readiness as collect_runtime_dependency_readiness};
|
use crate::server::{DependencyReadiness, collect_dependency_readiness as collect_runtime_dependency_readiness};
|
||||||
use rustfs_data_usage::DataUsageInfo;
|
use rustfs_data_usage::DataUsageInfo;
|
||||||
@@ -31,6 +34,9 @@ use std::sync::Arc;
|
|||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
|
|
||||||
pub type AdminUsecaseResult<T> = Result<T, ApiError>;
|
pub type AdminUsecaseResult<T> = Result<T, ApiError>;
|
||||||
|
pub const ADMIN_CLUSTER_SNAPSHOT_ROUTE: &str = "/rustfs/admin/v4/cluster/snapshot";
|
||||||
|
pub const ADMIN_EXTENSIONS_CATALOG_ROUTE: &str = "/rustfs/admin/v4/extensions/catalog";
|
||||||
|
pub const ADMIN_RUNTIME_CAPABILITIES_ROUTE: &str = "/rustfs/admin/v4/runtime/capabilities";
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||||
pub struct QueryServerInfoRequest {
|
pub struct QueryServerInfoRequest {
|
||||||
@@ -524,6 +530,23 @@ impl DefaultAdminUsecase {
|
|||||||
pub async fn execute_collect_dependency_readiness(&self) -> DependencyReadiness {
|
pub async fn execute_collect_dependency_readiness(&self) -> DependencyReadiness {
|
||||||
collect_runtime_dependency_readiness().await
|
collect_runtime_dependency_readiness().await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn execute_collect_cluster_read_only_snapshot(&self) -> Option<ClusterReadOnlySnapshot> {
|
||||||
|
let endpoint_pools = resolve_endpoints_handle()?;
|
||||||
|
collect_cluster_read_only_snapshot(&endpoint_pools).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cluster_snapshot_route(&self) -> &'static str {
|
||||||
|
ADMIN_CLUSTER_SNAPSHOT_ROUTE
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn runtime_capabilities_route(&self) -> &'static str {
|
||||||
|
ADMIN_RUNTIME_CAPABILITIES_ROUTE
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn extensions_catalog_route(&self) -> &'static str {
|
||||||
|
ADMIN_EXTENSIONS_CATALOG_ROUTE
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -557,6 +580,24 @@ mod tests {
|
|||||||
let _ = readiness.iam_ready;
|
let _ = readiness.iam_ready;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn execute_collect_cluster_read_only_snapshot_returns_none_without_context() {
|
||||||
|
let usecase = DefaultAdminUsecase::without_context();
|
||||||
|
|
||||||
|
let snapshot = usecase.execute_collect_cluster_read_only_snapshot().await;
|
||||||
|
|
||||||
|
assert!(snapshot.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn admin_usecase_exposes_stable_discovery_routes() {
|
||||||
|
let usecase = DefaultAdminUsecase::without_context();
|
||||||
|
|
||||||
|
assert_eq!(usecase.cluster_snapshot_route(), "/rustfs/admin/v4/cluster/snapshot");
|
||||||
|
assert_eq!(usecase.runtime_capabilities_route(), "/rustfs/admin/v4/runtime/capabilities");
|
||||||
|
assert_eq!(usecase.extensions_catalog_route(), "/rustfs/admin/v4/extensions/catalog");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn admin_query_pool_status_by_id_rejects_non_numeric_index() {
|
fn admin_query_pool_status_by_id_rejects_non_numeric_index() {
|
||||||
assert_eq!(DefaultAdminUsecase::parse_pool_idx_by_id("pool-a", 4), None);
|
assert_eq!(DefaultAdminUsecase::parse_pool_idx_by_id("pool-a", 4), None);
|
||||||
|
|||||||
@@ -0,0 +1,304 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use crate::runtime_capabilities::runtime_observability_snapshot;
|
||||||
|
use crate::server::{
|
||||||
|
DependencyReadiness, DependencyReadinessReport, ReadinessDegradedReason, snapshot_dependency_readiness_report,
|
||||||
|
};
|
||||||
|
use crate::storage::EndpointServerPools;
|
||||||
|
use crate::workload_admission::workload_admission_registry_snapshot;
|
||||||
|
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot};
|
||||||
|
use rustfs_ecstore::api::cluster::{
|
||||||
|
ClusterControlPlane, ClusterControlPlaneSnapshot, ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot,
|
||||||
|
ClusterPeerHealthSnapshot, ClusterPoolStateSnapshot,
|
||||||
|
};
|
||||||
|
use rustfs_storage_api::{ObservabilitySnapshot, TopologySnapshot};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct ClusterReadOnlySnapshot {
|
||||||
|
pub topology: TopologySnapshot,
|
||||||
|
pub membership: ClusterMembershipSnapshot,
|
||||||
|
pub pool_state: ClusterPoolStateSnapshot,
|
||||||
|
pub local_storage: ClusterLocalNodeStorageSnapshot,
|
||||||
|
pub peer_health: ClusterPeerHealthSnapshot,
|
||||||
|
pub observability: ObservabilitySnapshot,
|
||||||
|
pub workload_admission: WorkloadAdmissionRegistrySnapshot,
|
||||||
|
pub runtime_status: ClusterRuntimeStatusSnapshot,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct ClusterRuntimeStatusSnapshot {
|
||||||
|
pub readiness: DependencyReadiness,
|
||||||
|
pub state: ClusterRuntimeReadinessState,
|
||||||
|
pub degraded_reasons: Vec<ReadinessDegradedReason>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum ClusterRuntimeReadinessState {
|
||||||
|
Ready,
|
||||||
|
Degraded,
|
||||||
|
Unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ClusterRuntimeReadinessState {
|
||||||
|
fn from_report(report: &DependencyReadinessReport) -> Self {
|
||||||
|
if report.readiness.storage_ready && report.readiness.iam_ready && report.readiness.lock_quorum_ready {
|
||||||
|
Self::Ready
|
||||||
|
} else if report.degraded_reasons.is_empty() {
|
||||||
|
Self::Unknown
|
||||||
|
} else {
|
||||||
|
Self::Degraded
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ClusterRuntimeStatusSnapshot {
|
||||||
|
pub fn from_readiness_report(report: DependencyReadinessReport) -> Self {
|
||||||
|
let state = ClusterRuntimeReadinessState::from_report(&report);
|
||||||
|
Self {
|
||||||
|
readiness: report.readiness,
|
||||||
|
state,
|
||||||
|
degraded_reasons: report.degraded_reasons,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cluster_read_only_snapshot_from_endpoint_pools(
|
||||||
|
endpoint_pools: &EndpointServerPools,
|
||||||
|
runtime_status: ClusterRuntimeStatusSnapshot,
|
||||||
|
) -> ClusterReadOnlySnapshot {
|
||||||
|
let control_plane = ClusterControlPlane::new(endpoint_pools.clone());
|
||||||
|
let control_plane_snapshot = control_plane.read_snapshot();
|
||||||
|
cluster_read_only_snapshot_from_control_plane(control_plane_snapshot, runtime_status)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cluster_read_only_snapshot_from_control_plane(
|
||||||
|
control_plane: ClusterControlPlaneSnapshot,
|
||||||
|
runtime_status: ClusterRuntimeStatusSnapshot,
|
||||||
|
) -> ClusterReadOnlySnapshot {
|
||||||
|
ClusterReadOnlySnapshot {
|
||||||
|
topology: control_plane.topology,
|
||||||
|
membership: control_plane.membership,
|
||||||
|
pool_state: control_plane.pool_state,
|
||||||
|
local_storage: control_plane.local_storage,
|
||||||
|
peer_health: control_plane.peer_health,
|
||||||
|
observability: runtime_observability_snapshot(),
|
||||||
|
workload_admission: workload_admission_registry_snapshot(),
|
||||||
|
runtime_status,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn collect_cluster_read_only_snapshot(endpoint_pools: &EndpointServerPools) -> Option<ClusterReadOnlySnapshot> {
|
||||||
|
let runtime_status = ClusterRuntimeStatusSnapshot::from_readiness_report(snapshot_dependency_readiness_report().await);
|
||||||
|
Some(cluster_read_only_snapshot_from_endpoint_pools(endpoint_pools, runtime_status))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cluster_has_actionable_pressure(snapshot: &ClusterReadOnlySnapshot) -> bool {
|
||||||
|
snapshot.runtime_status.state == ClusterRuntimeReadinessState::Degraded
|
||||||
|
|| snapshot
|
||||||
|
.workload_admission
|
||||||
|
.entries()
|
||||||
|
.iter()
|
||||||
|
.any(|entry| entry.state != AdmissionState::Open)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::storage::{Endpoint, Endpoints, PoolEndpoints};
|
||||||
|
use rustfs_concurrency::{WorkloadAdmissionSnapshot, WorkloadClass};
|
||||||
|
use rustfs_storage_api::{CapabilityState, CapabilityStatus, DiskCapabilities, TopologyCapabilities};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_status_snapshot_maps_ready_and_degraded_reports() {
|
||||||
|
let ready = ClusterRuntimeStatusSnapshot::from_readiness_report(DependencyReadinessReport {
|
||||||
|
readiness: DependencyReadiness {
|
||||||
|
storage_ready: true,
|
||||||
|
iam_ready: true,
|
||||||
|
lock_quorum_ready: true,
|
||||||
|
},
|
||||||
|
degraded_reasons: Vec::new(),
|
||||||
|
});
|
||||||
|
assert_eq!(ready.state, ClusterRuntimeReadinessState::Ready);
|
||||||
|
|
||||||
|
let degraded = ClusterRuntimeStatusSnapshot::from_readiness_report(DependencyReadinessReport {
|
||||||
|
readiness: DependencyReadiness {
|
||||||
|
storage_ready: false,
|
||||||
|
iam_ready: true,
|
||||||
|
lock_quorum_ready: true,
|
||||||
|
},
|
||||||
|
degraded_reasons: vec![ReadinessDegradedReason::StorageQuorumUnavailable],
|
||||||
|
});
|
||||||
|
assert_eq!(degraded.state, ClusterRuntimeReadinessState::Degraded);
|
||||||
|
assert_eq!(degraded.degraded_reasons, vec![ReadinessDegradedReason::StorageQuorumUnavailable]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cluster_snapshot_preserves_read_only_sections_and_partial_states() {
|
||||||
|
let endpoint_pools = sample_endpoint_pools();
|
||||||
|
let runtime_status = ClusterRuntimeStatusSnapshot::from_readiness_report(DependencyReadinessReport {
|
||||||
|
readiness: DependencyReadiness {
|
||||||
|
storage_ready: false,
|
||||||
|
iam_ready: true,
|
||||||
|
lock_quorum_ready: false,
|
||||||
|
},
|
||||||
|
degraded_reasons: vec![ReadinessDegradedReason::StorageAndLockUnavailable],
|
||||||
|
});
|
||||||
|
|
||||||
|
let snapshot = cluster_read_only_snapshot_from_endpoint_pools(&endpoint_pools, runtime_status);
|
||||||
|
|
||||||
|
assert_eq!(snapshot.topology.pools.len(), 1);
|
||||||
|
assert_eq!(snapshot.membership.nodes.len(), 2);
|
||||||
|
assert_eq!(snapshot.pool_state.pools[0].endpoint_count, 4);
|
||||||
|
assert_eq!(snapshot.local_storage.nodes.len(), 1);
|
||||||
|
assert_eq!(snapshot.peer_health.peers.len(), 2);
|
||||||
|
assert_eq!(snapshot.runtime_status.state, ClusterRuntimeReadinessState::Degraded);
|
||||||
|
assert_eq!(
|
||||||
|
snapshot.runtime_status.degraded_reasons,
|
||||||
|
vec![ReadinessDegradedReason::StorageAndLockUnavailable]
|
||||||
|
);
|
||||||
|
assert_eq!(snapshot.peer_health.peers[0].status.state, CapabilityState::Unknown);
|
||||||
|
if cfg!(target_os = "linux") {
|
||||||
|
assert_eq!(snapshot.observability.platform.numa.state, CapabilityState::Unknown);
|
||||||
|
} else {
|
||||||
|
assert_eq!(snapshot.observability.platform.numa.state, CapabilityState::Unsupported);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
snapshot
|
||||||
|
.workload_admission
|
||||||
|
.entries()
|
||||||
|
.iter()
|
||||||
|
.any(|entry| entry.state == AdmissionState::Unknown)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cluster_pressure_helper_detects_degraded_runtime_or_non_open_admission() {
|
||||||
|
let no_pressure = ClusterReadOnlySnapshot {
|
||||||
|
topology: TopologySnapshot::default(),
|
||||||
|
membership: ClusterMembershipSnapshot::default(),
|
||||||
|
pool_state: ClusterPoolStateSnapshot::default(),
|
||||||
|
local_storage: ClusterLocalNodeStorageSnapshot::default(),
|
||||||
|
peer_health: ClusterPeerHealthSnapshot::default(),
|
||||||
|
observability: ObservabilitySnapshot::default(),
|
||||||
|
workload_admission: WorkloadAdmissionRegistrySnapshot::new(vec![WorkloadAdmissionSnapshot::new(
|
||||||
|
WorkloadClass::ForegroundRead,
|
||||||
|
AdmissionState::Open,
|
||||||
|
)]),
|
||||||
|
runtime_status: ClusterRuntimeStatusSnapshot {
|
||||||
|
readiness: DependencyReadiness {
|
||||||
|
storage_ready: true,
|
||||||
|
iam_ready: true,
|
||||||
|
lock_quorum_ready: true,
|
||||||
|
},
|
||||||
|
state: ClusterRuntimeReadinessState::Ready,
|
||||||
|
degraded_reasons: Vec::new(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
assert!(!cluster_has_actionable_pressure(&no_pressure));
|
||||||
|
|
||||||
|
let runtime_pressure = ClusterReadOnlySnapshot {
|
||||||
|
runtime_status: ClusterRuntimeStatusSnapshot {
|
||||||
|
readiness: DependencyReadiness {
|
||||||
|
storage_ready: false,
|
||||||
|
iam_ready: true,
|
||||||
|
lock_quorum_ready: true,
|
||||||
|
},
|
||||||
|
state: ClusterRuntimeReadinessState::Degraded,
|
||||||
|
degraded_reasons: vec![ReadinessDegradedReason::StorageQuorumUnavailable],
|
||||||
|
},
|
||||||
|
..no_pressure.clone()
|
||||||
|
};
|
||||||
|
assert!(cluster_has_actionable_pressure(&runtime_pressure));
|
||||||
|
|
||||||
|
let admission_pressure = ClusterReadOnlySnapshot {
|
||||||
|
workload_admission: WorkloadAdmissionRegistrySnapshot::new(vec![WorkloadAdmissionSnapshot::new(
|
||||||
|
WorkloadClass::Repair,
|
||||||
|
AdmissionState::Unknown,
|
||||||
|
)]),
|
||||||
|
..no_pressure
|
||||||
|
};
|
||||||
|
assert!(cluster_has_actionable_pressure(&admission_pressure));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn topology_capability_states_can_remain_unknown_disabled_and_unsupported() {
|
||||||
|
let endpoint_pools = sample_endpoint_pools();
|
||||||
|
let mut control_plane = ClusterControlPlane::new(endpoint_pools).read_snapshot();
|
||||||
|
control_plane.topology.capabilities = TopologyCapabilities {
|
||||||
|
profiling: CapabilityStatus::disabled().with_reason("disabled for test"),
|
||||||
|
numa: CapabilityStatus::unsupported().with_reason("unsupported for test"),
|
||||||
|
failure_domain_labels: CapabilityStatus::unknown().with_reason("unknown for test"),
|
||||||
|
media_labels: CapabilityStatus::supported(),
|
||||||
|
};
|
||||||
|
|
||||||
|
for pool in &mut control_plane.topology.pools {
|
||||||
|
for set in &mut pool.sets {
|
||||||
|
for disk in &mut set.disks {
|
||||||
|
disk.capabilities = DiskCapabilities {
|
||||||
|
media_type: CapabilityStatus::unknown().with_reason("media unknown"),
|
||||||
|
failure_domain: CapabilityStatus::unknown().with_reason("fd unknown"),
|
||||||
|
numa: CapabilityStatus::unsupported().with_reason("numa unsupported"),
|
||||||
|
profiling: CapabilityStatus::disabled().with_reason("profiling disabled"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let snapshot = cluster_read_only_snapshot_from_control_plane(
|
||||||
|
control_plane,
|
||||||
|
ClusterRuntimeStatusSnapshot {
|
||||||
|
readiness: DependencyReadiness {
|
||||||
|
storage_ready: true,
|
||||||
|
iam_ready: true,
|
||||||
|
lock_quorum_ready: true,
|
||||||
|
},
|
||||||
|
state: ClusterRuntimeReadinessState::Ready,
|
||||||
|
degraded_reasons: Vec::new(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(snapshot.topology.capabilities.profiling.state, CapabilityState::Disabled);
|
||||||
|
assert_eq!(snapshot.topology.capabilities.numa.state, CapabilityState::Unsupported);
|
||||||
|
assert_eq!(snapshot.topology.capabilities.failure_domain_labels.state, CapabilityState::Unknown);
|
||||||
|
assert_eq!(
|
||||||
|
snapshot.topology.pools[0].sets[0].disks[0].capabilities.profiling.state,
|
||||||
|
CapabilityState::Disabled
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_endpoint_pools() -> EndpointServerPools {
|
||||||
|
let endpoints = (0..4)
|
||||||
|
.map(|index| {
|
||||||
|
let host = if index < 2 { "node1.example" } else { "node2.example" };
|
||||||
|
let mut endpoint =
|
||||||
|
Endpoint::try_from(format!("http://{host}:9000/export{index}").as_str()).expect("url endpoint");
|
||||||
|
endpoint.set_pool_index(0);
|
||||||
|
endpoint.set_set_index(index / 2);
|
||||||
|
endpoint.set_disk_index(index % 2);
|
||||||
|
endpoint.is_local = index < 2;
|
||||||
|
endpoint
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
EndpointServerPools::from(vec![PoolEndpoints {
|
||||||
|
legacy: false,
|
||||||
|
set_count: 2,
|
||||||
|
drives_per_set: 2,
|
||||||
|
endpoints: Endpoints::from(endpoints),
|
||||||
|
cmd_line: "http://node{1...2}.example:9000/export{0...3}".to_owned(),
|
||||||
|
platform: "OS: test | Arch: test".to_owned(),
|
||||||
|
}])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -56,6 +56,7 @@ pub mod app;
|
|||||||
pub mod auth;
|
pub mod auth;
|
||||||
pub mod auth_keystone;
|
pub mod auth_keystone;
|
||||||
pub mod capacity;
|
pub mod capacity;
|
||||||
|
pub mod cluster_snapshot;
|
||||||
pub mod config;
|
pub mod config;
|
||||||
pub mod delete_tail_activity;
|
pub mod delete_tail_activity;
|
||||||
pub mod embedded;
|
pub mod embedded;
|
||||||
|
|||||||
@@ -62,6 +62,7 @@ pub(crate) use readiness::ReadinessGateLayer;
|
|||||||
pub(crate) use readiness::collect_dependency_readiness;
|
pub(crate) use readiness::collect_dependency_readiness;
|
||||||
pub(crate) use readiness::collect_dependency_readiness_report;
|
pub(crate) use readiness::collect_dependency_readiness_report;
|
||||||
pub use readiness::publish_ready_when_runtime_ready;
|
pub use readiness::publish_ready_when_runtime_ready;
|
||||||
|
pub(crate) use readiness::snapshot_dependency_readiness_report;
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug)]
|
#[derive(Clone, Copy, Debug)]
|
||||||
pub struct RemoteAddr(pub std::net::SocketAddr);
|
pub struct RemoteAddr(pub std::net::SocketAddr);
|
||||||
|
|||||||
@@ -433,6 +433,13 @@ fn record_readiness_report(report: &DependencyReadinessReport) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn dependency_readiness_report_from_readiness(readiness: DependencyReadiness) -> DependencyReadinessReport {
|
||||||
|
DependencyReadinessReport {
|
||||||
|
degraded_reasons: degraded_reasons(readiness.storage_ready, readiness.iam_ready, readiness.lock_quorum_ready),
|
||||||
|
readiness,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn collect_dependency_readiness() -> DependencyReadiness {
|
pub async fn collect_dependency_readiness() -> DependencyReadiness {
|
||||||
collect_dependency_readiness_report().await.readiness
|
collect_dependency_readiness_report().await.readiness
|
||||||
}
|
}
|
||||||
@@ -453,14 +460,21 @@ pub async fn collect_dependency_readiness_report() -> DependencyReadinessReport
|
|||||||
iam_ready: iam_ready_raw,
|
iam_ready: iam_ready_raw,
|
||||||
lock_quorum_ready: lock_quorum_status.ready,
|
lock_quorum_ready: lock_quorum_status.ready,
|
||||||
};
|
};
|
||||||
let report = DependencyReadinessReport {
|
let report = dependency_readiness_report_from_readiness(readiness);
|
||||||
degraded_reasons: degraded_reasons(readiness.storage_ready, iam_ready_raw, readiness.lock_quorum_ready),
|
|
||||||
readiness,
|
|
||||||
};
|
|
||||||
record_readiness_report(&report);
|
record_readiness_report(&report);
|
||||||
report
|
report
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn snapshot_dependency_readiness_report() -> DependencyReadinessReport {
|
||||||
|
let readiness = DependencyReadiness {
|
||||||
|
storage_ready: collect_storage_readiness_uncached().await,
|
||||||
|
iam_ready: resolve_iam_ready(),
|
||||||
|
lock_quorum_ready: collect_lock_quorum_status_uncached().await.ready,
|
||||||
|
};
|
||||||
|
|
||||||
|
dependency_readiness_report_from_readiness(readiness)
|
||||||
|
}
|
||||||
|
|
||||||
async fn collect_lock_quorum_status() -> LockQuorumStatus {
|
async fn collect_lock_quorum_status() -> LockQuorumStatus {
|
||||||
if let Some(cached) = load_cached_lock_quorum_status().await {
|
if let Some(cached) = load_cached_lock_quorum_status().await {
|
||||||
cached
|
cached
|
||||||
@@ -481,10 +495,7 @@ async fn collect_dependency_readiness_uncached() -> DependencyReadiness {
|
|||||||
iam_ready: iam_ready_raw,
|
iam_ready: iam_ready_raw,
|
||||||
lock_quorum_ready: lock_quorum_status.ready,
|
lock_quorum_ready: lock_quorum_status.ready,
|
||||||
};
|
};
|
||||||
let report = DependencyReadinessReport {
|
let report = dependency_readiness_report_from_readiness(readiness);
|
||||||
degraded_reasons: degraded_reasons(readiness.storage_ready, iam_ready_raw, readiness.lock_quorum_ready),
|
|
||||||
readiness,
|
|
||||||
};
|
|
||||||
record_readiness_report(&report);
|
record_readiness_report(&report);
|
||||||
report.readiness
|
report.readiness
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user