mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-08 06:13:14 +00:00
feat: harden runtime capability snapshots (#3784)
* feat(admin): expose runtime capability snapshots * feat(runtime): refine workload admission snapshots * test(ci): align architecture migration checks * build(deps): bump memmap2 for RUSTSEC-2026-0186 * build(deps): refresh cargo deny lockfile
This commit is contained in:
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::admin::{
|
||||
auth::validate_admin_request,
|
||||
handlers::plugins_instances,
|
||||
handlers::{plugins_instances, system},
|
||||
plugin_contract::{
|
||||
PluginContractDomain, PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry,
|
||||
PluginInstanceSource, PluginOperationalStateContract,
|
||||
@@ -31,6 +31,7 @@ use rustfs_extension_schema::{
|
||||
OPS_PROFILER_CAPABILITY, OpsDiagnosticsContract, OpsProfilerContract,
|
||||
};
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use rustfs_storage_api::CapabilityStatus;
|
||||
use rustfs_targets::{
|
||||
OpsDiagnosticsRegistry, OpsProfilerRegistry, TargetPluginExternalFlowGate, TargetPluginExternalFlowGateStatus,
|
||||
builtin_extension_schemas, builtin_ops_diagnostics_contract, builtin_ops_diagnostics_extension_schema,
|
||||
@@ -80,6 +81,10 @@ where
|
||||
pub runtime: ExtensionRuntimeContract,
|
||||
pub disabled_by_default: bool,
|
||||
pub startup_fatal: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub runtime_capability_path: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub runtime_capability_summary: Option<CapabilityStatus>,
|
||||
pub contract: T,
|
||||
}
|
||||
|
||||
@@ -112,20 +117,23 @@ pub(crate) struct ExtensionInstancesResponse {
|
||||
pub next_marker: Option<String>,
|
||||
}
|
||||
|
||||
fn build_extension_catalog_response() -> ExtensionCatalogResponse {
|
||||
async fn build_extension_catalog_response() -> Result<ExtensionCatalogResponse, rustfs_storage_api::CapabilitySnapshotError> {
|
||||
let mut extensions = builtin_extension_schemas();
|
||||
let example = example_external_webhook_plugin();
|
||||
extensions.push(target_marketplace_extension_schema(&example.manifest));
|
||||
extensions.sort_by(|a, b| a.extension_id.cmp(&b.extension_id));
|
||||
let runtime_capabilities = system::build_runtime_capabilities_response().await?;
|
||||
|
||||
ExtensionCatalogResponse {
|
||||
Ok(ExtensionCatalogResponse {
|
||||
extensions,
|
||||
runtime_capabilities: build_extension_runtime_capabilities_response(),
|
||||
runtime_capabilities: build_extension_runtime_capabilities_response(&runtime_capabilities),
|
||||
external_plugin_flow: TargetPluginExternalFlowGate::default().status(),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn build_extension_runtime_capabilities_response() -> ExtensionRuntimeCapabilitiesResponse {
|
||||
fn build_extension_runtime_capabilities_response(
|
||||
runtime_capabilities: &system::RuntimeCapabilitiesResponse,
|
||||
) -> ExtensionRuntimeCapabilitiesResponse {
|
||||
let ops_diagnostics_schema = builtin_ops_diagnostics_extension_schema();
|
||||
let ops_diagnostics_contract = builtin_ops_diagnostics_contract();
|
||||
let mut ops_diagnostics_registry = OpsDiagnosticsRegistry::new();
|
||||
@@ -143,6 +151,7 @@ fn build_extension_runtime_capabilities_response() -> ExtensionRuntimeCapabiliti
|
||||
.register_schema(&ops_profiler_schema, &ops_profiler_contract)
|
||||
.is_ok()
|
||||
);
|
||||
let runtime_capability_path = Some(format!("{}{}", ADMIN_PREFIX, system::RUNTIME_CAPABILITIES_ROUTE_SUFFIX));
|
||||
|
||||
ExtensionRuntimeCapabilitiesResponse {
|
||||
ops_diagnostics: ExtensionRuntimeCapabilityResponse {
|
||||
@@ -151,6 +160,8 @@ fn build_extension_runtime_capabilities_response() -> ExtensionRuntimeCapabiliti
|
||||
runtime: ops_diagnostics_schema.runtime,
|
||||
disabled_by_default: ops_diagnostics_schema.disabled_by_default,
|
||||
startup_fatal: false,
|
||||
runtime_capability_path: runtime_capability_path.clone(),
|
||||
runtime_capability_summary: Some(runtime_capabilities.summary.observability.clone()),
|
||||
contract: ops_diagnostics_contract,
|
||||
},
|
||||
ops_profiler: ExtensionRuntimeCapabilityResponse {
|
||||
@@ -159,6 +170,8 @@ fn build_extension_runtime_capabilities_response() -> ExtensionRuntimeCapabiliti
|
||||
runtime: ops_profiler_schema.runtime,
|
||||
disabled_by_default: ops_profiler_schema.disabled_by_default,
|
||||
startup_fatal: false,
|
||||
runtime_capability_path,
|
||||
runtime_capability_summary: Some(runtime_capabilities.summary.userspace_profiling.clone()),
|
||||
contract: ops_profiler_contract,
|
||||
},
|
||||
}
|
||||
@@ -240,7 +253,10 @@ pub struct GetExtensionCatalogHandler {}
|
||||
impl Operation for GetExtensionCatalogHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
authorize_extension_catalog_request(&req).await?;
|
||||
build_json_response(StatusCode::OK, &build_extension_catalog_response(), req.headers.get("x-request-id"))
|
||||
let response = build_extension_catalog_response()
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to build extension catalog response: {}", err))?;
|
||||
build_json_response(StatusCode::OK, &response, req.headers.get("x-request-id"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -267,10 +283,12 @@ impl Operation for ListExtensionInstancesHandler {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{build_extension_catalog_response, map_extension_instance};
|
||||
use crate::admin::handlers::system::{self, RUNTIME_CAPABILITIES_ROUTE_SUFFIX};
|
||||
use crate::admin::plugin_contract::{
|
||||
PluginContractDomain, PluginEnableState, PluginInstallState, PluginInstanceEntry, PluginInstanceSource,
|
||||
PluginOperationalRuntimeState, PluginOperationalStateContract,
|
||||
};
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
use rustfs_extension_schema::{
|
||||
ExtensionKind, ExtensionRuntimeBoundary, OPS_DIAGNOSTICS_CAPABILITY, OPS_PROFILER_CAPABILITY, validate_extension_schemas,
|
||||
validate_ops_diagnostics_contract, validate_ops_profiler_contract,
|
||||
@@ -314,9 +332,11 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_catalog_exposes_valid_extension_schemas() {
|
||||
let response = build_extension_catalog_response();
|
||||
#[tokio::test]
|
||||
async fn extension_catalog_exposes_valid_extension_schemas() {
|
||||
let response = build_extension_catalog_response()
|
||||
.await
|
||||
.expect("extension catalog response should build");
|
||||
let webhook = response
|
||||
.extensions
|
||||
.iter()
|
||||
@@ -373,9 +393,14 @@ mod tests {
|
||||
assert!(validate_extension_schemas(&response.extensions).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_catalog_exposes_read_only_runtime_capability_snapshots() {
|
||||
let response = build_extension_catalog_response();
|
||||
#[tokio::test]
|
||||
async fn extension_catalog_exposes_read_only_runtime_capability_snapshots() {
|
||||
let runtime_capabilities = system::build_runtime_capabilities_response()
|
||||
.await
|
||||
.expect("runtime capabilities response should build");
|
||||
let response = build_extension_catalog_response()
|
||||
.await
|
||||
.expect("extension catalog response should build");
|
||||
|
||||
assert_eq!(response.runtime_capabilities.ops_diagnostics.extension_id, "builtin:ops-diagnostics");
|
||||
assert_eq!(
|
||||
@@ -388,6 +413,22 @@ mod tests {
|
||||
);
|
||||
assert!(!response.runtime_capabilities.ops_diagnostics.disabled_by_default);
|
||||
assert!(!response.runtime_capabilities.ops_diagnostics.startup_fatal);
|
||||
assert_eq!(
|
||||
response
|
||||
.runtime_capabilities
|
||||
.ops_diagnostics
|
||||
.runtime_capability_path
|
||||
.as_deref(),
|
||||
Some(format!("{}{}", ADMIN_PREFIX, RUNTIME_CAPABILITIES_ROUTE_SUFFIX).as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.runtime_capabilities
|
||||
.ops_diagnostics
|
||||
.runtime_capability_summary
|
||||
.as_ref(),
|
||||
Some(&runtime_capabilities.summary.observability)
|
||||
);
|
||||
assert!(response.runtime_capabilities.ops_diagnostics.contract.requires_admin_action);
|
||||
assert!(!response.runtime_capabilities.ops_diagnostics.contract.mutates_object_data);
|
||||
assert!(validate_ops_diagnostics_contract(&response.runtime_capabilities.ops_diagnostics.contract).is_ok());
|
||||
@@ -400,6 +441,14 @@ mod tests {
|
||||
);
|
||||
assert!(!response.runtime_capabilities.ops_profiler.disabled_by_default);
|
||||
assert!(!response.runtime_capabilities.ops_profiler.startup_fatal);
|
||||
assert_eq!(
|
||||
response.runtime_capabilities.ops_profiler.runtime_capability_path.as_deref(),
|
||||
Some(format!("{}{}", ADMIN_PREFIX, RUNTIME_CAPABILITIES_ROUTE_SUFFIX).as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
response.runtime_capabilities.ops_profiler.runtime_capability_summary.as_ref(),
|
||||
Some(&runtime_capabilities.summary.userspace_profiling)
|
||||
);
|
||||
assert!(validate_ops_profiler_contract(&response.runtime_capabilities.ops_profiler.contract).is_ok());
|
||||
}
|
||||
|
||||
|
||||
@@ -16,14 +16,22 @@ use super::metrics;
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::app::admin_usecase::{DefaultAdminUsecase, QueryServerInfoRequest};
|
||||
use crate::app::context::resolve_endpoints_handle;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::runtime_capabilities::{EndpointTopologySnapshotProvider, RustFsObservabilitySnapshotProvider};
|
||||
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||
use crate::workload_admission::workload_admission_registry_snapshot;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
use rustfs_concurrency::WorkloadAdmissionRegistrySnapshot;
|
||||
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
||||
use rustfs_storage_api::{
|
||||
CapabilityState, CapabilityStatus, ObservabilitySnapshotProvider, TopologySnapshot, TopologySnapshotProvider,
|
||||
};
|
||||
use s3s::header::CONTENT_TYPE;
|
||||
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use serde::Serialize;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
const LOG_COMPONENT_ADMIN_API: &str = "admin_api";
|
||||
@@ -31,6 +39,10 @@ const LOG_SUBSYSTEM_SYSTEM_ADMIN: &str = "system_admin";
|
||||
const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected";
|
||||
const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed";
|
||||
const EVENT_ADMIN_RESPONSE_EMITTED: &str = "admin_response_emitted";
|
||||
const OBSERVABILITY_SUMMARY_RESOLVED: &str = "observability summary resolved from provider snapshots";
|
||||
const TOPOLOGY_SUMMARY_RESOLVED: &str = "topology summary resolved from capability snapshot";
|
||||
const TOPOLOGY_SNAPSHOT_NOT_AVAILABLE: &str = "endpoint topology is not available before storage endpoint pools initialize";
|
||||
pub(crate) const RUNTIME_CAPABILITIES_ROUTE_SUFFIX: &str = "/v4/runtime/capabilities";
|
||||
|
||||
macro_rules! log_system_request_rejected {
|
||||
($operation:expr, $reason:expr) => {
|
||||
@@ -117,6 +129,12 @@ pub fn register_system_route(r: &mut S3Router<AdminOperation>) -> std::io::Resul
|
||||
AdminOperation(&metrics::MetricsHandler {}),
|
||||
)?;
|
||||
|
||||
r.insert(
|
||||
Method::GET,
|
||||
format!("{}{}", ADMIN_PREFIX, RUNTIME_CAPABILITIES_ROUTE_SUFFIX).as_str(),
|
||||
AdminOperation(&RuntimeCapabilitiesHandler {}),
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
pub struct ServiceHandle {}
|
||||
@@ -225,6 +243,176 @@ impl Operation for StorageInfoHandler {
|
||||
|
||||
pub struct DataUsageInfoHandler {}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct RuntimeCapabilitiesSummary {
|
||||
pub observability: CapabilityStatus,
|
||||
pub userspace_profiling: CapabilityStatus,
|
||||
pub memory_sampling: CapabilityStatus,
|
||||
pub platform: CapabilityStatus,
|
||||
pub topology: CapabilityStatus,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct RuntimeCapabilitiesResponse {
|
||||
pub summary: RuntimeCapabilitiesSummary,
|
||||
pub observability: rustfs_storage_api::ObservabilitySnapshot,
|
||||
pub workload_admission: WorkloadAdmissionRegistrySnapshot,
|
||||
pub topology: Option<TopologySnapshot>,
|
||||
pub topology_status: CapabilityStatus,
|
||||
}
|
||||
|
||||
pub struct RuntimeCapabilitiesHandler {}
|
||||
|
||||
pub(crate) async fn build_runtime_capabilities_response()
|
||||
-> Result<RuntimeCapabilitiesResponse, rustfs_storage_api::CapabilitySnapshotError> {
|
||||
let observability_provider = RustFsObservabilitySnapshotProvider;
|
||||
let observability = observability_provider.observability_snapshot().await?;
|
||||
let workload_admission = workload_admission_registry_snapshot();
|
||||
|
||||
let (topology, topology_status) = if let Some(endpoint_pools) = resolve_endpoints_handle() {
|
||||
let topology_provider = EndpointTopologySnapshotProvider::new(endpoint_pools);
|
||||
let topology = topology_provider.topology_snapshot().await?;
|
||||
(Some(topology), CapabilityStatus::supported())
|
||||
} else {
|
||||
(None, CapabilityStatus::unknown().with_reason(TOPOLOGY_SNAPSHOT_NOT_AVAILABLE))
|
||||
};
|
||||
let summary = build_runtime_capabilities_summary(&observability, topology.as_ref(), &topology_status);
|
||||
|
||||
Ok(RuntimeCapabilitiesResponse {
|
||||
summary,
|
||||
observability,
|
||||
workload_admission,
|
||||
topology,
|
||||
topology_status,
|
||||
})
|
||||
}
|
||||
|
||||
fn build_runtime_capabilities_summary(
|
||||
observability: &rustfs_storage_api::ObservabilitySnapshot,
|
||||
topology: Option<&TopologySnapshot>,
|
||||
topology_status: &CapabilityStatus,
|
||||
) -> RuntimeCapabilitiesSummary {
|
||||
let userspace_profiling = summarize_named_capability_statuses(
|
||||
[
|
||||
("cpu", &observability.userspace_profiling.cpu),
|
||||
("memory", &observability.userspace_profiling.memory),
|
||||
("continuous_cpu", &observability.userspace_profiling.continuous_cpu),
|
||||
("periodic_cpu", &observability.userspace_profiling.periodic_cpu),
|
||||
],
|
||||
"userspace profiling",
|
||||
);
|
||||
let memory_sampling = summarize_named_capability_statuses(
|
||||
[
|
||||
("process", &observability.memory_sampling.process),
|
||||
("system", &observability.memory_sampling.system),
|
||||
("cgroup", &observability.memory_sampling.cgroup),
|
||||
],
|
||||
"memory sampling",
|
||||
);
|
||||
let platform = summarize_named_capability_statuses(
|
||||
[
|
||||
("allocator", &observability.platform.allocator),
|
||||
("ebpf", &observability.platform.ebpf),
|
||||
("numa", &observability.platform.numa),
|
||||
],
|
||||
"platform support",
|
||||
);
|
||||
let observability_status = if [userspace_profiling.state, memory_sampling.state, platform.state]
|
||||
.into_iter()
|
||||
.any(|state| state == CapabilityState::Unknown)
|
||||
{
|
||||
CapabilityStatus::unknown().with_reason(OBSERVABILITY_SUMMARY_RESOLVED)
|
||||
} else {
|
||||
CapabilityStatus::supported().with_reason(OBSERVABILITY_SUMMARY_RESOLVED)
|
||||
};
|
||||
let topology_summary = topology.map_or_else(
|
||||
|| topology_status.clone(),
|
||||
|topology| {
|
||||
summarize_named_capability_statuses(
|
||||
[
|
||||
("profiling", &topology.capabilities.profiling),
|
||||
("numa", &topology.capabilities.numa),
|
||||
("failure_domain_labels", &topology.capabilities.failure_domain_labels),
|
||||
("media_labels", &topology.capabilities.media_labels),
|
||||
],
|
||||
"topology capability",
|
||||
)
|
||||
.with_reason(TOPOLOGY_SUMMARY_RESOLVED)
|
||||
},
|
||||
);
|
||||
|
||||
RuntimeCapabilitiesSummary {
|
||||
observability: observability_status,
|
||||
userspace_profiling,
|
||||
memory_sampling,
|
||||
platform,
|
||||
topology: topology_summary,
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_named_capability_statuses<const N: usize>(
|
||||
statuses: [(&'static str, &CapabilityStatus); N],
|
||||
subject: &'static str,
|
||||
) -> CapabilityStatus {
|
||||
let unknown = statuses
|
||||
.iter()
|
||||
.filter_map(|(name, status)| (status.state == CapabilityState::Unknown).then_some(*name))
|
||||
.collect::<Vec<_>>();
|
||||
if !unknown.is_empty() {
|
||||
return CapabilityStatus::unknown().with_reason(format!("{subject} unresolved fields: {}", unknown.join(", ")));
|
||||
}
|
||||
|
||||
let supported_or_disabled = statuses
|
||||
.iter()
|
||||
.any(|(_, status)| matches!(status.state, CapabilityState::Supported | CapabilityState::Disabled));
|
||||
if supported_or_disabled {
|
||||
return CapabilityStatus::supported();
|
||||
}
|
||||
|
||||
let unsupported = statuses.iter().map(|(name, _)| *name).collect::<Vec<_>>();
|
||||
CapabilityStatus::unsupported().with_reason(format!("{subject} unsupported fields: {}", unsupported.join(", ")))
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for RuntimeCapabilitiesHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let Some(input_cred) = req.credentials else {
|
||||
log_system_request_rejected!("runtime_capabilities", "missing_credentials");
|
||||
return Err(s3_error!(InvalidRequest, "get cred failed"));
|
||||
};
|
||||
|
||||
let (cred, owner) =
|
||||
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
|
||||
|
||||
let remote_addr = req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0));
|
||||
validate_admin_request(
|
||||
&req.headers,
|
||||
&cred,
|
||||
owner,
|
||||
false,
|
||||
vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)],
|
||||
remote_addr,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let response = build_runtime_capabilities_response().await.map_err(|err| {
|
||||
log_system_request_failed!("runtime_capabilities", "build_runtime_capabilities_failed", err);
|
||||
S3Error::with_message(S3ErrorCode::InternalError, "failed to build runtime capabilities snapshot")
|
||||
})?;
|
||||
|
||||
let data = serde_json::to_vec(&response).map_err(|e| {
|
||||
log_system_request_failed!("runtime_capabilities", "serialize_runtime_capabilities_failed", e);
|
||||
S3Error::with_message(S3ErrorCode::InternalError, "failed to serialize runtime capabilities snapshot")
|
||||
})?;
|
||||
|
||||
let mut header = HeaderMap::new();
|
||||
header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
log_system_response_emitted!("runtime_capabilities");
|
||||
|
||||
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for DataUsageInfoHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
@@ -265,3 +453,149 @@ impl Operation for DataUsageInfoHandler {
|
||||
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
OBSERVABILITY_SUMMARY_RESOLVED, TOPOLOGY_SNAPSHOT_NOT_AVAILABLE, TOPOLOGY_SUMMARY_RESOLVED,
|
||||
build_runtime_capabilities_response, build_runtime_capabilities_summary,
|
||||
};
|
||||
use rustfs_concurrency::{AdmissionState, WorkloadClass};
|
||||
use rustfs_storage_api::{
|
||||
CapabilityState, CapabilityStatus, MemorySamplingState, ObservabilitySnapshot, PlatformSupport, TopologyCapabilities,
|
||||
TopologySnapshot, UserspaceProfilingCapability,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_capabilities_response_reports_missing_topology_before_storage_init() {
|
||||
let response = build_runtime_capabilities_response()
|
||||
.await
|
||||
.expect("runtime capabilities response should build");
|
||||
|
||||
assert_eq!(response.topology, None);
|
||||
assert_eq!(response.topology_status.state, CapabilityState::Unknown);
|
||||
assert_eq!(response.topology_status.reason.as_deref(), Some(TOPOLOGY_SNAPSHOT_NOT_AVAILABLE));
|
||||
assert_eq!(response.summary.topology.state, CapabilityState::Unknown);
|
||||
assert_eq!(response.summary.topology.reason.as_deref(), Some(TOPOLOGY_SNAPSHOT_NOT_AVAILABLE));
|
||||
assert_eq!(response.observability.platform.os.as_deref(), Some(std::env::consts::OS));
|
||||
assert_eq!(
|
||||
response
|
||||
.workload_admission
|
||||
.get(WorkloadClass::ForegroundRead)
|
||||
.map(|snapshot| snapshot.state),
|
||||
Some(AdmissionState::Open)
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.workload_admission
|
||||
.get(WorkloadClass::ForegroundWrite)
|
||||
.map(|snapshot| snapshot.state),
|
||||
Some(AdmissionState::Disabled)
|
||||
);
|
||||
assert_eq!(response.workload_admission.entries().len(), WorkloadClass::REQUIRED.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_capabilities_summary_marks_observability_supported_when_groups_are_resolved() {
|
||||
let observability = ObservabilitySnapshot {
|
||||
runtime_telemetry: CapabilityStatus::supported(),
|
||||
userspace_profiling: UserspaceProfilingCapability {
|
||||
cpu: CapabilityStatus::supported(),
|
||||
memory: CapabilityStatus::unsupported().with_reason("platform"),
|
||||
continuous_cpu: CapabilityStatus::supported(),
|
||||
periodic_cpu: CapabilityStatus::disabled(),
|
||||
},
|
||||
memory_sampling: MemorySamplingState {
|
||||
process: CapabilityStatus::supported(),
|
||||
system: CapabilityStatus::supported(),
|
||||
cgroup: CapabilityStatus::unsupported().with_reason("platform"),
|
||||
},
|
||||
platform: PlatformSupport {
|
||||
target_triple: Some("x86_64-unknown-linux-gnu".to_owned()),
|
||||
os: Some("linux".to_owned()),
|
||||
arch: Some("x86_64".to_owned()),
|
||||
allocator: CapabilityStatus::supported(),
|
||||
ebpf: CapabilityStatus::unsupported().with_reason("platform"),
|
||||
numa: CapabilityStatus::unsupported().with_reason("platform"),
|
||||
},
|
||||
};
|
||||
let topology = TopologySnapshot {
|
||||
pools: Vec::new(),
|
||||
capabilities: TopologyCapabilities {
|
||||
profiling: CapabilityStatus::supported(),
|
||||
numa: CapabilityStatus::unsupported().with_reason("platform"),
|
||||
failure_domain_labels: CapabilityStatus::supported(),
|
||||
media_labels: CapabilityStatus::unsupported().with_reason("not reported"),
|
||||
},
|
||||
};
|
||||
|
||||
let summary = build_runtime_capabilities_summary(&observability, Some(&topology), &CapabilityStatus::supported());
|
||||
|
||||
assert_eq!(summary.observability.state, CapabilityState::Supported);
|
||||
assert_eq!(summary.observability.reason.as_deref(), Some(OBSERVABILITY_SUMMARY_RESOLVED));
|
||||
assert_eq!(summary.userspace_profiling.state, CapabilityState::Supported);
|
||||
assert_eq!(summary.memory_sampling.state, CapabilityState::Supported);
|
||||
assert_eq!(summary.platform.state, CapabilityState::Supported);
|
||||
assert_eq!(summary.topology.state, CapabilityState::Supported);
|
||||
assert_eq!(summary.topology.reason.as_deref(), Some(TOPOLOGY_SUMMARY_RESOLVED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_capabilities_summary_marks_unknown_groups_when_snapshot_is_not_fully_wired() {
|
||||
let observability = ObservabilitySnapshot {
|
||||
runtime_telemetry: CapabilityStatus::supported(),
|
||||
userspace_profiling: UserspaceProfilingCapability {
|
||||
cpu: CapabilityStatus::unknown().with_reason("not wired"),
|
||||
memory: CapabilityStatus::unsupported().with_reason("platform"),
|
||||
continuous_cpu: CapabilityStatus::supported(),
|
||||
periodic_cpu: CapabilityStatus::disabled(),
|
||||
},
|
||||
memory_sampling: MemorySamplingState {
|
||||
process: CapabilityStatus::supported(),
|
||||
system: CapabilityStatus::supported(),
|
||||
cgroup: CapabilityStatus::unknown().with_reason("not wired"),
|
||||
},
|
||||
platform: PlatformSupport {
|
||||
target_triple: Some("x86_64-unknown-linux-gnu".to_owned()),
|
||||
os: Some("linux".to_owned()),
|
||||
arch: Some("x86_64".to_owned()),
|
||||
allocator: CapabilityStatus::supported(),
|
||||
ebpf: CapabilityStatus::unknown().with_reason("not wired"),
|
||||
numa: CapabilityStatus::unknown().with_reason("not wired"),
|
||||
},
|
||||
};
|
||||
let topology = TopologySnapshot {
|
||||
pools: Vec::new(),
|
||||
capabilities: TopologyCapabilities {
|
||||
profiling: CapabilityStatus::supported(),
|
||||
numa: CapabilityStatus::unknown().with_reason("not wired"),
|
||||
failure_domain_labels: CapabilityStatus::unknown().with_reason("not reported"),
|
||||
media_labels: CapabilityStatus::unknown().with_reason("not reported"),
|
||||
},
|
||||
};
|
||||
|
||||
let summary = build_runtime_capabilities_summary(&observability, Some(&topology), &CapabilityStatus::supported());
|
||||
|
||||
assert_eq!(summary.observability.state, CapabilityState::Unknown);
|
||||
assert_eq!(summary.userspace_profiling.state, CapabilityState::Unknown);
|
||||
assert_eq!(summary.memory_sampling.state, CapabilityState::Unknown);
|
||||
assert_eq!(summary.platform.state, CapabilityState::Unknown);
|
||||
assert_eq!(summary.topology.state, CapabilityState::Unknown);
|
||||
assert!(
|
||||
summary
|
||||
.userspace_profiling
|
||||
.reason
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.contains("cpu")
|
||||
);
|
||||
assert!(
|
||||
summary
|
||||
.topology
|
||||
.reason
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.contains(TOPOLOGY_SUMMARY_RESOLVED)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -350,6 +350,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
GET_BUCKET_TARGET,
|
||||
RouteRiskLevel::Sensitive,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/v4/runtime/capabilities",
|
||||
SERVER_INFO,
|
||||
RouteRiskLevel::Sensitive,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/v4/plugins/catalog",
|
||||
|
||||
@@ -226,6 +226,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
admin_route(Method::PUT, "/v3/module-switches"),
|
||||
admin_route(Method::GET, "/v4/extensions/catalog"),
|
||||
admin_route(Method::GET, "/v4/extensions/instances"),
|
||||
admin_route(Method::GET, "/v4/runtime/capabilities"),
|
||||
admin_route(Method::POST, "/v3/object-zip-downloads"),
|
||||
admin_route_sample(
|
||||
Method::GET,
|
||||
@@ -724,6 +725,7 @@ fn test_register_routes_cover_representative_admin_paths() {
|
||||
assert_route(&router, Method::PUT, &admin_path("/v3/module-switches"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v4/extensions/catalog"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v4/extensions/instances"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v4/runtime/capabilities"));
|
||||
assert_route(&router, Method::POST, &admin_path("/v3/object-zip-downloads"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v4/plugins/catalog"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v4/plugins/instances"));
|
||||
|
||||
@@ -20,9 +20,11 @@ use rustfs_storage_api::{
|
||||
};
|
||||
|
||||
const NOT_WIRED_INTO_RUNTIME: &str = "not wired into runtime";
|
||||
const EBPF_LINUX_ONLY: &str = "eBPF support is only available on linux targets";
|
||||
const STORAGE_MEDIA_NOT_REPORTED: &str = "storage media not reported by endpoints";
|
||||
const FAILURE_DOMAIN_NOT_REPORTED: &str = "failure domain labels not reported by endpoints";
|
||||
const NUMA_NOT_WIRED: &str = "NUMA topology not wired into runtime";
|
||||
const NUMA_LINUX_ONLY: &str = "NUMA topology reporting currently targets linux runtimes";
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub struct RustFsObservabilitySnapshotProvider;
|
||||
@@ -49,13 +51,13 @@ pub fn runtime_observability_snapshot() -> ObservabilitySnapshot {
|
||||
cgroup: cgroup_memory_status(),
|
||||
},
|
||||
platform: PlatformSupport {
|
||||
target_triple: option_env!("TARGET").map(str::to_owned),
|
||||
target_triple: Some(compiled_target_triple()),
|
||||
os: Some(std::env::consts::OS.to_owned()),
|
||||
arch: Some(std::env::consts::ARCH.to_owned()),
|
||||
allocator: CapabilityStatus::supported()
|
||||
.with_reason(format!("backend={}", crate::allocator_reclaim::allocator_backend())),
|
||||
ebpf: CapabilityStatus::unknown().with_reason(NOT_WIRED_INTO_RUNTIME),
|
||||
numa: CapabilityStatus::unsupported().with_reason(NUMA_NOT_WIRED),
|
||||
ebpf: ebpf_status(),
|
||||
numa: numa_status(),
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -104,6 +106,42 @@ fn runtime_telemetry_status() -> CapabilityStatus {
|
||||
}
|
||||
}
|
||||
|
||||
fn compiled_target_triple() -> String {
|
||||
let arch = std::env::consts::ARCH;
|
||||
let vendor = target_vendor_name();
|
||||
let os = std::env::consts::OS;
|
||||
let env = target_env_name();
|
||||
|
||||
match env {
|
||||
Some(env) => format!("{arch}-{vendor}-{os}-{env}"),
|
||||
None => format!("{arch}-{vendor}-{os}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn target_vendor_name() -> &'static str {
|
||||
if cfg!(target_vendor = "apple") {
|
||||
"apple"
|
||||
} else if cfg!(target_vendor = "pc") {
|
||||
"pc"
|
||||
} else {
|
||||
"unknown"
|
||||
}
|
||||
}
|
||||
|
||||
fn target_env_name() -> Option<&'static str> {
|
||||
if cfg!(target_env = "gnu") {
|
||||
Some("gnu")
|
||||
} else if cfg!(target_env = "musl") {
|
||||
Some("musl")
|
||||
} else if cfg!(target_env = "msvc") {
|
||||
Some("msvc")
|
||||
} else if cfg!(target_env = "sgx") {
|
||||
Some("sgx")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn cpu_profiling_status() -> CapabilityStatus {
|
||||
if cfg!(any(target_os = "linux", target_os = "macos")) {
|
||||
CapabilityStatus::supported()
|
||||
@@ -112,6 +150,22 @@ fn cpu_profiling_status() -> CapabilityStatus {
|
||||
}
|
||||
}
|
||||
|
||||
fn ebpf_status() -> CapabilityStatus {
|
||||
if cfg!(target_os = "linux") {
|
||||
CapabilityStatus::unknown().with_reason(NOT_WIRED_INTO_RUNTIME)
|
||||
} else {
|
||||
CapabilityStatus::unsupported().with_reason(EBPF_LINUX_ONLY)
|
||||
}
|
||||
}
|
||||
|
||||
fn numa_status() -> CapabilityStatus {
|
||||
if cfg!(target_os = "linux") {
|
||||
CapabilityStatus::unknown().with_reason(NUMA_NOT_WIRED)
|
||||
} else {
|
||||
CapabilityStatus::unsupported().with_reason(NUMA_LINUX_ONLY)
|
||||
}
|
||||
}
|
||||
|
||||
fn memory_profiling_status() -> CapabilityStatus {
|
||||
if cfg!(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")) {
|
||||
CapabilityStatus::supported().with_reason("jemalloc memory profiling target")
|
||||
@@ -141,6 +195,7 @@ mod tests {
|
||||
|
||||
assert_eq!(snapshot.platform.os.as_deref(), Some(std::env::consts::OS));
|
||||
assert_eq!(snapshot.platform.arch.as_deref(), Some(std::env::consts::ARCH));
|
||||
assert_eq!(snapshot.platform.target_triple.as_deref(), Some(compiled_target_triple().as_str()));
|
||||
assert!(matches!(
|
||||
snapshot.runtime_telemetry.state,
|
||||
CapabilityState::Supported | CapabilityState::Disabled
|
||||
@@ -154,6 +209,13 @@ mod tests {
|
||||
.unwrap_or_default()
|
||||
.contains("backend=")
|
||||
);
|
||||
if cfg!(target_os = "linux") {
|
||||
assert_eq!(snapshot.platform.ebpf.state, CapabilityState::Unknown);
|
||||
assert_eq!(snapshot.platform.numa.state, CapabilityState::Unknown);
|
||||
} else {
|
||||
assert_eq!(snapshot.platform.ebpf.state, CapabilityState::Unsupported);
|
||||
assert_eq!(snapshot.platform.numa.state, CapabilityState::Unsupported);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -22,10 +22,14 @@ use rustfs_concurrency::{
|
||||
use crate::storage::concurrency::get_concurrency_manager;
|
||||
|
||||
const BUCKET_METADATA_RUNTIME_NOT_INITIALIZED: &str = "bucket metadata runtime not initialized";
|
||||
const FOREGROUND_WRITE_NOT_EXPOSED_BY_PROVIDER: &str = "foreground write admission not yet exposed by RustFS runtime";
|
||||
const HEAL_MANAGER_NOT_INITIALIZED: &str = "heal manager not initialized";
|
||||
const NOT_EXPOSED_BY_PROVIDER: &str = "not exposed by RustFS workload admission provider";
|
||||
const REPAIR_QUEUE_BACKLOG_PRESENT: &str = "repair queue has pending work";
|
||||
const REPLICATION_RUNTIME_NOT_INITIALIZED: &str = "replication runtime not initialized";
|
||||
const REPLICATION_QUEUE_BACKLOG_PRESENT: &str = "replication queue has pending work";
|
||||
const REPLICATION_QUEUE_STATS_UNAVAILABLE: &str = "replication queue stats unavailable";
|
||||
const SCANNER_ADMISSION_DISABLED: &str = "scanner admission disabled because max concurrent set scans is zero";
|
||||
const SCANNER_ADMISSION_SATURATED: &str = "scanner active work reached configured set-scan limit";
|
||||
const SCANNER_ACTIVITY_IDLE_OR_NOT_INITIALIZED: &str = "scanner activity idle or not initialized";
|
||||
const STORAGE_CONCURRENCY_PROVIDER_MISSING_FOREGROUND_READ: &str =
|
||||
"storage concurrency provider did not expose foreground read admission";
|
||||
@@ -49,18 +53,13 @@ fn storage_concurrency_workload_admission_snapshot() -> WorkloadAdmissionRegistr
|
||||
}
|
||||
|
||||
fn runtime_owner_workload_admission_registry_snapshot() -> WorkloadAdmissionRegistrySnapshot {
|
||||
let entries = WorkloadClass::REQUIRED
|
||||
.iter()
|
||||
.copied()
|
||||
.filter_map(|class| match class {
|
||||
WorkloadClass::ForegroundRead => None,
|
||||
WorkloadClass::Metadata => Some(metadata_workload_admission_snapshot()),
|
||||
WorkloadClass::Scanner => Some(scanner_workload_admission_snapshot()),
|
||||
WorkloadClass::Repair => Some(repair_workload_admission_snapshot()),
|
||||
WorkloadClass::Replication => Some(replication_workload_admission_snapshot()),
|
||||
class => Some(WorkloadAdmissionSnapshot::new(class, AdmissionState::Unknown).with_reason(NOT_EXPOSED_BY_PROVIDER)),
|
||||
})
|
||||
.collect();
|
||||
let entries = vec![
|
||||
foreground_write_workload_admission_snapshot(),
|
||||
metadata_workload_admission_snapshot(),
|
||||
scanner_workload_admission_snapshot(),
|
||||
repair_workload_admission_snapshot(),
|
||||
replication_workload_admission_snapshot(),
|
||||
];
|
||||
|
||||
WorkloadAdmissionRegistrySnapshot::new(entries)
|
||||
}
|
||||
@@ -75,6 +74,12 @@ pub fn foreground_read_workload_admission_snapshot() -> WorkloadAdmissionSnapsho
|
||||
})
|
||||
}
|
||||
|
||||
pub fn foreground_write_workload_admission_snapshot() -> WorkloadAdmissionSnapshot {
|
||||
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Disabled)
|
||||
.with_counts(Some(0), None, Some(0))
|
||||
.with_reason(FOREGROUND_WRITE_NOT_EXPOSED_BY_PROVIDER)
|
||||
}
|
||||
|
||||
pub fn metadata_workload_admission_snapshot() -> WorkloadAdmissionSnapshot {
|
||||
metadata_workload_admission_snapshot_from_initialized(bucket_metadata_runtime_initialized())
|
||||
}
|
||||
@@ -96,11 +101,19 @@ fn metadata_workload_admission_snapshot_from_initialized(runtime_initialized: bo
|
||||
}
|
||||
|
||||
pub fn scanner_workload_admission_snapshot() -> WorkloadAdmissionSnapshot {
|
||||
scanner_workload_admission_snapshot_from_activity(rustfs_scanner::current_scanner_activity())
|
||||
let runtime_config = rustfs_scanner::scanner_runtime_config_status();
|
||||
scanner_workload_admission_snapshot_from_activity(
|
||||
rustfs_scanner::current_scanner_activity(),
|
||||
runtime_config.max_concurrent_set_scans.value,
|
||||
)
|
||||
}
|
||||
|
||||
fn scanner_workload_admission_snapshot_from_activity(active: u64) -> WorkloadAdmissionSnapshot {
|
||||
let state = if active > 0 {
|
||||
fn scanner_workload_admission_snapshot_from_activity(active: u64, limit: usize) -> WorkloadAdmissionSnapshot {
|
||||
let state = if limit == 0 {
|
||||
AdmissionState::Disabled
|
||||
} else if usize::try_from(active).ok().is_some_and(|active| active >= limit) {
|
||||
AdmissionState::Saturated
|
||||
} else if active > 0 {
|
||||
AdmissionState::Open
|
||||
} else {
|
||||
AdmissionState::Unknown
|
||||
@@ -109,13 +122,14 @@ fn scanner_workload_admission_snapshot_from_activity(active: u64) -> WorkloadAdm
|
||||
let snapshot = WorkloadAdmissionSnapshot::new(WorkloadClass::Scanner, state).with_counts(
|
||||
Some(u64_to_usize_saturated(active)),
|
||||
None,
|
||||
None,
|
||||
Some(limit),
|
||||
);
|
||||
|
||||
if state == AdmissionState::Unknown {
|
||||
snapshot.with_reason(SCANNER_ACTIVITY_IDLE_OR_NOT_INITIALIZED)
|
||||
} else {
|
||||
snapshot
|
||||
match state {
|
||||
AdmissionState::Disabled => snapshot.with_reason(SCANNER_ADMISSION_DISABLED),
|
||||
AdmissionState::Saturated => snapshot.with_reason(SCANNER_ADMISSION_SATURATED),
|
||||
AdmissionState::Unknown => snapshot.with_reason(SCANNER_ACTIVITY_IDLE_OR_NOT_INITIALIZED),
|
||||
_ => snapshot,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,7 +146,11 @@ fn repair_workload_admission_snapshot_from_counts(
|
||||
active: u64,
|
||||
queued: u64,
|
||||
) -> WorkloadAdmissionSnapshot {
|
||||
let state = if manager_initialized || active > 0 || queued > 0 {
|
||||
let state = if !manager_initialized && active == 0 && queued == 0 {
|
||||
AdmissionState::Unknown
|
||||
} else if queued > 0 {
|
||||
AdmissionState::Throttled
|
||||
} else if manager_initialized || active > 0 {
|
||||
AdmissionState::Open
|
||||
} else {
|
||||
AdmissionState::Unknown
|
||||
@@ -144,10 +162,10 @@ fn repair_workload_admission_snapshot_from_counts(
|
||||
None,
|
||||
);
|
||||
|
||||
if state == AdmissionState::Unknown {
|
||||
snapshot.with_reason(HEAL_MANAGER_NOT_INITIALIZED)
|
||||
} else {
|
||||
snapshot
|
||||
match state {
|
||||
AdmissionState::Unknown => snapshot.with_reason(HEAL_MANAGER_NOT_INITIALIZED),
|
||||
AdmissionState::Throttled => snapshot.with_reason(REPAIR_QUEUE_BACKLOG_PRESENT),
|
||||
_ => snapshot,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,7 +188,11 @@ fn replication_workload_admission_snapshot_from_counts(
|
||||
active: Option<usize>,
|
||||
queued: Option<usize>,
|
||||
) -> WorkloadAdmissionSnapshot {
|
||||
let state = if runtime_initialized && queued.is_some() {
|
||||
let state = if !runtime_initialized {
|
||||
AdmissionState::Unknown
|
||||
} else if queued.is_some_and(|queued| queued > 0) {
|
||||
AdmissionState::Throttled
|
||||
} else if queued.is_some() {
|
||||
AdmissionState::Open
|
||||
} else {
|
||||
AdmissionState::Unknown
|
||||
@@ -178,12 +200,11 @@ fn replication_workload_admission_snapshot_from_counts(
|
||||
|
||||
let snapshot = WorkloadAdmissionSnapshot::new(WorkloadClass::Replication, state).with_counts(active, queued, None);
|
||||
|
||||
if !runtime_initialized {
|
||||
snapshot.with_reason(REPLICATION_RUNTIME_NOT_INITIALIZED)
|
||||
} else if queued.is_none() {
|
||||
snapshot.with_reason(REPLICATION_QUEUE_STATS_UNAVAILABLE)
|
||||
} else {
|
||||
snapshot
|
||||
match state {
|
||||
AdmissionState::Unknown if !runtime_initialized => snapshot.with_reason(REPLICATION_RUNTIME_NOT_INITIALIZED),
|
||||
AdmissionState::Unknown => snapshot.with_reason(REPLICATION_QUEUE_STATS_UNAVAILABLE),
|
||||
AdmissionState::Throttled => snapshot.with_reason(REPLICATION_QUEUE_BACKLOG_PRESENT),
|
||||
_ => snapshot,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -238,38 +259,70 @@ mod tests {
|
||||
assert_eq!(snapshot.reason.as_deref(), Some(BUCKET_METADATA_RUNTIME_NOT_INITIALIZED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_write_snapshot_reports_explicit_gap_reason() {
|
||||
let snapshot = foreground_write_workload_admission_snapshot();
|
||||
|
||||
assert_eq!(snapshot.class, WorkloadClass::ForegroundWrite);
|
||||
assert_eq!(snapshot.state, AdmissionState::Disabled);
|
||||
assert_eq!(snapshot.active, Some(0));
|
||||
assert_eq!(snapshot.limit, Some(0));
|
||||
assert_eq!(snapshot.reason.as_deref(), Some(FOREGROUND_WRITE_NOT_EXPOSED_BY_PROVIDER));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_snapshot_reports_active_work_units() {
|
||||
let snapshot = scanner_workload_admission_snapshot_from_activity(5);
|
||||
let snapshot = scanner_workload_admission_snapshot_from_activity(5, 8);
|
||||
|
||||
assert_eq!(snapshot.class, WorkloadClass::Scanner);
|
||||
assert_eq!(snapshot.state, AdmissionState::Open);
|
||||
assert_eq!(snapshot.active, Some(5));
|
||||
assert_eq!(snapshot.queued, None);
|
||||
assert_eq!(snapshot.limit, None);
|
||||
assert_eq!(snapshot.limit, Some(8));
|
||||
assert_eq!(snapshot.reason, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_snapshot_is_unknown_when_idle_or_uninitialized() {
|
||||
let snapshot = scanner_workload_admission_snapshot_from_activity(0);
|
||||
let snapshot = scanner_workload_admission_snapshot_from_activity(0, 8);
|
||||
|
||||
assert_eq!(snapshot.class, WorkloadClass::Scanner);
|
||||
assert_eq!(snapshot.state, AdmissionState::Unknown);
|
||||
assert_eq!(snapshot.active, Some(0));
|
||||
assert_eq!(snapshot.limit, Some(8));
|
||||
assert_eq!(snapshot.reason.as_deref(), Some(SCANNER_ACTIVITY_IDLE_OR_NOT_INITIALIZED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_snapshot_reports_disabled_when_set_scan_limit_is_zero() {
|
||||
let snapshot = scanner_workload_admission_snapshot_from_activity(0, 0);
|
||||
|
||||
assert_eq!(snapshot.class, WorkloadClass::Scanner);
|
||||
assert_eq!(snapshot.state, AdmissionState::Disabled);
|
||||
assert_eq!(snapshot.limit, Some(0));
|
||||
assert_eq!(snapshot.reason.as_deref(), Some(SCANNER_ADMISSION_DISABLED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_snapshot_reports_saturation_when_active_work_reaches_limit() {
|
||||
let snapshot = scanner_workload_admission_snapshot_from_activity(4, 4);
|
||||
|
||||
assert_eq!(snapshot.class, WorkloadClass::Scanner);
|
||||
assert_eq!(snapshot.state, AdmissionState::Saturated);
|
||||
assert_eq!(snapshot.limit, Some(4));
|
||||
assert_eq!(snapshot.reason.as_deref(), Some(SCANNER_ADMISSION_SATURATED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repair_snapshot_reports_heal_counters() {
|
||||
let snapshot = repair_workload_admission_snapshot_from_counts(true, 2, 3);
|
||||
|
||||
assert_eq!(snapshot.class, WorkloadClass::Repair);
|
||||
assert_eq!(snapshot.state, AdmissionState::Open);
|
||||
assert_eq!(snapshot.state, AdmissionState::Throttled);
|
||||
assert_eq!(snapshot.active, Some(2));
|
||||
assert_eq!(snapshot.queued, Some(3));
|
||||
assert_eq!(snapshot.limit, None);
|
||||
assert_eq!(snapshot.reason, None);
|
||||
assert_eq!(snapshot.reason.as_deref(), Some(REPAIR_QUEUE_BACKLOG_PRESENT));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -288,11 +341,11 @@ mod tests {
|
||||
let snapshot = replication_workload_admission_snapshot_from_counts(true, Some(4), Some(9));
|
||||
|
||||
assert_eq!(snapshot.class, WorkloadClass::Replication);
|
||||
assert_eq!(snapshot.state, AdmissionState::Open);
|
||||
assert_eq!(snapshot.state, AdmissionState::Throttled);
|
||||
assert_eq!(snapshot.active, Some(4));
|
||||
assert_eq!(snapshot.queued, Some(9));
|
||||
assert_eq!(snapshot.limit, None);
|
||||
assert_eq!(snapshot.reason, None);
|
||||
assert_eq!(snapshot.reason.as_deref(), Some(REPLICATION_QUEUE_BACKLOG_PRESENT));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -350,8 +403,8 @@ mod tests {
|
||||
assert_eq!(
|
||||
registry
|
||||
.get(WorkloadClass::ForegroundWrite)
|
||||
.and_then(|snapshot| snapshot.reason.as_deref()),
|
||||
Some(NOT_EXPOSED_BY_PROVIDER)
|
||||
.map(|snapshot| (snapshot.state, snapshot.reason.as_deref())),
|
||||
Some((AdmissionState::Disabled, Some(FOREGROUND_WRITE_NOT_EXPOSED_BY_PROVIDER)))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user