gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-11 07:36:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(obs): add bounded metrics dimensions (#5645)

Browse Source 
* feat(obs): add drive topology detail metrics

Expose additive drive info, topology, state, and per-drive API metrics while preserving the existing drive metric label sets.

Backlog: rustfs/backlog#1655

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): preserve suspect drive runtime state

Keep suspect as a bounded drive runtime state and avoid all-zero runtime_state samples for that storage health state.

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): skip unknown drive inode samples

Avoid exporting zero inode gauges for missing or stale drive snapshots and ignore zero-count API latency buckets.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add scanner source work detail metrics

Expose additive scanner source and cycle work metrics with bounded server/source/state labels while leaving the existing aggregate scanner metrics unchanged.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add ilm action detail metrics

Expose additive ILM action/state task metrics with a server label while preserving the existing aggregate ILM series.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add delivery target server metrics

Expose additive audit and notification delivery target metrics with server labels and extend removed-target tombstones for the server-aware series.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add replication target flow metrics

Expose additive bucket replication target sent and failed-flow metrics while preserving existing bucket aggregates and target backlog series.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add request server metrics

Expose additive API request metrics with server labels while preserving the existing request and traffic metric label sets.

Co-Authored-By: heihutu <heihutu@gmail.com>

* style(obs): apply rustfmt to metrics changes

Apply rustfmt output to the metrics dimension changes without altering behavior.

Co-Authored-By: heihutu <heihutu@gmail.com>

* style(obs): reuse audit target label constant

Use the exported audit target_id label constant for legacy audit target metrics.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): populate drive disk metrics

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add scanner bucket drive result metrics

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add replication proxy server metrics

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): address metric liveness review

Use checked division for drive API latency aggregation and keep recovered drive, scanner current-cycle, replication flow, audit target, and notification target series from retaining stale values.

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): address metric dimension review

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): address additional metric review

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): count drive calls at start

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): address metrics dimension review

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): address dimension review gaps

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): address scanner review follow-ups

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): address runtime review follow-ups

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): reduce disk metric contention

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): address runtime review follow-ups

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(metrics): retire stale dimension series

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-03 09:03:34 +08:00
committed by GitHub
parent 988cd8adbb
commit 035ce5d784
36 changed files with 6282 additions and 666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/metrics/collectors/audit.rs
+77 -5
View File
@@ -41,6 +41,13 @@ pub struct AuditTargetStats {
pub total_messages: u64,
}
/// Audit target statistics with runtime-local node identity.
#[derive(Debug, Clone, Default)]
pub(crate) struct AuditTargetRuntimeStats {
pub(crate) server: String,
pub(crate) target: AuditTargetStats,
}
/// Collects audit metrics from the provided audit target statistics.
///
/// Uses the metric descriptors from `metrics_type::audit` module.
@@ -56,19 +63,53 @@ pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec<PrometheusMetric
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_MD, stat.failed_messages as f64)
.with_label("target_id", target_id_label.clone()),
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_FAILED_STORE_LENGTH_MD, stat.failed_store_length as f64)
.with_label("target_id", target_id_label.clone()),
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_MD, stat.queue_length as f64)
.with_label("target_id", target_id_label.clone()),
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_MD, stat.total_messages as f64)
.with_label("target_id", target_id_label),
.with_label(TARGET_ID, target_id_label),
);
}
metrics
}
pub(crate) fn collect_audit_runtime_metrics(stats: &[AuditTargetRuntimeStats]) -> Vec<PrometheusMetric> {
let legacy_stats = stats.iter().map(|stat| stat.target.clone()).collect::<Vec<_>>();
let mut metrics = collect_audit_metrics(&legacy_stats);
metrics.reserve(stats.len() * 4);
for stat in stats.iter().filter(|stat| !stat.server.is_empty()) {
let server_label: Cow<'static, str> = Cow::Owned(stat.server.clone());
let target_id_label: Cow<'static, str> = Cow::Owned(stat.target.target_id.clone());
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_BY_SERVER_MD, stat.target.failed_messages as f64)
.with_label(SERVER, server_label.clone())
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_FAILED_STORE_LENGTH_BY_SERVER_MD, stat.target.failed_store_length as f64)
.with_label(SERVER, server_label.clone())
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD, stat.target.queue_length as f64)
.with_label(SERVER, server_label.clone())
.with_label(TARGET_ID, target_id_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_BY_SERVER_MD, stat.target.total_messages as f64)
.with_label(SERVER, server_label)
.with_label(TARGET_ID, target_id_label),
);
}
@@ -101,7 +142,7 @@ mod tests {
let metrics = collect_audit_metrics(&stats);
assert_eq!(metrics.len(), 8); // 2 targets * 4 metrics each
assert_eq!(metrics.len(), 8);
let failed = metrics
.iter()
@@ -114,6 +155,37 @@ mod tests {
&& m.labels.iter().any(|(k, v)| *k == "target_id" && v == "target-1")
});
assert!(failed_store.is_some());
assert!(
metrics
.iter()
.all(|m| m.name != AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name())
);
}
#[test]
fn collect_audit_runtime_metrics_adds_server_dimensions() {
let stats = vec![AuditTargetRuntimeStats {
server: "node1:9000".to_string(),
target: AuditTargetStats {
target_id: "target-1".to_string(),
failed_messages: 5,
failed_store_length: 3,
queue_length: 10,
total_messages: 1000,
},
}];
let metrics = collect_audit_runtime_metrics(&stats);
assert_eq!(metrics.len(), 8);
let server_queue = metrics.iter().find(|m| {
m.value == 10.0
&& m.name == AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name()
&& m.labels.iter().any(|(k, v)| *k == SERVER && v == "node1:9000")
&& m.labels.iter().any(|(k, v)| *k == TARGET_ID && v == "target-1")
});
assert!(server_queue.is_some());
}
#[test]
crates/obs/src/metrics/collectors/bucket_replication.rs
+253 -38
View File
@@ -30,14 +30,18 @@ use crate::metrics::schema::bucket_replication::{
BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD,
BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD,
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD,
BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD,
BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD,
BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L,
TARGET_ARN_L,
BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD, BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD,
BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD,
BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD,
BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD, BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD,
BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_TARGET_SENT_BYTES_MD, BUCKET_REPL_TARGET_SENT_COUNT_MD,
BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD,
BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, RESULT_L, TARGET_ARN_L,
};
use std::borrow::Cow;
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 37;
const BUCKET_REPLICATION_RUNTIME_FLOW_METRICS_PER_TARGET: usize = 8;
const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 11;
const BUCKET_REPLICATION_BACKLOG_METRICS_PER_TARGET: usize = 4;
@@ -49,6 +53,19 @@ pub struct BucketReplicationTargetStats {
pub latency_ms: f64,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct BucketReplicationTargetFlowStats {
pub(crate) target_arn: String,
pub sent_bytes: u64,
pub sent_count: u64,
pub total_failed_bytes: u64,
pub total_failed_count: u64,
pub last_min_failed_bytes: u64,
pub last_min_failed_count: u64,
pub last_hour_failed_bytes: u64,
pub last_hour_failed_count: u64,
}
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationBandwidthStats {
pub bucket: String,
@@ -88,6 +105,12 @@ pub struct BucketReplicationStats {
pub targets: Vec<BucketReplicationTargetStats>,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct BucketReplicationRuntimeStats {
pub(crate) stats: BucketReplicationStats,
pub(crate) target_flows: Vec<BucketReplicationTargetFlowStats>,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct BucketReplicationBacklogStats {
pub(crate) bucket: String,
@@ -140,6 +163,25 @@ pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBa
metrics
}
fn push_proxy_request_result_metrics(
metrics: &mut Vec<PrometheusMetric>,
bucket_label: Cow<'static, str>,
operation: &'static str,
total: u64,
failures: u64,
) {
let failure_count = failures.min(total);
let success_count = total.saturating_sub(failure_count);
for (result, value) in [("success", success_count), ("failure", failure_count)] {
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD, value as f64)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(OPERATION_L, operation)
.with_label(RESULT_L, result),
);
}
}
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
@@ -263,6 +305,48 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
)
.with_label(BUCKET_L, bucket_label.clone()),
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"get",
stat.proxied_get_requests_total,
stat.proxied_get_requests_failures,
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"head",
stat.proxied_head_requests_total,
stat.proxied_head_requests_failures,
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"put",
stat.proxied_put_requests_total,
stat.proxied_put_requests_failures,
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"put_tagging",
stat.proxied_put_tagging_requests_total,
stat.proxied_put_tagging_requests_failures,
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"get_tagging",
stat.proxied_get_tagging_requests_total,
stat.proxied_get_tagging_requests_failures,
);
push_proxy_request_result_metrics(
&mut metrics,
bucket_label.clone(),
"delete_tagging",
stat.proxied_delete_tagging_requests_total,
stat.proxied_delete_tagging_requests_failures,
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, stat.resync_started_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
@@ -298,6 +382,81 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
metrics
}
pub(crate) fn collect_bucket_replication_runtime_metrics(stats: &[BucketReplicationRuntimeStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
let legacy_stats = stats.iter().map(|stat| stat.stats.clone()).collect::<Vec<_>>();
let mut metrics = collect_bucket_replication_metrics(&legacy_stats);
let flow_count = stats
.iter()
.map(|stat| stat.target_flows.len() * BUCKET_REPLICATION_RUNTIME_FLOW_METRICS_PER_TARGET)
.sum();
metrics.reserve(flow_count);
for stat in stats {
let bucket_label: Cow<'static, str> = Cow::Owned(stat.stats.bucket.clone());
for target in &stat.target_flows {
let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_SENT_BYTES_MD, target.sent_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_SENT_COUNT_MD, target.sent_count as f64)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD, target.total_failed_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD, target.total_failed_count as f64)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD,
target.last_min_failed_bytes as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD,
target.last_min_failed_count as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD,
target.last_hour_failed_bytes as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD,
target.last_hour_failed_count as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
}
}
metrics
}
pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicationBacklogStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
@@ -412,43 +571,56 @@ mod tests {
#[test]
fn test_collect_bucket_replication_metrics() {
let stats = vec![BucketReplicationStats {
bucket: "b1".to_string(),
total_failed_bytes: 64,
total_failed_count: 2,
last_min_failed_bytes: 32,
last_min_failed_count: 1,
last_hour_failed_bytes: 64,
last_hour_failed_count: 2,
sent_bytes: 1024,
sent_count: 8,
proxied_get_requests_total: 5,
proxied_get_requests_failures: 1,
proxied_head_requests_total: 4,
proxied_head_requests_failures: 0,
proxied_put_requests_total: 6,
proxied_put_requests_failures: 2,
proxied_put_tagging_requests_total: 3,
proxied_put_tagging_requests_failures: 1,
proxied_get_tagging_requests_total: 2,
proxied_get_tagging_requests_failures: 0,
proxied_delete_tagging_requests_total: 1,
proxied_delete_tagging_requests_failures: 1,
resync_started_count: 2,
resync_completed_count: 1,
resync_failed_count: 1,
resync_canceled_count: 0,
resync_duration_ms: 1500,
targets: vec![BucketReplicationTargetStats {
let stats = vec![BucketReplicationRuntimeStats {
stats: BucketReplicationStats {
bucket: "b1".to_string(),
total_failed_bytes: 64,
total_failed_count: 2,
last_min_failed_bytes: 32,
last_min_failed_count: 1,
last_hour_failed_bytes: 64,
last_hour_failed_count: 2,
sent_bytes: 1024,
sent_count: 8,
proxied_get_requests_total: 5,
proxied_get_requests_failures: 1,
proxied_head_requests_total: 4,
proxied_head_requests_failures: 0,
proxied_put_requests_total: 6,
proxied_put_requests_failures: 2,
proxied_put_tagging_requests_total: 3,
proxied_put_tagging_requests_failures: 1,
proxied_get_tagging_requests_total: 2,
proxied_get_tagging_requests_failures: 0,
proxied_delete_tagging_requests_total: 1,
proxied_delete_tagging_requests_failures: 1,
resync_started_count: 2,
resync_completed_count: 1,
resync_failed_count: 1,
resync_canceled_count: 0,
resync_duration_ms: 1500,
targets: vec![BucketReplicationTargetStats {
target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(),
bandwidth_limit_bytes_per_sec: 2048,
current_bandwidth_bytes_per_sec: 1024.0,
latency_ms: 15.0,
}],
},
target_flows: vec![BucketReplicationTargetFlowStats {
target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(),
bandwidth_limit_bytes_per_sec: 2048,
current_bandwidth_bytes_per_sec: 1024.0,
latency_ms: 15.0,
sent_bytes: 512,
sent_count: 4,
total_failed_bytes: 96,
total_failed_count: 3,
last_min_failed_bytes: 32,
last_min_failed_count: 1,
last_hour_failed_bytes: 64,
last_hour_failed_count: 2,
}],
}];
let metrics = collect_bucket_replication_metrics(&stats);
assert_eq!(metrics.len(), 26);
let metrics = collect_bucket_replication_runtime_metrics(&stats);
assert_eq!(metrics.len(), 46);
let sent_name = BUCKET_REPL_SENT_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
@@ -471,6 +643,28 @@ mod tests {
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let proxy_requests_name = BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == proxy_requests_name
&& metric.value == 4.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric.labels.iter().any(|(key, value)| *key == OPERATION_L && value == "put")
&& metric
.labels
.iter()
.any(|(key, value)| *key == RESULT_L && value == "success")
}));
assert!(metrics.iter().any(|metric| {
metric.name == proxy_requests_name
&& metric.value == 2.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric.labels.iter().any(|(key, value)| *key == OPERATION_L && value == "put")
&& metric
.labels
.iter()
.any(|(key, value)| *key == RESULT_L && value == "failure")
}));
let latency_name = BUCKET_REPL_LATENCY_MS_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == latency_name
@@ -481,6 +675,27 @@ mod tests {
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
}));
let target_sent_name = BUCKET_REPL_TARGET_SENT_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == target_sent_name
&& metric.value == 4.0
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
}));
let target_last_min_failed_name = BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == target_last_min_failed_name
&& metric.value == 32.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
}));
let delete_tagging_total_name = BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == delete_tagging_total_name
crates/obs/src/metrics/collectors/ilm.rs
+87 -2
View File
@@ -48,6 +48,26 @@ pub struct IlmStats {
pub versions_scanned: u64,
}
/// ILM task metrics by action and state.
#[derive(Debug, Clone, Default)]
pub(crate) struct IlmActionTaskStats {
pub(crate) action: String,
pub(crate) state: String,
pub(crate) value: u64,
}
/// ILM statistics with runtime-local node identity and bounded action/state details.
#[derive(Debug, Clone, Default)]
pub(crate) struct IlmRuntimeStats {
pub(crate) server: String,
pub(crate) stats: IlmStats,
pub(crate) action_tasks: Vec<IlmActionTaskStats>,
}
fn is_live_action_task_state(state: &str) -> bool {
matches!(state, "pending" | "active" | "compensation_running")
}
/// Collects ILM metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::ilm` module.
@@ -78,6 +98,25 @@ pub fn collect_ilm_metrics(stats: &IlmStats) -> Vec<PrometheusMetric> {
]
}
pub(crate) fn collect_ilm_runtime_metrics(stats: &IlmRuntimeStats) -> Vec<PrometheusMetric> {
let mut metrics = collect_ilm_metrics(&stats.stats);
metrics.extend(
stats
.action_tasks
.iter()
.filter(|task| is_live_action_task_state(&task.state))
.map(|task| {
PrometheusMetric::from_descriptor(&ILM_ACTION_TASKS_MD, task.value as f64)
.with_label_owned(SERVER_LABEL, stats.server.clone())
.with_label_owned(ACTION_LABEL, task.action.clone())
.with_label_owned(STATE_LABEL, task.state.clone())
}),
);
metrics
}
#[cfg(test)]
mod tests {
use super::*;
@@ -95,16 +134,62 @@ mod tests {
transition_compensation_running_tasks: 1,
versions_scanned: 1000000,
};
let runtime_stats = IlmRuntimeStats {
server: "node1:9000".to_string(),
stats,
action_tasks: vec![
IlmActionTaskStats {
action: "expiry".to_string(),
state: "pending".to_string(),
value: 100,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "queue_send_timeout".to_string(),
value: 3,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "active".to_string(),
value: 5,
},
],
};
let metrics = collect_ilm_metrics(&stats);
let metrics = collect_ilm_runtime_metrics(&runtime_stats);
assert_eq!(metrics.len(), 9);
assert_eq!(metrics.len(), 11);
let pending = metrics.iter().find(|m| m.value == 100.0);
assert!(pending.is_some());
let scanned = metrics.iter().find(|m| m.value == 1000000.0);
assert!(scanned.is_some());
let transition_timeout = metrics.iter().find(|m| {
m.name == ILM_ACTION_TASKS_MD.get_full_metric_name()
&& m.labels
.iter()
.any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
&& m.labels
.iter()
.any(|(name, value)| *name == ACTION_LABEL && value.as_ref() == "transition")
&& m.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "queue_send_timeout")
});
assert!(transition_timeout.is_none());
let transition_active = metrics.iter().find(|m| {
m.name == ILM_ACTION_TASKS_MD.get_full_metric_name()
&& m.labels
.iter()
.any(|(name, value)| *name == ACTION_LABEL && value.as_ref() == "transition")
&& m.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "active")
});
assert_eq!(transition_active.map(|m| m.value), Some(5.0));
}
#[test]
crates/obs/src/metrics/collectors/mod.rs
+9 -1
View File
@@ -40,10 +40,12 @@ pub mod system_network;
pub mod system_network_host;
pub mod system_process;
pub(crate) use audit::{AuditTargetRuntimeStats, collect_audit_runtime_metrics};
pub use audit::{AuditTargetStats, collect_audit_metrics};
pub use bucket::{BucketStats, collect_bucket_metrics};
pub(crate) use bucket_replication::{
BucketReplicationBacklogStats, BucketReplicationTargetBacklogStats, collect_bucket_replication_backlog_metrics,
BucketReplicationBacklogStats, BucketReplicationRuntimeStats, BucketReplicationTargetBacklogStats,
BucketReplicationTargetFlowStats, collect_bucket_replication_backlog_metrics, collect_bucket_replication_runtime_metrics,
};
pub use bucket_replication::{
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
@@ -57,18 +59,24 @@ pub use cluster_iam::{IamStats, collect_iam_metrics};
pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics};
pub use compression::{CompressionClusterStats, collect_compression_cluster_metrics};
pub use dial9::{Dial9Stats, collect_current_dial9_metrics, collect_dial9_metrics, is_dial9_enabled};
pub(crate) use ilm::{IlmActionTaskStats, IlmRuntimeStats, collect_ilm_runtime_metrics};
pub use ilm::{IlmStats, collect_ilm_metrics};
pub use node::{DiskStats, collect_node_metrics};
pub use notification::{NotificationStats, collect_notification_metrics};
pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};
pub(crate) use replication::{ReplicationRuntimeStats, collect_replication_runtime_metrics};
pub use replication::{ReplicationStats, collect_replication_metrics};
pub(crate) use request::{ApiRequestMetricSupport, ApiRequestStats, collect_request_metrics};
pub use resource::{ResourceStats, collect_resource_metrics};
pub(crate) use scanner::{ScannerRuntimeStats, collect_scanner_runtime_metrics};
pub use scanner::{ScannerStats, collect_scanner_metrics};
pub use system_cpu::{CpuStats, ProcessCpuStats, collect_cpu_metrics, collect_process_cpu_metrics};
pub use system_drive::{
DriveCountStats, DriveDetailedStats, ProcessDiskStats, collect_drive_count_metrics, collect_drive_detailed_metrics,
collect_process_disk_metrics,
};
pub(crate) use system_drive::{DriveRuntimeDetailedStats, collect_drive_runtime_detailed_metrics};
#[cfg(feature = "gpu")]
pub use system_gpu::{GpuCollector, GpuError, GpuStats, collect_gpu_metrics};
pub use system_memory::{MemoryStats, ProcessMemoryStats, collect_memory_metrics, collect_process_memory_metrics};
crates/obs/src/metrics/collectors/notification_target.rs
+98 -6
View File
@@ -16,8 +16,10 @@
use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::notification_target::{
NOTIFICATION_TARGET_FAILED_MESSAGES_MD, NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD,
NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, TARGET_ID, TARGET_TYPE,
NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD, NOTIFICATION_TARGET_FAILED_MESSAGES_MD,
NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD, NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD,
NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD,
NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD, NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, SERVER, TARGET_ID, TARGET_TYPE,
};
use std::borrow::Cow;
@@ -31,12 +33,18 @@ pub struct NotificationTargetStats {
pub total_messages: u64,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct NotificationTargetRuntimeStats {
pub(crate) server: String,
pub(crate) target: NotificationTargetStats,
}
pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
let mut metrics = Vec::with_capacity(stats.len() * 4);
let mut metrics = Vec::with_capacity(stats.len() * 8);
for stat in stats {
let target_id: Cow<'static, str> = Cow::Owned(stat.target_id.clone());
let target_type: Cow<'static, str> = Cow::Owned(stat.target_type.clone());
@@ -66,6 +74,57 @@ pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) ->
metrics
}
pub(crate) fn collect_notification_target_runtime_metrics(stats: &[NotificationTargetRuntimeStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
let legacy_stats = stats.iter().map(|stat| stat.target.clone()).collect::<Vec<_>>();
let mut metrics = collect_notification_target_metrics(&legacy_stats);
metrics.reserve(stats.len() * 4);
for stat in stats {
let server: Cow<'static, str> = Cow::Owned(stat.server.clone());
let target_id: Cow<'static, str> = Cow::Owned(stat.target.target_id.clone());
let target_type: Cow<'static, str> = Cow::Owned(stat.target.target_type.clone());
metrics.push(
PrometheusMetric::from_descriptor(
&NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD,
stat.target.failed_messages as f64,
)
.with_label(SERVER, server.clone())
.with_label(TARGET_ID, target_id.clone())
.with_label(TARGET_TYPE, target_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD,
stat.target.failed_store_length as f64,
)
.with_label(SERVER, server.clone())
.with_label(TARGET_ID, target_id.clone())
.with_label(TARGET_TYPE, target_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD, stat.target.queue_length as f64)
.with_label(SERVER, server.clone())
.with_label(TARGET_ID, target_id.clone())
.with_label(TARGET_TYPE, target_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD,
stat.target.total_messages as f64,
)
.with_label(SERVER, server)
.with_label(TARGET_ID, target_id)
.with_label(TARGET_TYPE, target_type),
);
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
@@ -73,7 +132,7 @@ mod tests {
#[test]
fn test_collect_notification_target_metrics() {
let stats = vec![NotificationTargetStats {
let stats = [NotificationTargetStats {
failed_messages: 2,
failed_store_length: 3,
queue_length: 4,
@@ -82,9 +141,12 @@ mod tests {
total_messages: 42,
}];
let metrics = collect_notification_target_metrics(&stats);
let metrics = collect_notification_target_runtime_metrics(&[NotificationTargetRuntimeStats {
server: "node1:9000".to_string(),
target: stats[0].clone(),
}]);
assert_eq!(metrics.len(), 4);
assert_eq!(metrics.len(), 8);
assert!(metrics.iter().any(|metric| {
metric.value == 3.0
&& metric.name == NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD.get_full_metric_name()
@@ -104,6 +166,22 @@ mod tests {
.iter()
.any(|(key, value)| *key == TARGET_TYPE && value == "webhook")
}));
assert!(metrics.iter().any(|metric| {
metric.value == 4.0
&& metric.name == NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name()
&& metric
.labels
.iter()
.any(|(key, value)| *key == SERVER && value == "node1:9000")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ID && value == "primary:webhook")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_TYPE && value == "webhook")
}));
}
#[test]
@@ -113,4 +191,18 @@ mod tests {
assert_eq!(NOTIFICATION_TARGET_QUEUE_LENGTH_MD.metric_type, MetricType::Gauge);
assert_eq!(NOTIFICATION_TARGET_TOTAL_MESSAGES_MD.metric_type, MetricType::Gauge);
}
#[test]
fn notification_target_stats_struct_literal_keeps_legacy_fields() {
let stats = vec![NotificationTargetStats {
failed_messages: 2,
failed_store_length: 3,
queue_length: 4,
target_id: "primary:webhook".to_string(),
target_type: "webhook".to_string(),
total_messages: 42,
}];
assert_eq!(collect_notification_target_metrics(&stats).len(), 4);
}
}
crates/obs/src/metrics/collectors/replication.rs
+92 -2
View File
@@ -53,6 +53,12 @@ pub struct ReplicationStats {
pub recent_backlog_count: u64,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct ReplicationRuntimeStats {
pub(crate) server: String,
pub(crate) stats: ReplicationStats,
}
/// Collects replication metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for replication statistics.
@@ -74,6 +80,41 @@ pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMe
]
}
pub(crate) fn collect_replication_runtime_metrics(runtime: &ReplicationRuntimeStats) -> Vec<PrometheusMetric> {
let stats = &runtime.stats;
let mut metrics = collect_replication_metrics(stats);
metrics.extend([
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_BY_SERVER_MD, stats.average_active_workers)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_BY_SERVER_MD, stats.average_queued_bytes as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_COUNT_BY_SERVER_MD, stats.average_queued_count as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD, stats.active_workers as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_BY_SERVER_MD, stats.current_data_transfer_rate)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(
&REPLICATION_LAST_MINUTE_QUEUED_BYTES_BY_SERVER_MD,
stats.last_minute_queued_bytes as f64,
)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(
&REPLICATION_LAST_MINUTE_QUEUED_COUNT_BY_SERVER_MD,
stats.last_minute_queued_count as f64,
)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_ACTIVE_WORKERS_BY_SERVER_MD, stats.max_active_workers as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_BYTES_BY_SERVER_MD, stats.max_queued_bytes as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_COUNT_BY_SERVER_MD, stats.max_queued_count as f64)
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
]);
metrics
}
#[cfg(test)]
mod tests {
use super::*;
@@ -97,10 +138,13 @@ mod tests {
recent_backlog_count: 1500,
};
let metrics = collect_replication_metrics(&stats);
let metrics = collect_replication_runtime_metrics(&ReplicationRuntimeStats {
server: "node-a:9000".to_string(),
stats,
});
report_metrics(&metrics);
assert_eq!(metrics.len(), 13);
assert_eq!(metrics.len(), 23);
// Verify active workers
let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name();
@@ -111,6 +155,31 @@ mod tests {
let avg_active_name = REPLICATION_AVERAGE_ACTIVE_WORKERS_MD.get_full_metric_name();
let avg_active = metrics.iter().find(|m| m.name == avg_active_name);
assert_eq!(avg_active.map(|m| m.value), Some(8.5));
let active_by_server_name = REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD.get_full_metric_name();
let active_by_server = metrics.iter().find(|m| m.name == active_by_server_name);
assert_eq!(active_by_server.map(|m| m.value), Some(10.0));
assert_eq!(
active_by_server
.and_then(|m| m.labels.iter().find(|(name, _)| *name == SERVER_LABEL))
.map(|(_, value)| value.as_ref()),
Some("node-a:9000")
);
assert!(
metrics
.iter()
.all(|m| m.name != REPLICATION_AVERAGE_DATA_TRANSFER_RATE_BY_SERVER_MD.get_full_metric_name())
);
assert!(
metrics
.iter()
.all(|m| m.name != REPLICATION_MAX_DATA_TRANSFER_RATE_BY_SERVER_MD.get_full_metric_name())
);
assert!(
metrics
.iter()
.all(|m| m.name != REPLICATION_RECENT_BACKLOG_COUNT_BY_SERVER_MD.get_full_metric_name())
);
}
#[test]
@@ -124,4 +193,25 @@ mod tests {
assert!(metric.labels.is_empty());
}
}
#[test]
fn replication_stats_struct_literal_keeps_legacy_fields() {
let stats = ReplicationStats {
average_active_workers: 1.0,
average_queued_bytes: 2,
average_queued_count: 3,
average_data_transfer_rate: 4.0,
active_workers: 5,
current_data_transfer_rate: 6.0,
last_minute_queued_bytes: 7,
last_minute_queued_count: 8,
max_active_workers: 9,
max_queued_bytes: 10,
max_queued_count: 11,
max_data_transfer_rate: 12.0,
recent_backlog_count: 13,
};
assert_eq!(collect_replication_metrics(&stats).len(), 13);
}
}
crates/obs/src/metrics/collectors/request.rs
+239 -47
View File
@@ -23,9 +23,32 @@ use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::request::*;
use std::collections::HashMap;
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub(crate) struct ApiRequestMetricSupport {
pub(crate) lifecycle: bool,
pub(crate) traffic: bool,
pub(crate) ttfb: bool,
}
impl ApiRequestMetricSupport {
pub(crate) const ALL: Self = Self {
lifecycle: true,
traffic: true,
ttfb: true,
};
pub(crate) const TOTALS_ONLY: Self = Self {
lifecycle: false,
traffic: false,
ttfb: false,
};
}
/// API request statistics for a specific API endpoint.
#[derive(Debug, Clone, Default)]
#[derive(Debug, Clone)]
pub struct ApiRequestStats {
/// Server identifier
pub server: String,
/// API name (e.g., "GetObject", "PutObject")
pub name: String,
/// Request type (e.g., "s3", "admin")
@@ -48,6 +71,27 @@ pub struct ApiRequestStats {
pub sent_bytes: u64,
/// Bytes received
pub recv_bytes: u64,
pub(crate) supported_metrics: ApiRequestMetricSupport,
}
impl Default for ApiRequestStats {
fn default() -> Self {
Self {
server: String::new(),
name: String::new(),
req_type: String::new(),
in_flight: 0,
total: 0,
errors_total: 0,
errors_5xx: 0,
errors_4xx: 0,
canceled: 0,
ttfb_distribution: Vec::new(),
sent_bytes: 0,
recv_bytes: 0,
supported_metrics: ApiRequestMetricSupport::ALL,
}
}
}
/// Collects API request metrics from the given stats.
@@ -56,62 +100,119 @@ pub struct ApiRequestStats {
pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec<PrometheusMetric> {
let mut metrics = Vec::new();
let mut traffic_by_type: HashMap<&str, (u64, u64)> = HashMap::with_capacity(stats.len());
let mut traffic_by_server_type: HashMap<(&str, &str), (u64, u64)> = HashMap::with_capacity(stats.len());
for stat in stats {
let entry = traffic_by_type.entry(stat.req_type.as_str()).or_default();
entry.0 = entry.0.saturating_add(stat.sent_bytes);
entry.1 = entry.1.saturating_add(stat.recv_bytes);
if stat.supported_metrics.traffic {
let entry = traffic_by_type.entry(stat.req_type.as_str()).or_default();
entry.0 = entry.0.saturating_add(stat.sent_bytes);
entry.1 = entry.1.saturating_add(stat.recv_bytes);
if !stat.server.is_empty() {
let entry = traffic_by_server_type
.entry((stat.server.as_str(), stat.req_type.as_str()))
.or_default();
entry.0 = entry.0.saturating_add(stat.sent_bytes);
entry.1 = entry.1.saturating_add(stat.recv_bytes);
}
}
// In-flight requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Total requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_MD, stat.total as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Total errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// 5xx errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// 4xx errors
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// Canceled requests
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
// TTFB distribution (histogram buckets)
for (le, value) in &stat.ttfb_distribution {
if stat.supported_metrics.lifecycle {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value)
PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone())
.with_label_owned(LE_LABEL, le.clone()),
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
}
if stat.supported_metrics.ttfb {
for (le, value) in &stat.ttfb_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value)
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone())
.with_label_owned(LE_LABEL, le.clone()),
);
}
}
if !stat.server.is_empty() {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_BY_SERVER_MD, stat.total as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
if stat.supported_metrics.lifecycle {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_BY_SERVER_MD, stat.in_flight as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_total as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_5xx as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_4xx as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_BY_SERVER_MD, stat.canceled as f64)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
);
}
if stat.supported_metrics.ttfb {
for (le, value) in &stat.ttfb_distribution {
metrics.push(
PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_BY_SERVER_MD, *value)
.with_label_owned(SERVER_LABEL, stat.server.clone())
.with_label_owned(NAME_LABEL, stat.name.clone())
.with_label_owned(TYPE_LABEL, stat.req_type.clone())
.with_label_owned(LE_LABEL, le.clone()),
);
}
}
}
}
@@ -126,6 +227,19 @@ pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec<PrometheusMetri
);
}
for ((server, req_type), (sent_bytes, recv_bytes)) in traffic_by_server_type {
metrics.push(
PrometheusMetric::from_descriptor(&API_TRAFFIC_SENT_BYTES_BY_SERVER_MD, sent_bytes as f64)
.with_label_owned(SERVER_LABEL, server.to_string())
.with_label_owned(TYPE_LABEL, req_type.to_string()),
);
metrics.push(
PrometheusMetric::from_descriptor(&API_TRAFFIC_RECV_BYTES_BY_SERVER_MD, recv_bytes as f64)
.with_label_owned(SERVER_LABEL, server.to_string())
.with_label_owned(TYPE_LABEL, req_type.to_string()),
);
}
metrics
}
@@ -137,6 +251,7 @@ mod tests {
#[test]
fn test_collect_request_metrics() {
let stats = vec![ApiRequestStats {
server: "node1:9000".to_string(),
name: "GetObject".to_string(),
req_type: "s3".to_string(),
in_flight: 10,
@@ -153,13 +268,13 @@ mod tests {
],
sent_bytes: 1024 * 1024 * 500, // 500 MB
recv_bytes: 1024 * 1024 * 100, // 100 MB
supported_metrics: ApiRequestMetricSupport::ALL,
}];
let metrics = collect_request_metrics(&stats);
report_metrics(&metrics);
// 6 base metrics + 4 TTFB buckets + 2 traffic metrics = 12
assert_eq!(metrics.len(), 12);
assert_eq!(metrics.len(), 24);
let total_name = API_REQUESTS_TOTAL_MD.get_full_metric_name();
let total = metrics.iter().find(|m| m.name == total_name);
@@ -170,6 +285,27 @@ mod tests {
let in_flight = metrics.iter().find(|m| m.name == in_flight_name);
assert!(in_flight.is_some());
assert_eq!(in_flight.map(|m| m.value), Some(10.0));
let by_server_total_name = API_REQUESTS_TOTAL_BY_SERVER_MD.get_full_metric_name();
let by_server_total = metrics.iter().find(|m| {
m.name == by_server_total_name
&& m.labels
.iter()
.any(|(key, value)| *key == SERVER_LABEL && value == "node1:9000")
&& m.labels.iter().any(|(key, value)| *key == NAME_LABEL && value == "GetObject")
&& m.labels.iter().any(|(key, value)| *key == TYPE_LABEL && value == "s3")
});
assert_eq!(by_server_total.map(|m| m.value), Some(10000.0));
let by_server_sent_name = API_TRAFFIC_SENT_BYTES_BY_SERVER_MD.get_full_metric_name();
let by_server_sent = metrics.iter().find(|m| {
m.name == by_server_sent_name
&& m.labels
.iter()
.any(|(key, value)| *key == SERVER_LABEL && value == "node1:9000")
&& m.labels.iter().any(|(key, value)| *key == TYPE_LABEL && value == "s3")
});
assert_eq!(by_server_sent.map(|m| m.value), Some((1024 * 1024 * 500) as f64));
}
#[test]
@@ -179,10 +315,63 @@ mod tests {
assert!(metrics.is_empty());
}
#[test]
fn test_collect_request_metrics_totals_only_skips_unsupported_dimensions() {
let stats = vec![ApiRequestStats {
server: "node1:9000".to_string(),
name: "GetObject".to_string(),
req_type: "s3".to_string(),
in_flight: 10,
total: 100,
errors_total: 5,
errors_5xx: 2,
errors_4xx: 3,
canceled: 1,
ttfb_distribution: vec![("+Inf".to_string(), 100.0)],
sent_bytes: 2048,
recv_bytes: 1024,
supported_metrics: ApiRequestMetricSupport::TOTALS_ONLY,
}];
let metrics = collect_request_metrics(&stats);
assert!(
metrics
.iter()
.any(|metric| metric.name == API_REQUESTS_TOTAL_MD.get_full_metric_name())
);
assert!(
metrics
.iter()
.any(|metric| metric.name == API_REQUESTS_TOTAL_BY_SERVER_MD.get_full_metric_name())
);
assert!(
!metrics
.iter()
.any(|metric| metric.name == API_REQUESTS_IN_FLIGHT_TOTAL_MD.get_full_metric_name())
);
assert!(
!metrics
.iter()
.any(|metric| metric.name == API_REQUESTS_ERRORS_TOTAL_MD.get_full_metric_name())
);
assert!(
!metrics
.iter()
.any(|metric| metric.name == API_TRAFFIC_SENT_BYTES_MD.get_full_metric_name())
);
assert!(
!metrics
.iter()
.any(|metric| metric.name == API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD.get_full_metric_name())
);
}
#[test]
fn test_collect_request_metrics_aggregates_traffic_per_type() {
let stats = vec![
ApiRequestStats {
server: String::new(),
name: "GetObject".to_string(),
req_type: "s3".to_string(),
in_flight: 1,
@@ -194,8 +383,10 @@ mod tests {
ttfb_distribution: vec![],
sent_bytes: 100,
recv_bytes: 10,
supported_metrics: ApiRequestMetricSupport::ALL,
},
ApiRequestStats {
server: String::new(),
name: "HeadObject".to_string(),
req_type: "s3".to_string(),
in_flight: 2,
@@ -207,6 +398,7 @@ mod tests {
ttfb_distribution: vec![],
sent_bytes: 200,
recv_bytes: 20,
supported_metrics: ApiRequestMetricSupport::ALL,
},
];
crates/obs/src/metrics/collectors/scanner.rs
+318 -102
View File
@@ -20,34 +20,7 @@
//! directory scans, and object scans.
use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::scanner::{
SCANNER_ACTIVE_PATHS_MD, SCANNER_BITROT_CYCLE_ENABLED_MD, SCANNER_BITROT_CYCLE_SECONDS_MD, SCANNER_BUCKET_SCANS_FAILED_MD,
SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD,
SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD,
SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, SCANNER_CURRENT_CYCLE_MD,
SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD,
SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD, SCANNER_CURRENT_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD,
SCANNER_CURRENT_CYCLE_THROTTLE_SLEEP_EVENTS_MD, SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD,
SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD,
SCANNER_CURRENT_DISK_BUCKET_SCANS_ACTIVE_MD, SCANNER_CURRENT_DISK_BUCKET_SCANS_QUEUED_MD,
SCANNER_CURRENT_DISK_SCAN_CONCURRENCY_LIMIT_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_CURRENT_SET_SCAN_CONCURRENCY_LIMIT_MD,
SCANNER_CURRENT_SET_SCANS_ACTIVE_MD, SCANNER_CURRENT_SET_SCANS_QUEUED_MD, SCANNER_CYCLE_INTERVAL_SECONDS_MD,
SCANNER_CYCLE_MAX_DIRECTORIES_MD, SCANNER_CYCLE_MAX_DURATION_SECONDS_MD, SCANNER_CYCLE_MAX_OBJECTS_MD,
SCANNER_DIRECTORIES_SCANNED_MD, SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD,
SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD,
SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD,
SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD,
SCANNER_LAST_CYCLE_PARTIAL_REASON_MD, SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, SCANNER_LAST_CYCLE_RESULT_MD,
SCANNER_LAST_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_THROTTLE_SLEEP_EVENTS_MD,
SCANNER_LAST_CYCLE_USAGE_SAVES_MD, SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD,
SCANNER_OBJECTS_SCANNED_MD, SCANNER_OLDEST_ACTIVE_PATH_AGE_SECONDS_MD, SCANNER_PARTIAL_CYCLES_BY_REASON_MD,
SCANNER_PARTIAL_CYCLES_MD, SCANNER_SUPERSEDED_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD,
SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD,
SCANNER_YIELD_EVERY_N_OBJECTS_MD,
};
use crate::metrics::schema::scanner::*;
/// Scanner statistics.
#[derive(Debug, Clone, Default)]
@@ -192,12 +165,123 @@ pub struct ScannerStats {
pub partial_cycles_directories: u64,
}
/// Scanner source-work metrics for a source.
#[derive(Debug, Clone, Default)]
pub struct ScannerSourceWorkStats {
pub source: String,
pub checked: u64,
pub queued: u64,
pub executed: u64,
pub failed: u64,
pub skipped: u64,
pub missed: u64,
}
/// Scanner bucket-drive result metrics for a structured bucket/drive pair.
#[derive(Debug, Clone, Default)]
pub struct ScannerBucketDriveResultStats {
pub bucket: String,
pub drive: String,
pub result: String,
pub count: u64,
}
/// Scanner statistics with runtime-local node identity and bounded source/result details.
#[derive(Debug, Clone, Default)]
pub(crate) struct ScannerRuntimeStats {
pub(crate) server: String,
pub(crate) stats: ScannerStats,
pub(crate) source_work: Vec<ScannerSourceWorkStats>,
pub(crate) current_cycle_source_work: Vec<ScannerSourceWorkStats>,
pub(crate) last_cycle_source_work: Vec<ScannerSourceWorkStats>,
pub(crate) bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
pub(crate) current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
pub(crate) last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
}
/// Collects scanner metrics from the given stats.
///
/// Uses the metric descriptors from `metrics_type::scanner` module.
/// Returns a vector of Prometheus metrics for scanner statistics.
pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
vec![
collect_scanner_metrics_with_runtime(stats, None)
}
pub(crate) fn collect_scanner_runtime_metrics(stats: &ScannerRuntimeStats) -> Vec<PrometheusMetric> {
collect_scanner_metrics_with_runtime(&stats.stats, Some(stats))
}
fn collect_scanner_metrics_with_runtime(stats: &ScannerStats, runtime: Option<&ScannerRuntimeStats>) -> Vec<PrometheusMetric> {
fn push_source_work_metric(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
server: &str,
source: &str,
state: &str,
value: u64,
cycle_scope: Option<&str>,
) {
let mut metric =
PrometheusMetric::from_descriptor(descriptor, value as f64).with_label_owned(SERVER_LABEL, server.to_string());
if let Some(cycle_scope) = cycle_scope {
metric = metric.with_label_owned(CYCLE_SCOPE_LABEL, cycle_scope.to_string());
}
metric = metric
.with_label_owned(SOURCE_LABEL, source.to_string())
.with_label_owned(STATE_LABEL, state.to_string());
metrics.push(metric);
}
fn push_source_work_metrics(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
server: &str,
source_work: &[ScannerSourceWorkStats],
cycle_scope: Option<&str>,
) {
for work in source_work {
push_source_work_metric(metrics, descriptor, server, &work.source, "checked", work.checked, cycle_scope);
push_source_work_metric(metrics, descriptor, server, &work.source, "queued", work.queued, cycle_scope);
push_source_work_metric(metrics, descriptor, server, &work.source, "executed", work.executed, cycle_scope);
push_source_work_metric(metrics, descriptor, server, &work.source, "failed", work.failed, cycle_scope);
push_source_work_metric(metrics, descriptor, server, &work.source, "skipped", work.skipped, cycle_scope);
push_source_work_metric(metrics, descriptor, server, &work.source, "missed", work.missed, cycle_scope);
}
}
fn push_bucket_drive_result_metric(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
server: &str,
result: &ScannerBucketDriveResultStats,
cycle_scope: Option<&str>,
) {
let mut metric =
PrometheusMetric::from_descriptor(descriptor, result.count as f64).with_label_owned(SERVER_LABEL, server.to_string());
if let Some(cycle_scope) = cycle_scope {
metric = metric.with_label_owned(CYCLE_SCOPE_LABEL, cycle_scope.to_string());
}
metrics.push(
metric
.with_label_owned(BUCKET_LABEL, result.bucket.clone())
.with_label_owned(DRIVE_LABEL, result.drive.clone())
.with_label_owned(RESULT_LABEL, result.result.clone()),
);
}
fn push_bucket_drive_result_metrics(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
server: &str,
results: &[ScannerBucketDriveResultStats],
cycle_scope: Option<&str>,
) {
for result in results {
push_bucket_drive_result_metric(metrics, descriptor, server, result, cycle_scope);
}
}
let mut metrics = vec![
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FAILED_MD, stats.bucket_scans_failed as f64),
@@ -331,7 +415,48 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
.with_label("reason", "objects"),
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_directories as f64)
.with_label("reason", "directories"),
]
];
if let Some(runtime) = runtime {
push_source_work_metrics(&mut metrics, &SCANNER_SOURCE_WORK_TOTAL_MD, &runtime.server, &runtime.source_work, None);
push_source_work_metrics(
&mut metrics,
&SCANNER_CYCLE_SOURCE_WORK_MD,
&runtime.server,
&runtime.current_cycle_source_work,
Some("current"),
);
push_source_work_metrics(
&mut metrics,
&SCANNER_CYCLE_SOURCE_WORK_MD,
&runtime.server,
&runtime.last_cycle_source_work,
Some("last"),
);
push_bucket_drive_result_metrics(
&mut metrics,
&SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD,
&runtime.server,
&runtime.bucket_drive_results,
None,
);
push_bucket_drive_result_metrics(
&mut metrics,
&SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD,
&runtime.server,
&runtime.current_cycle_bucket_drive_results,
Some("current"),
);
push_bucket_drive_result_metrics(
&mut metrics,
&SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD,
&runtime.server,
&runtime.last_cycle_bucket_drive_results,
Some("last"),
);
}
metrics
}
fn bool_metric_value(enabled: bool) -> f64 {
@@ -345,82 +470,130 @@ mod tests {
#[test]
fn test_collect_scanner_metrics() {
let stats = ScannerStats {
bucket_scans_finished: 100,
bucket_scans_started: 100,
bucket_scans_failed: 2,
directories_scanned: 50000,
objects_scanned: 1000000,
versions_scanned: 1500000,
last_activity_seconds: 30,
active_paths: 4,
oldest_active_path_age_seconds: 17,
current_set_scan_concurrency_limit: 3,
current_set_scans_queued: 5,
current_set_scans_active: 2,
current_disk_scan_concurrency_limit: 6,
current_disk_bucket_scans_queued: 18,
current_disk_bucket_scans_active: 4,
throttle_idle_mode_enabled: true,
throttle_sleep_factor: 10.0,
throttle_max_sleep_seconds: 15.0,
yield_every_n_objects: 128,
cycle_interval_seconds: 3600.0,
cycle_max_duration_seconds: 1800.0,
cycle_max_objects: 1_000_000,
cycle_max_directories: 100_000,
bitrot_cycle_enabled: true,
bitrot_cycle_seconds: 86400.0,
current_cycle: 12,
completed_cycles: 11,
current_cycle_age_seconds: 90,
current_cycle_objects_scanned: 250,
current_cycle_directories_scanned: 20,
current_cycle_bucket_drive_scans: 2,
current_cycle_bucket_drive_failures: 1,
current_cycle_objects_per_second: 12.5,
current_cycle_directories_per_second: 1.0,
current_cycle_bucket_drive_scans_per_second: 0.1,
current_cycle_yield_events: 8,
current_cycle_yield_duration_seconds: 1.25,
current_cycle_throttle_sleep_events: 4,
current_cycle_throttle_sleep_duration_seconds: 2.5,
current_cycle_ilm_actions: 6,
current_cycle_heal_objects: 2,
current_cycle_replication_checks: 5,
current_cycle_usage_saves: 3,
current_scan_mode: 2,
last_cycle_result: 1,
last_cycle_partial_reason: 3,
last_cycle_duration_seconds: 42.5,
last_cycle_objects_scanned: 900,
last_cycle_directories_scanned: 80,
last_cycle_bucket_drive_scans: 6,
last_cycle_bucket_drive_failures: 2,
last_cycle_objects_per_second: 18.0,
last_cycle_directories_per_second: 1.6,
last_cycle_bucket_drive_scans_per_second: 0.12,
last_cycle_yield_events: 30,
last_cycle_yield_duration_seconds: 9.5,
last_cycle_throttle_sleep_events: 12,
last_cycle_throttle_sleep_duration_seconds: 6.75,
last_cycle_ilm_actions: 44,
last_cycle_heal_objects: 7,
last_cycle_replication_checks: 12,
last_cycle_usage_saves: 9,
failed_cycles: 3,
superseded_cycles: 5,
partial_cycles: 10,
partial_cycles_unknown: 1,
partial_cycles_runtime: 2,
partial_cycles_objects: 3,
partial_cycles_directories: 4,
let stats = ScannerRuntimeStats {
server: "node1:9000".to_string(),
source_work: vec![ScannerSourceWorkStats {
source: "lifecycle".to_string(),
checked: 11,
queued: 2,
executed: 3,
failed: 4,
skipped: 5,
missed: 6,
}],
current_cycle_source_work: vec![ScannerSourceWorkStats {
source: "usage".to_string(),
checked: 21,
queued: 7,
executed: 8,
failed: 9,
skipped: 10,
missed: 11,
}],
last_cycle_source_work: vec![ScannerSourceWorkStats {
source: "heal".to_string(),
checked: 31,
queued: 12,
executed: 13,
failed: 14,
skipped: 15,
missed: 16,
}],
bucket_drive_results: vec![ScannerBucketDriveResultStats {
bucket: "photos".to_string(),
drive: "/data1".to_string(),
result: "success".to_string(),
count: 3,
}],
current_cycle_bucket_drive_results: vec![ScannerBucketDriveResultStats {
bucket: "photos".to_string(),
drive: "/data1".to_string(),
result: "partial".to_string(),
count: 1,
}],
last_cycle_bucket_drive_results: vec![ScannerBucketDriveResultStats {
bucket: "videos".to_string(),
drive: "/data2".to_string(),
result: "error".to_string(),
count: 2,
}],
stats: ScannerStats {
bucket_scans_finished: 100,
bucket_scans_started: 100,
bucket_scans_failed: 2,
directories_scanned: 50000,
objects_scanned: 1000000,
versions_scanned: 1500000,
last_activity_seconds: 30,
active_paths: 4,
oldest_active_path_age_seconds: 17,
current_set_scan_concurrency_limit: 3,
current_set_scans_queued: 5,
current_set_scans_active: 2,
current_disk_scan_concurrency_limit: 6,
current_disk_bucket_scans_queued: 18,
current_disk_bucket_scans_active: 4,
throttle_idle_mode_enabled: true,
throttle_sleep_factor: 10.0,
throttle_max_sleep_seconds: 15.0,
yield_every_n_objects: 128,
cycle_interval_seconds: 3600.0,
cycle_max_duration_seconds: 1800.0,
cycle_max_objects: 1_000_000,
cycle_max_directories: 100_000,
bitrot_cycle_enabled: true,
bitrot_cycle_seconds: 86400.0,
current_cycle: 12,
completed_cycles: 11,
current_cycle_age_seconds: 90,
current_cycle_objects_scanned: 250,
current_cycle_directories_scanned: 20,
current_cycle_bucket_drive_scans: 2,
current_cycle_bucket_drive_failures: 1,
current_cycle_objects_per_second: 12.5,
current_cycle_directories_per_second: 1.0,
current_cycle_bucket_drive_scans_per_second: 0.1,
current_cycle_yield_events: 8,
current_cycle_yield_duration_seconds: 1.25,
current_cycle_throttle_sleep_events: 4,
current_cycle_throttle_sleep_duration_seconds: 2.5,
current_cycle_ilm_actions: 6,
current_cycle_heal_objects: 2,
current_cycle_replication_checks: 5,
current_cycle_usage_saves: 3,
current_scan_mode: 2,
last_cycle_result: 1,
last_cycle_partial_reason: 3,
last_cycle_duration_seconds: 42.5,
last_cycle_objects_scanned: 900,
last_cycle_directories_scanned: 80,
last_cycle_bucket_drive_scans: 6,
last_cycle_bucket_drive_failures: 2,
last_cycle_objects_per_second: 18.0,
last_cycle_directories_per_second: 1.6,
last_cycle_bucket_drive_scans_per_second: 0.12,
last_cycle_yield_events: 30,
last_cycle_yield_duration_seconds: 9.5,
last_cycle_throttle_sleep_events: 12,
last_cycle_throttle_sleep_duration_seconds: 6.75,
last_cycle_ilm_actions: 44,
last_cycle_heal_objects: 7,
last_cycle_replication_checks: 12,
last_cycle_usage_saves: 9,
failed_cycles: 3,
superseded_cycles: 5,
partial_cycles: 10,
partial_cycles_unknown: 1,
partial_cycles_runtime: 2,
partial_cycles_objects: 3,
partial_cycles_directories: 4,
},
};
let metrics = collect_scanner_metrics(&stats);
let metrics = collect_scanner_runtime_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 69);
assert_eq!(metrics.len(), 90);
let objects = metrics.iter().find(|m| m.value == 1000000.0);
assert!(objects.is_some());
@@ -432,6 +605,18 @@ mod tests {
.iter()
.find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
assert_eq!(active_paths.map(|m| m.value), Some(4.0));
assert_eq!(active_paths.map(|m| m.labels.len()), Some(0));
let bucket_drive_result = metrics
.iter()
.find(|m| m.name == SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD.get_full_metric_name());
assert_eq!(bucket_drive_result.map(|m| m.value), Some(3.0));
assert_eq!(
bucket_drive_result
.and_then(|m| m.labels.iter().find(|(name, _)| *name == BUCKET_LABEL))
.map(|(_, value)| value.as_ref()),
Some("photos")
);
let oldest_active_path_age = metrics
.iter()
@@ -746,6 +931,37 @@ mod tests {
.any(|(name, value)| *name == "reason" && value.as_ref() == "directories")
});
assert_eq!(partial_cycles_directories.map(|m| m.value), Some(4.0));
let lifecycle_failed = metrics.iter().find(|m| {
m.name == SCANNER_SOURCE_WORK_TOTAL_MD.get_full_metric_name()
&& m.labels
.iter()
.any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
&& m.labels
.iter()
.any(|(name, value)| *name == SOURCE_LABEL && value.as_ref() == "lifecycle")
&& m.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "failed")
});
assert_eq!(lifecycle_failed.map(|m| m.value), Some(4.0));
let current_usage_executed = metrics.iter().find(|m| {
m.name == SCANNER_CYCLE_SOURCE_WORK_MD.get_full_metric_name()
&& m.labels
.iter()
.any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
&& m.labels
.iter()
.any(|(name, value)| *name == CYCLE_SCOPE_LABEL && value.as_ref() == "current")
&& m.labels
.iter()
.any(|(name, value)| *name == SOURCE_LABEL && value.as_ref() == "usage")
&& m.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "executed")
});
assert_eq!(current_usage_executed.map(|m| m.value), Some(8.0));
}
#[test]
crates/obs/src/metrics/collectors/system_drive.rs
+321 -50
View File
@@ -78,6 +78,24 @@ pub struct DriveDetailedStats {
pub perc_util: Option<f64>,
}
/// Detailed drive statistics with runtime topology and per-operation dimensions.
#[derive(Debug, Clone, Default)]
pub(crate) struct DriveRuntimeDetailedStats {
pub(crate) stats: DriveDetailedStats,
pub(crate) pool_index: Option<String>,
pub(crate) set_index: Option<String>,
pub(crate) drive_index: Option<String>,
pub(crate) disk_id: Option<String>,
pub(crate) runtime_state: Option<String>,
pub(crate) healing: bool,
pub(crate) scanning: bool,
pub(crate) offline_duration_seconds: Option<u64>,
/// Drive API calls by operation
pub(crate) api_calls: Vec<(String, u64)>,
/// Last-minute API latency by operation, in microseconds
pub(crate) api_latency_by_api_micros: Vec<(String, u64)>,
}
/// Aggregate drive count statistics.
#[derive(Debug, Clone, Default)]
pub struct DriveCountStats {
@@ -93,6 +111,58 @@ pub struct DriveCountStats {
///
/// Returns a vector of Prometheus metrics for each drive.
pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<PrometheusMetric> {
let runtime_stats = stats
.iter()
.cloned()
.map(|stats| DriveRuntimeDetailedStats {
stats,
..Default::default()
})
.collect::<Vec<_>>();
collect_drive_runtime_detailed_metrics(&runtime_stats)
}
pub(crate) fn collect_drive_runtime_detailed_metrics(stats: &[DriveRuntimeDetailedStats]) -> Vec<PrometheusMetric> {
const DRIVE_RUNTIME_STATES: [&str; 5] = ["online", "offline", "returning", "suspect", "unknown"];
fn topology_labels(stat: &DriveRuntimeDetailedStats) -> Option<[Cow<'static, str>; 5]> {
Some([
Cow::Owned(stat.stats.server.clone()),
Cow::Owned(stat.stats.drive.clone()),
Cow::Owned(stat.pool_index.as_ref()?.clone()),
Cow::Owned(stat.set_index.as_ref()?.clone()),
Cow::Owned(stat.drive_index.as_ref()?.clone()),
])
}
fn has_topology_labels(stat: &DriveRuntimeDetailedStats) -> bool {
stat.pool_index.is_some() && stat.set_index.is_some() && stat.drive_index.is_some()
}
fn normalized_runtime_state(runtime_state: &str) -> &str {
DRIVE_RUNTIME_STATES
.iter()
.copied()
.find(|state| state.eq_ignore_ascii_case(runtime_state))
.unwrap_or("unknown")
}
fn push_topology_metric(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
value: f64,
labels: &[Cow<'static, str>; 5],
) {
metrics.push(
PrometheusMetric::from_descriptor(descriptor, value)
.with_label(SERVER_LABEL, labels[0].clone())
.with_label(DRIVE_LABEL, labels[1].clone())
.with_label(POOL_INDEX_LABEL, labels[2].clone())
.with_label(SET_INDEX_LABEL, labels[3].clone())
.with_label(DRIVE_INDEX_LABEL, labels[4].clone()),
);
}
fn push_drive_metric(
metrics: &mut Vec<PrometheusMetric>,
descriptor: &'static crate::metrics::schema::MetricDescriptor,
@@ -107,19 +177,49 @@ pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<Prome
);
}
let mut metrics = Vec::with_capacity(stats.len() * 23);
let metric_capacity = stats
.iter()
.map(|stat| {
let api_metrics = if has_topology_labels(stat) {
stat.api_calls.len() + stat.api_latency_by_api_micros.len()
} else {
0
};
31 + api_metrics
})
.sum();
let mut metrics = Vec::with_capacity(metric_capacity);
for stat in stats {
let server_label = stat.server.as_str();
let drive_label = stat.drive.as_str();
let server_label = stat.stats.server.as_str();
let drive_label = stat.stats.drive.as_str();
let topology_labels = topology_labels(stat);
push_drive_metric(&mut metrics, &DRIVE_TOTAL_BYTES_MD, stat.total_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_USED_BYTES_MD, stat.used_bytes as f64, server_label, drive_label);
push_drive_metric(&mut metrics, &DRIVE_FREE_BYTES_MD, stat.free_bytes as f64, server_label, drive_label);
push_drive_metric(
&mut metrics,
&DRIVE_TOTAL_BYTES_MD,
stat.stats.total_bytes as f64,
server_label,
drive_label,
);
push_drive_metric(
&mut metrics,
&DRIVE_USED_BYTES_MD,
stat.stats.used_bytes as f64,
server_label,
drive_label,
);
push_drive_metric(
&mut metrics,
&DRIVE_FREE_BYTES_MD,
stat.stats.free_bytes as f64,
server_label,
drive_label,
);
push_drive_metric(
&mut metrics,
&DRIVE_CAPACITY_OBSERVATION_AGE_SECONDS_MD,
stat.capacity_observation_age_seconds as f64,
stat.stats.capacity_observation_age_seconds as f64,
server_label,
drive_label,
);
@@ -127,59 +227,123 @@ pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<Prome
metrics.push(
PrometheusMetric::from_descriptor(
&DRIVE_CAPACITY_OBSERVATION_STATE_MD,
if state == stat.capacity_observation_state { 1.0 } else { 0.0 },
if state == stat.stats.capacity_observation_state {
1.0
} else {
0.0
},
)
.with_label_owned(SERVER_LABEL, server_label.to_string())
.with_label_owned(DRIVE_LABEL, drive_label.to_string())
.with_label_owned("state", state.to_string()),
);
}
if let Some(value) = stat.used_inodes {
if let Some(value) = stat.stats.used_inodes {
push_drive_metric(&mut metrics, &DRIVE_USED_INODES_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.free_inodes {
if let Some(value) = stat.stats.free_inodes {
push_drive_metric(&mut metrics, &DRIVE_FREE_INODES_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.total_inodes {
if let Some(value) = stat.stats.total_inodes {
push_drive_metric(&mut metrics, &DRIVE_TOTAL_INODES_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.timeout_errors_total {
if let Some(value) = stat.stats.timeout_errors_total {
push_drive_metric(&mut metrics, &DRIVE_TIMEOUT_ERRORS_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.io_errors_total {
if let Some(value) = stat.stats.io_errors_total {
push_drive_metric(&mut metrics, &DRIVE_IO_ERRORS_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.availability_errors_total {
if let Some(value) = stat.stats.availability_errors_total {
push_drive_metric(&mut metrics, &DRIVE_AVAILABILITY_ERRORS_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.waiting_io {
if let Some(value) = stat.stats.waiting_io {
push_drive_metric(&mut metrics, &DRIVE_WAITING_IO_MD, value as f64, server_label, drive_label);
}
if let Some(value) = stat.api_latency_micros {
if let Some(value) = stat.stats.api_latency_micros {
push_drive_metric(&mut metrics, &DRIVE_API_LATENCY_MD, value as f64, server_label, drive_label);
}
push_drive_metric(&mut metrics, &DRIVE_HEALTH_MD, stat.health as f64, server_label, drive_label);
if let Some(value) = stat.reads_per_sec {
push_drive_metric(&mut metrics, &DRIVE_HEALTH_MD, stat.stats.health as f64, server_label, drive_label);
if let Some(value) = stat.stats.reads_per_sec {
push_drive_metric(&mut metrics, &DRIVE_READS_PER_SEC_MD, value, server_label, drive_label);
}
if let Some(value) = stat.reads_kb_per_sec {
if let Some(value) = stat.stats.reads_kb_per_sec {
push_drive_metric(&mut metrics, &DRIVE_READS_KB_PER_SEC_MD, value, server_label, drive_label);
}
if let Some(value) = stat.reads_await {
if let Some(value) = stat.stats.reads_await {
push_drive_metric(&mut metrics, &DRIVE_READS_AWAIT_MD, value, server_label, drive_label);
}
if let Some(value) = stat.writes_per_sec {
if let Some(value) = stat.stats.writes_per_sec {
push_drive_metric(&mut metrics, &DRIVE_WRITES_PER_SEC_MD, value, server_label, drive_label);
}
if let Some(value) = stat.writes_kb_per_sec {
if let Some(value) = stat.stats.writes_kb_per_sec {
push_drive_metric(&mut metrics, &DRIVE_WRITES_KB_PER_SEC_MD, value, server_label, drive_label);
}
if let Some(value) = stat.writes_await {
if let Some(value) = stat.stats.writes_await {
push_drive_metric(&mut metrics, &DRIVE_WRITES_AWAIT_MD, value, server_label, drive_label);
}
if let Some(value) = stat.perc_util {
if let Some(value) = stat.stats.perc_util {
push_drive_metric(&mut metrics, &DRIVE_PERC_UTIL_MD, value, server_label, drive_label);
}
if let Some(labels) = &topology_labels {
if let Some(disk_id) = stat.disk_id.as_ref().filter(|disk_id| !disk_id.is_empty()) {
metrics.push(
PrometheusMetric::from_descriptor(&DRIVE_INFO_MD, 1.0)
.with_label(SERVER_LABEL, labels[0].clone())
.with_label(DRIVE_LABEL, labels[1].clone())
.with_label(POOL_INDEX_LABEL, labels[2].clone())
.with_label(SET_INDEX_LABEL, labels[3].clone())
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
.with_label_owned(DISK_ID_LABEL, disk_id.clone()),
);
}
if let Some(runtime_state) = stat.runtime_state.as_ref().filter(|state| !state.is_empty()) {
let runtime_state = normalized_runtime_state(runtime_state);
for state in DRIVE_RUNTIME_STATES {
metrics.push(
PrometheusMetric::from_descriptor(
&DRIVE_RUNTIME_STATE_MD,
if state == runtime_state { 1.0 } else { 0.0 },
)
.with_label(SERVER_LABEL, labels[0].clone())
.with_label(DRIVE_LABEL, labels[1].clone())
.with_label(POOL_INDEX_LABEL, labels[2].clone())
.with_label(SET_INDEX_LABEL, labels[3].clone())
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
.with_label(STATE_LABEL, state),
);
}
}
push_topology_metric(&mut metrics, &DRIVE_HEALING_MD, if stat.healing { 1.0 } else { 0.0 }, labels);
push_topology_metric(&mut metrics, &DRIVE_SCANNING_MD, if stat.scanning { 1.0 } else { 0.0 }, labels);
push_topology_metric(
&mut metrics,
&DRIVE_OFFLINE_DURATION_SECONDS_MD,
stat.offline_duration_seconds.unwrap_or(0) as f64,
labels,
);
for (api, value) in &stat.api_calls {
metrics.push(
PrometheusMetric::from_descriptor(&DRIVE_API_CALLS_MD, *value as f64)
.with_label(SERVER_LABEL, labels[0].clone())
.with_label(DRIVE_LABEL, labels[1].clone())
.with_label(POOL_INDEX_LABEL, labels[2].clone())
.with_label(SET_INDEX_LABEL, labels[3].clone())
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
.with_label_owned(API_LABEL, api.clone()),
);
}
for (api, value) in &stat.api_latency_by_api_micros {
metrics.push(
PrometheusMetric::from_descriptor(&DRIVE_API_LATENCY_BY_API_MD, *value as f64)
.with_label(SERVER_LABEL, labels[0].clone())
.with_label(DRIVE_LABEL, labels[1].clone())
.with_label(POOL_INDEX_LABEL, labels[2].clone())
.with_label(SET_INDEX_LABEL, labels[3].clone())
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
.with_label_owned(API_LABEL, api.clone()),
);
}
}
}
metrics
@@ -259,36 +423,48 @@ mod tests {
#[test]
fn test_collect_drive_detailed_metrics() {
let stats = vec![DriveDetailedStats {
server: "node1:9000".to_string(),
drive: "/data/disk1".to_string(),
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
used_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
free_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
capacity_observation_state: "live",
capacity_observation_age_seconds: 0,
used_inodes: Some(100000),
free_inodes: Some(900000),
total_inodes: Some(1000000),
timeout_errors_total: Some(5),
io_errors_total: Some(10),
availability_errors_total: Some(2),
waiting_io: Some(3),
api_latency_micros: Some(1500),
health: 1,
reads_per_sec: Some(100.0),
reads_kb_per_sec: Some(1024.0),
reads_await: Some(5.5),
writes_per_sec: Some(50.0),
writes_kb_per_sec: Some(512.0),
writes_await: Some(10.2),
perc_util: Some(75.5),
let stats = vec![DriveRuntimeDetailedStats {
pool_index: Some("0".to_string()),
set_index: Some("1".to_string()),
drive_index: Some("2".to_string()),
disk_id: Some("disk-uuid-1".to_string()),
runtime_state: Some("online".to_string()),
healing: true,
scanning: false,
offline_duration_seconds: Some(0),
api_calls: vec![("read".to_string(), 7)],
api_latency_by_api_micros: vec![("read".to_string(), 2500)],
stats: DriveDetailedStats {
server: "node1:9000".to_string(),
drive: "/data/disk1".to_string(),
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
used_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
free_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
capacity_observation_state: "live",
capacity_observation_age_seconds: 0,
used_inodes: Some(100000),
free_inodes: Some(900000),
total_inodes: Some(1000000),
timeout_errors_total: Some(5),
io_errors_total: Some(10),
availability_errors_total: Some(2),
waiting_io: Some(3),
api_latency_micros: Some(1500),
health: 1,
reads_per_sec: Some(100.0),
reads_kb_per_sec: Some(1024.0),
reads_await: Some(5.5),
writes_per_sec: Some(50.0),
writes_kb_per_sec: Some(512.0),
writes_await: Some(10.2),
perc_util: Some(75.5),
},
}];
let metrics = collect_drive_detailed_metrics(&stats);
let metrics = collect_drive_runtime_detailed_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 23);
assert_eq!(metrics.len(), 34);
// Verify total bytes metric
let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name();
@@ -303,6 +479,32 @@ mod tests {
);
assert_metric_label_keys(&metrics, &DRIVE_API_LATENCY_MD, 1500.0, &[SERVER_LABEL, DRIVE_LABEL]);
assert_metric_label_keys(&metrics, &DRIVE_CAPACITY_OBSERVATION_STATE_MD, 1.0, &[SERVER_LABEL, DRIVE_LABEL, "state"]);
assert_metric_label_keys(
&metrics,
&DRIVE_INFO_MD,
1.0,
&[
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
DISK_ID_LABEL,
],
);
assert_metric_label_keys(
&metrics,
&DRIVE_API_CALLS_MD,
7.0,
&[
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
API_LABEL,
],
);
}
#[test]
@@ -353,6 +555,75 @@ mod tests {
);
}
#[test]
fn drive_runtime_state_metrics_keep_suspect_state_active() {
let stats = vec![DriveRuntimeDetailedStats {
pool_index: Some("0".to_string()),
set_index: Some("1".to_string()),
drive_index: Some("2".to_string()),
runtime_state: Some("suspect".to_string()),
stats: DriveDetailedStats {
server: "node1:9000".to_string(),
drive: "/data/disk1".to_string(),
..Default::default()
},
..Default::default()
}];
let metrics = collect_drive_runtime_detailed_metrics(&stats);
let state_metrics = metrics
.iter()
.filter(|metric| metric.name == DRIVE_RUNTIME_STATE_MD.get_full_metric_name())
.collect::<Vec<_>>();
assert_eq!(state_metrics.len(), 5);
assert!(state_metrics.iter().any(|metric| {
metric.value == 1.0
&& metric
.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "suspect")
}));
assert!(state_metrics.iter().filter(|metric| metric.value == 1.0).all(|metric| {
metric
.labels
.iter()
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "suspect")
}));
}
#[test]
fn drive_offline_duration_zeroes_recovered_topology_drive() {
let stats = vec![DriveRuntimeDetailedStats {
pool_index: Some("0".to_string()),
set_index: Some("1".to_string()),
drive_index: Some("2".to_string()),
runtime_state: Some("online".to_string()),
offline_duration_seconds: None,
stats: DriveDetailedStats {
server: "node1:9000".to_string(),
drive: "/data/disk1".to_string(),
..Default::default()
},
..Default::default()
}];
let metrics = collect_drive_runtime_detailed_metrics(&stats);
assert!(metrics.iter().any(|metric| {
metric.name == DRIVE_OFFLINE_DURATION_SECONDS_MD.get_full_metric_name()
&& metric.value == 0.0
&& metric
.labels
.iter()
.any(|(name, value)| *name == SERVER_LABEL && value == "node1:9000")
&& metric
.labels
.iter()
.any(|(name, value)| *name == DRIVE_INDEX_LABEL && value == "2")
}));
}
#[test]
fn test_collect_drive_count_metrics() {
let stats = DriveCountStats {
crates/obs/src/metrics/scheduler.rs
+973 -62
View File
File diff suppressed because it is too large Load Diff
crates/obs/src/metrics/schema/audit.rs
+45 -5
View File
@@ -17,18 +17,31 @@
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
use std::sync::LazyLock;
const TARGET_ID: &str = "target_id";
pub const TARGET_ID: &str = "target_id";
pub const SERVER: &str = "server";
pub const RESULT: &str = "result"; // success / failure
pub const STATUS: &str = "status"; // success / failure
pub const SUCCESS: &str = "success";
pub const FAILURE: &str = "failure";
const TARGET_LABELS: [&str; 1] = [TARGET_ID];
const TARGET_SERVER_LABELS: [&str; 2] = [SERVER, TARGET_ID];
pub static AUDIT_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::AuditFailedMessages,
"Total number of messages that failed to send since start",
&[TARGET_ID],
&TARGET_LABELS,
subsystems::AUDIT,
)
});
pub static AUDIT_FAILED_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("failed_messages_by_server".to_string()),
"Total number of messages that failed to send since start by server and target",
&TARGET_SERVER_LABELS,
subsystems::AUDIT,
)
});
@@ -37,7 +50,16 @@ pub static AUDIT_FAILED_STORE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::
new_gauge_md(
MetricName::AuditFailedStoreLength,
"Number of audit messages held in the failed-events store for target",
&[TARGET_ID],
&TARGET_LABELS,
subsystems::AUDIT,
)
});
pub static AUDIT_FAILED_STORE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("failed_store_length_by_server".to_string()),
"Number of audit messages held in the failed-events store by server and target",
&TARGET_SERVER_LABELS,
subsystems::AUDIT,
)
});
@@ -46,7 +68,16 @@ pub static AUDIT_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::
new_gauge_md(
MetricName::AuditTargetQueueLength,
"Number of unsent messages in queue for target",
&[TARGET_ID],
&TARGET_LABELS,
subsystems::AUDIT,
)
});
pub static AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("target_queue_length_by_server".to_string()),
"Number of unsent audit messages in queue by server and target",
&TARGET_SERVER_LABELS,
subsystems::AUDIT,
)
});
@@ -55,7 +86,16 @@ pub static AUDIT_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
new_gauge_md(
MetricName::AuditTotalMessages,
"Total number of messages sent since start",
&[TARGET_ID],
&TARGET_LABELS,
subsystems::AUDIT,
)
});
pub static AUDIT_TOTAL_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("total_messages_by_server".to_string()),
"Total number of messages sent since start by server and target",
&TARGET_SERVER_LABELS,
subsystems::AUDIT,
)
});
crates/obs/src/metrics/schema/bucket_replication.rs
+93
View File
@@ -21,6 +21,8 @@ use std::sync::LazyLock;
pub const BUCKET_L: &str = "bucket";
/// Replication operation
pub const OPERATION_L: &str = "operation";
/// Replication proxy result
pub const RESULT_L: &str = "result";
/// Replication target ARN
pub const TARGET_ARN_L: &str = "target_arn";
/// Replication range
@@ -48,6 +50,16 @@ const MRF_DROPPED_COUNT: &str = "mrf_dropped_count";
const MRF_MISSED_COUNT: &str = "mrf_missed_count";
const MRF_FLUSH_FAILURES: &str = "mrf_flush_failures";
const MRF_LAST_FLUSH_DURATION_MILLIS: &str = "mrf_last_flush_duration_millis";
const TARGET_SENT_BYTES: &str = "target_sent_bytes";
const TARGET_SENT_COUNT: &str = "target_sent_count";
const TARGET_TOTAL_FAILED_BYTES: &str = "target_total_failed_bytes";
const TARGET_TOTAL_FAILED_COUNT: &str = "target_total_failed_count";
const TARGET_LAST_MIN_FAILED_BYTES: &str = "target_last_min_failed_bytes";
const TARGET_LAST_MIN_FAILED_COUNT: &str = "target_last_min_failed_count";
const TARGET_LAST_HOUR_FAILED_BYTES: &str = "target_last_hour_failed_bytes";
const TARGET_LAST_HOUR_FAILED_COUNT: &str = "target_last_hour_failed_count";
const BUCKET_TARGET_LABELS: [&str; 2] = [BUCKET_L, TARGET_ARN_L];
pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
@@ -58,6 +70,15 @@ pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = Laz
)
});
pub static BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(TARGET_LAST_HOUR_FAILED_BYTES),
"Total number of bytes failed at least once to replicate in the last hour on a bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::LastHourFailedCount,
@@ -67,6 +88,15 @@ pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = Laz
)
});
pub static BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(TARGET_LAST_HOUR_FAILED_COUNT),
"Total number of objects which failed replication in the last hour on a bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::LastMinFailedBytes,
@@ -76,6 +106,15 @@ pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = La
)
});
pub static BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(TARGET_LAST_MIN_FAILED_BYTES),
"Total number of bytes failed at least once to replicate in the last full minute on a bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::LastMinFailedCount,
@@ -85,6 +124,15 @@ pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = La
)
});
pub static BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(TARGET_LAST_MIN_FAILED_COUNT),
"Total number of objects which failed replication in the last full minute on a bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::LatencyMilliSec,
@@ -337,6 +385,15 @@ pub static BUCKET_REPL_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
)
});
pub static BUCKET_REPL_TARGET_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(TARGET_SENT_BYTES),
"Total number of bytes replicated to a bucket replication target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::SentCount,
@@ -346,6 +403,15 @@ pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new
)
});
pub static BUCKET_REPL_TARGET_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(TARGET_SENT_COUNT),
"Total number of objects replicated to a bucket replication target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_RESYNC_STARTED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(RESYNC_STARTED_TOTAL),
@@ -400,6 +466,15 @@ pub static BUCKET_REPL_TOTAL_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyL
)
});
pub static BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(TARGET_TOTAL_FAILED_BYTES),
"Total number of bytes failed at least once to replicate since server start by bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::TotalFailedCount,
@@ -409,6 +484,15 @@ pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyL
)
});
pub static BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(TARGET_TOTAL_FAILED_COUNT),
"Total number of objects which failed replication since server start by bucket and target ARN",
&BUCKET_TARGET_LABELS,
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_BANDWIDTH_LIMIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::BandwidthLimitBytesPerSecond,
@@ -435,3 +519,12 @@ pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD: LazyLock<Met
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("proxy_requests_total".to_string()),
"Total number of bucket replication proxy requests by operation and result",
&[BUCKET_L, OPERATION_L, RESULT_L],
subsystems::BUCKET_REPLICATION,
)
});
crates/obs/src/metrics/schema/ilm.rs
+13
View File
@@ -17,6 +17,19 @@
use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
use std::sync::LazyLock;
pub const SERVER_LABEL: &str = "server";
pub const ACTION_LABEL: &str = "action";
pub const STATE_LABEL: &str = "state";
pub static ILM_ACTION_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("action_tasks".to_string()),
"ILM task counts by server, action, and state",
&[SERVER_LABEL, ACTION_LABEL, STATE_LABEL],
subsystems::ILM,
)
});
pub static ILM_EXPIRY_PENDING_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::IlmExpiryPendingTasks,
crates/obs/src/metrics/schema/notification_target.rs
+38
View File
@@ -19,8 +19,10 @@ use std::sync::LazyLock;
pub const TARGET_ID: &str = "target_id";
pub const TARGET_TYPE: &str = "target_type";
pub const SERVER: &str = "server";
const NOTIFICATION_TARGET_LABELS: [&str; 2] = [TARGET_ID, TARGET_TYPE];
const NOTIFICATION_TARGET_SERVER_LABELS: [&str; 3] = [SERVER, TARGET_ID, TARGET_TYPE];
pub static NOTIFICATION_TARGET_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
@@ -31,6 +33,15 @@ pub static NOTIFICATION_TARGET_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> =
)
});
pub static NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("target_failed_messages_by_server".to_string()),
"Total number of notification messages that permanently failed to send by server and target",
&NOTIFICATION_TARGET_SERVER_LABELS,
subsystems::NOTIFICATION,
)
});
pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::NotificationTargetFailedStoreLength,
@@ -40,6 +51,15 @@ pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD: LazyLock<MetricDescriptor
)
});
pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("target_failed_store_length_by_server".to_string()),
"Number of notification messages held in the failed-events store by server and target",
&NOTIFICATION_TARGET_SERVER_LABELS,
subsystems::NOTIFICATION,
)
});
pub static NOTIFICATION_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::NotificationTargetQueueLength,
@@ -49,6 +69,15 @@ pub static NOTIFICATION_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = Laz
)
});
pub static NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("target_queue_length_by_server".to_string()),
"Number of queued notification messages pending delivery by server and target",
&NOTIFICATION_TARGET_SERVER_LABELS,
subsystems::NOTIFICATION,
)
});
pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::NotificationTargetTotalMessages,
@@ -57,3 +86,12 @@ pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = L
subsystems::NOTIFICATION,
)
});
pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("target_total_messages_by_server".to_string()),
"Total number of notification messages successfully delivered by server and target",
&NOTIFICATION_TARGET_SERVER_LABELS,
subsystems::NOTIFICATION,
)
});
crates/obs/src/metrics/schema/replication.rs
+119
View File
@@ -17,6 +17,8 @@
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
use std::sync::LazyLock;
pub const SERVER_LABEL: &str = "server";
pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationAverageActiveWorkers,
@@ -26,6 +28,15 @@ pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = L
)
});
pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("average_active_workers_by_server".to_string()),
"Average number of active replication workers by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_AVERAGE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationAverageQueuedBytes,
@@ -35,6 +46,15 @@ pub static REPLICATION_AVERAGE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = Laz
)
});
pub static REPLICATION_AVERAGE_QUEUED_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("average_queued_bytes_by_server".to_string()),
"Average number of bytes queued for replication since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_AVERAGE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationAverageQueuedCount,
@@ -44,6 +64,15 @@ pub static REPLICATION_AVERAGE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = Laz
)
});
pub static REPLICATION_AVERAGE_QUEUED_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("average_queued_count_by_server".to_string()),
"Average number of objects queued for replication since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationAverageDataTransferRate,
@@ -53,6 +82,15 @@ pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor>
)
});
pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("average_data_transfer_rate_by_server".to_string()),
"Average replication data transfer rate in bytes/sec by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_CURRENT_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationCurrentActiveWorkers,
@@ -62,6 +100,15 @@ pub static REPLICATION_CURRENT_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = L
)
});
pub static REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("current_active_workers_by_server".to_string()),
"Total number of active replication workers by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationCurrentDataTransferRate,
@@ -71,6 +118,15 @@ pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor>
)
});
pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("current_data_transfer_rate_by_server".to_string()),
"Current replication data transfer rate in bytes/sec by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationLastMinuteQueuedBytes,
@@ -80,6 +136,15 @@ pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> =
)
});
pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("last_minute_queued_bytes_by_server".to_string()),
"Number of bytes queued for replication in the last full minute by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationLastMinuteQueuedCount,
@@ -89,6 +154,15 @@ pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> =
)
});
pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("last_minute_queued_count_by_server".to_string()),
"Number of objects queued for replication in the last full minute by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_MAX_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationMaxActiveWorkers,
@@ -98,6 +172,15 @@ pub static REPLICATION_MAX_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyL
)
});
pub static REPLICATION_MAX_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("max_active_workers_by_server".to_string()),
"Maximum number of active replication workers seen since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_MAX_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationMaxQueuedBytes,
@@ -107,6 +190,15 @@ pub static REPLICATION_MAX_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLoc
)
});
pub static REPLICATION_MAX_QUEUED_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("max_queued_bytes_by_server".to_string()),
"Maximum number of bytes queued for replication since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_MAX_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationMaxQueuedCount,
@@ -116,6 +208,15 @@ pub static REPLICATION_MAX_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLoc
)
});
pub static REPLICATION_MAX_QUEUED_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("max_queued_count_by_server".to_string()),
"Maximum number of objects queued for replication since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationMaxDataTransferRate,
@@ -125,6 +226,15 @@ pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = L
)
});
pub static REPLICATION_MAX_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("max_data_transfer_rate_by_server".to_string()),
"Maximum replication data transfer rate in bytes/sec seen since server start by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationRecentBacklogCount,
@@ -133,3 +243,12 @@ pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = Laz
subsystems::REPLICATION,
)
});
pub static REPLICATION_RECENT_BACKLOG_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("recent_backlog_count_by_server".to_string()),
"Objects currently in replication backlog by server",
&[SERVER_LABEL],
subsystems::REPLICATION,
)
});
crates/obs/src/metrics/schema/request.rs
+99 -9
View File
@@ -22,6 +22,15 @@ pub const NAME_LABEL: &str = "name";
pub const TYPE_LABEL: &str = "type";
/// le label (for histogram buckets)
pub const LE_LABEL: &str = "le";
/// server label
pub const SERVER_LABEL: &str = "server";
const API_NAME_TYPE_LABELS: [&str; 2] = [NAME_LABEL, TYPE_LABEL];
const API_SERVER_NAME_TYPE_LABELS: [&str; 3] = [SERVER_LABEL, NAME_LABEL, TYPE_LABEL];
const API_NAME_TYPE_LE_LABELS: [&str; 3] = [NAME_LABEL, TYPE_LABEL, LE_LABEL];
const API_SERVER_NAME_TYPE_LE_LABELS: [&str; 4] = [SERVER_LABEL, NAME_LABEL, TYPE_LABEL, LE_LABEL];
const API_TYPE_LABELS: [&str; 1] = [TYPE_LABEL];
const API_SERVER_TYPE_LABELS: [&str; 2] = [SERVER_LABEL, TYPE_LABEL];
pub static API_REJECTED_AUTH_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
@@ -81,7 +90,16 @@ pub static API_REQUESTS_IN_FLIGHT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLoc
new_gauge_md(
MetricName::ApiRequestsInFlightTotal,
"Total number of requests currently in flight",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_IN_FLIGHT_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("requests_in_flight_total_by_server".to_string()),
"Total number of requests currently in flight by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -90,7 +108,16 @@ pub static API_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(||
new_counter_md(
MetricName::ApiRequestsTotal,
"Total number of requests",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_total_by_server".to_string()),
"Total number of requests by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -99,7 +126,16 @@ pub static API_REQUESTS_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::
new_counter_md(
MetricName::ApiRequestsErrorsTotal,
"Total number of requests with (4xx and 5xx) errors",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_ERRORS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_errors_total_by_server".to_string()),
"Total number of requests with (4xx and 5xx) errors by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -108,7 +144,16 @@ pub static API_REQUESTS_5XX_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLo
new_counter_md(
MetricName::ApiRequests5xxErrorsTotal,
"Total number of requests with 5xx errors",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_5XX_ERRORS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_5xx_errors_total_by_server".to_string()),
"Total number of requests with 5xx errors by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -117,7 +162,16 @@ pub static API_REQUESTS_4XX_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLo
new_counter_md(
MetricName::ApiRequests4xxErrorsTotal,
"Total number of requests with 4xx errors",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_4XX_ERRORS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_4xx_errors_total_by_server".to_string()),
"Total number of requests with 4xx errors by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -126,7 +180,16 @@ pub static API_REQUESTS_CANCELED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock
new_counter_md(
MetricName::ApiRequestsCanceledTotal,
"Total number of requests canceled by the client",
&["name", "type"],
&API_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_CANCELED_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_canceled_total_by_server".to_string()),
"Total number of requests canceled by the client by server",
&API_SERVER_NAME_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -135,7 +198,16 @@ pub static API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD: LazyLock<MetricDescriptor>
new_counter_md(
MetricName::ApiRequestsTTFBSecondsDistribution,
"Distribution of time to first byte across API calls",
&["name", "type", "le"],
&API_NAME_TYPE_LE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("requests_ttfb_seconds_distribution_by_server".to_string()),
"Distribution of time to first byte across API calls by server",
&API_SERVER_NAME_TYPE_LE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -144,7 +216,16 @@ pub static API_TRAFFIC_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
new_counter_md(
MetricName::ApiTrafficSentBytes,
"Total number of bytes sent",
&["type"],
&API_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_TRAFFIC_SENT_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("traffic_sent_bytes_by_server".to_string()),
"Total number of bytes sent by server",
&API_SERVER_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
@@ -153,7 +234,16 @@ pub static API_TRAFFIC_RECV_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
new_counter_md(
MetricName::ApiTrafficRecvBytes,
"Total number of bytes received",
&["type"],
&API_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
pub static API_TRAFFIC_RECV_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("traffic_recv_bytes_by_server".to_string()),
"Total number of bytes received by server",
&API_SERVER_TYPE_LABELS,
MetricSubsystem::ApiRequests,
)
});
crates/obs/src/metrics/schema/scanner.rs
+44
View File
@@ -17,6 +17,50 @@
use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
use std::sync::LazyLock;
pub const SERVER_LABEL: &str = "server";
pub const SOURCE_LABEL: &str = "source";
pub const STATE_LABEL: &str = "state";
pub const CYCLE_SCOPE_LABEL: &str = "cycle_scope";
pub const BUCKET_LABEL: &str = "bucket";
pub const DRIVE_LABEL: &str = "drive";
pub const RESULT_LABEL: &str = "result";
pub static SCANNER_SOURCE_WORK_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("source_work_total".to_string()),
"Total scanner work by source and state since server start",
&[SERVER_LABEL, SOURCE_LABEL, STATE_LABEL],
subsystems::SCANNER,
)
});
pub static SCANNER_CYCLE_SOURCE_WORK_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("cycle_source_work".to_string()),
"Scanner work by cycle scope, source, and state",
&[SERVER_LABEL, CYCLE_SCOPE_LABEL, SOURCE_LABEL, STATE_LABEL],
subsystems::SCANNER,
)
});
pub static SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("bucket_drive_result_total".to_string()),
"Total scanner bucket-drive scan results by server, bucket, drive, and result",
&[SERVER_LABEL, BUCKET_LABEL, DRIVE_LABEL, RESULT_LABEL],
subsystems::SCANNER,
)
});
pub static SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("cycle_bucket_drive_result".to_string()),
"Scanner bucket-drive scan results by cycle scope, server, bucket, drive, and result",
&[SERVER_LABEL, CYCLE_SCOPE_LABEL, BUCKET_LABEL, DRIVE_LABEL, RESULT_LABEL],
subsystems::SCANNER,
)
});
pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::ScannerBucketScansFinished,
crates/obs/src/metrics/schema/system_drive.rs
+102
View File
@@ -29,9 +29,111 @@ pub const SET_INDEX_LABEL: &str = "set_index";
pub const DRIVE_INDEX_LABEL: &str = "drive_index";
/// API label
pub const API_LABEL: &str = "api";
/// Disk id label
pub const DISK_ID_LABEL: &str = "disk_id";
/// State label
pub const STATE_LABEL: &str = "state";
/// All drive-related labels
pub const ALL_DRIVE_LABELS: [&str; 2] = [SERVER_LABEL, DRIVE_LABEL];
/// Drive labels with erasure-set topology.
pub const DRIVE_TOPOLOGY_LABELS: [&str; 5] = [
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
];
/// Drive info labels.
pub const DRIVE_INFO_LABELS: [&str; 6] = [
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
DISK_ID_LABEL,
];
/// Drive topology labels with a state dimension.
pub const DRIVE_TOPOLOGY_STATE_LABELS: [&str; 6] = [
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
STATE_LABEL,
];
/// Drive topology labels with an API dimension.
pub const DRIVE_TOPOLOGY_API_LABELS: [&str; 6] = [
SERVER_LABEL,
DRIVE_LABEL,
POOL_INDEX_LABEL,
SET_INDEX_LABEL,
DRIVE_INDEX_LABEL,
API_LABEL,
];
pub static DRIVE_INFO_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("info".to_string()),
"Drive topology and stable disk identity information",
&DRIVE_INFO_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_RUNTIME_STATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("runtime_state".to_string()),
"Drive runtime state (1 for the active state label, 0 otherwise)",
&DRIVE_TOPOLOGY_STATE_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_HEALING_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("healing".to_string()),
"Whether the drive is currently healing",
&DRIVE_TOPOLOGY_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_SCANNING_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("scanning".to_string()),
"Whether the drive is currently being scanned",
&DRIVE_TOPOLOGY_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_OFFLINE_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("offline_duration_seconds".to_string()),
"Duration in seconds the drive has been offline",
&DRIVE_TOPOLOGY_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_API_CALLS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::Custom("api_calls_total".to_string()),
"Total drive API calls by operation",
&DRIVE_TOPOLOGY_API_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_API_LATENCY_BY_API_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::Custom("api_latency_by_api_micros".to_string()),
"Average last minute drive API latency in microseconds by operation",
&DRIVE_TOPOLOGY_API_LABELS,
subsystems::SYSTEM_DRIVE,
)
});
pub static DRIVE_USED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
crates/obs/src/metrics/stats_collector.rs
+608 -166
View File
@@ -20,12 +20,15 @@
//! RustFS internal sources (storage layer, bucket monitor, system info)
//! and convert them to the Stats structs used by collectors.
use crate::metrics::collectors::scanner::{ScannerBucketDriveResultStats, ScannerSourceWorkStats};
use crate::metrics::collectors::{
BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetBacklogStats,
ApiRequestMetricSupport, ApiRequestStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats,
BucketReplicationRuntimeStats, BucketReplicationStats, BucketReplicationTargetBacklogStats, BucketReplicationTargetFlowStats,
BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats,
ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats,
HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats,
ResourceStats, ScannerStats,
ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats,
DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmRuntimeStats, IlmStats,
MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats, ScannerRuntimeStats,
ScannerStats,
};
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
use crate::metrics::{
@@ -37,11 +40,11 @@ use crate::metrics::{
use crate::node_identity::current_local_node_identity;
use chrono::Utc;
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{ScannerMetricsReport, global_metrics};
use rustfs_common::metrics::{ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot, global_metrics};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_io_metrics::{
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_resource_and_system,
snapshot_process_resource_and_system_with,
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, s3_op_metrics_snapshot,
snapshot_process_resource_and_system, snapshot_process_resource_and_system_with,
};
use std::{
collections::{HashMap, HashSet},
@@ -193,7 +196,7 @@ async fn obs_ilm_runtime_snapshot() -> ObsIlmRuntimeSnapshot {
ilm_runtime_snapshot().await
}
async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>) {
async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationRuntimeStats>, Vec<BucketReplicationBacklogStats>) {
let snapshots = obs_bucket_replication_stats_snapshot().await;
let mut detail_stats = Vec::with_capacity(snapshots.len());
let mut backlog_stats = Vec::with_capacity(snapshots.len());
@@ -230,44 +233,65 @@ async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>,
(detail_stats, backlog_stats)
}
fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnapshot) -> BucketReplicationStats {
BucketReplicationStats {
bucket: stats.bucket,
total_failed_bytes: stats.total_failed_bytes,
total_failed_count: stats.total_failed_count,
last_min_failed_bytes: stats.last_min_failed_bytes,
last_min_failed_count: stats.last_min_failed_count,
last_hour_failed_bytes: stats.last_hour_failed_bytes,
last_hour_failed_count: stats.last_hour_failed_count,
sent_bytes: stats.sent_bytes,
sent_count: stats.sent_count,
proxied_get_requests_total: stats.proxied_get_requests_total,
proxied_get_requests_failures: stats.proxied_get_requests_failures,
proxied_head_requests_total: stats.proxied_head_requests_total,
proxied_head_requests_failures: stats.proxied_head_requests_failures,
proxied_put_requests_total: stats.proxied_put_requests_total,
proxied_put_requests_failures: stats.proxied_put_requests_failures,
proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
resync_started_count: stats.resync_started_count,
resync_completed_count: stats.resync_completed_count,
resync_failed_count: stats.resync_failed_count,
resync_canceled_count: stats.resync_canceled_count,
resync_duration_ms: stats.resync_duration_ms,
targets: stats
.targets
.into_iter()
.map(|target| BucketReplicationTargetStats {
target_arn: target.target_arn,
bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
latency_ms: target.latency_ms,
})
.collect(),
fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnapshot) -> BucketReplicationRuntimeStats {
let bucket = stats.bucket;
let (targets, target_flows): (Vec<_>, Vec<_>) = stats
.targets
.into_iter()
.map(|target| {
(
BucketReplicationTargetStats {
target_arn: target.target_arn.clone(),
bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
latency_ms: target.latency_ms,
},
BucketReplicationTargetFlowStats {
target_arn: target.target_arn,
sent_bytes: target.sent_bytes,
sent_count: target.sent_count,
total_failed_bytes: target.total_failed_bytes,
total_failed_count: target.total_failed_count,
last_min_failed_bytes: target.last_min_failed_bytes,
last_min_failed_count: target.last_min_failed_count,
last_hour_failed_bytes: target.last_hour_failed_bytes,
last_hour_failed_count: target.last_hour_failed_count,
},
)
})
.unzip();
BucketReplicationRuntimeStats {
target_flows,
stats: BucketReplicationStats {
bucket,
total_failed_bytes: stats.total_failed_bytes,
total_failed_count: stats.total_failed_count,
last_min_failed_bytes: stats.last_min_failed_bytes,
last_min_failed_count: stats.last_min_failed_count,
last_hour_failed_bytes: stats.last_hour_failed_bytes,
last_hour_failed_count: stats.last_hour_failed_count,
sent_bytes: stats.sent_bytes,
sent_count: stats.sent_count,
proxied_get_requests_total: stats.proxied_get_requests_total,
proxied_get_requests_failures: stats.proxied_get_requests_failures,
proxied_head_requests_total: stats.proxied_head_requests_total,
proxied_head_requests_failures: stats.proxied_head_requests_failures,
proxied_put_requests_total: stats.proxied_put_requests_total,
proxied_put_requests_failures: stats.proxied_put_requests_failures,
proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
resync_started_count: stats.resync_started_count,
resync_completed_count: stats.resync_completed_count,
resync_failed_count: stats.resync_failed_count,
resync_canceled_count: stats.resync_canceled_count,
resync_duration_ms: stats.resync_duration_ms,
targets,
},
}
}
@@ -364,6 +388,63 @@ fn disk_capacity_observation_state(source: Option<&str>, age_seconds: Option<u64
}
}
fn disk_topology_label(index: i32) -> Option<String> {
if index >= 0 { Some(index.to_string()) } else { None }
}
fn non_empty_disk_id(uuid: &str) -> Option<String> {
let uuid = uuid.trim();
if uuid.is_empty() { None } else { Some(uuid.to_string()) }
}
fn drive_inode_stats(used_inodes: u64, free_inodes: u64) -> (Option<u64>, Option<u64>, Option<u64>) {
let total_inodes = used_inodes.saturating_add(free_inodes);
if total_inodes == 0 {
(None, None, None)
} else {
(Some(used_inodes), Some(free_inodes), Some(total_inodes))
}
}
fn drive_api_latency_micros(actions: impl Iterator<Item = (u64, u64)>) -> Option<u64> {
let mut count = 0u64;
let mut acc_time_ns = 0u64;
let mut saw_action = false;
for (action_count, action_acc_time_ns) in actions {
saw_action = true;
if action_count > 0 {
count = count.saturating_add(action_count);
acc_time_ns = acc_time_ns.saturating_add(action_acc_time_ns);
}
}
saw_action.then(|| acc_time_ns.checked_div(count).unwrap_or_default() / 1_000)
}
fn drive_api_latency_by_api_micros<'a>(actions: impl Iterator<Item = (&'a String, u64, u64)>) -> Vec<(String, u64)> {
let mut values = actions
.map(|(api, count, acc_time)| (api.clone(), acc_time.checked_div(count).unwrap_or_default() / 1_000))
.collect::<Vec<_>>();
values.sort_by(|left, right| left.0.cmp(&right.0));
values
}
fn drive_api_calls<'a>(api_calls: impl Iterator<Item = (&'a String, &'a u64)>) -> Vec<(String, u64)> {
let mut values = api_calls.map(|(api, calls)| (api.clone(), *calls)).collect::<Vec<_>>();
values.sort_by(|left, right| left.0.cmp(&right.0));
values
}
fn drive_server_label(endpoint: &str, local_server: &str) -> String {
endpoint
.strip_prefix("http://")
.or_else(|| endpoint.strip_prefix("https://"))
.and_then(|rest| rest.split('/').next())
.filter(|authority| !authority.is_empty())
.unwrap_or(local_server)
.to_string()
}
fn derive_erasure_set_quorum_shape(set_drive_count: usize, parity: usize) -> ErasureSetQuorumShape {
let data_shards = set_drive_count.saturating_sub(parity);
let read_quorum = data_shards.max(1);
@@ -563,12 +644,12 @@ pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationS
obs_bucket_replication_stats_snapshot()
.await
.into_iter()
.map(bucket_replication_detail_from_snapshot)
.map(|snapshot| bucket_replication_detail_from_snapshot(snapshot).stats)
.collect()
}
pub(crate) async fn collect_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>)
{
pub(crate) async fn collect_bucket_replication_stats_bundle()
-> (Vec<BucketReplicationRuntimeStats>, Vec<BucketReplicationBacklogStats>) {
obs_bucket_replication_stats_bundle().await
}
@@ -577,6 +658,22 @@ pub async fn collect_replication_stats() -> ReplicationStats {
obs_site_replication_stats().await
}
/// Collect S3 API request totals from the in-process operation recorder.
pub(crate) fn collect_api_request_stats() -> Vec<ApiRequestStats> {
let server = current_local_node_identity();
s3_op_metrics_snapshot()
.into_iter()
.map(|snapshot| ApiRequestStats {
server: server.clone(),
name: snapshot.op.to_string(),
req_type: "s3".to_string(),
total: snapshot.total,
supported_metrics: ApiRequestMetricSupport::TOTALS_ONLY,
..Default::default()
})
.collect()
}
/// Collect disk statistics from the storage layer.
pub async fn collect_disk_stats() -> Vec<DiskStats> {
let (disk_stats, _, _) = collect_disk_and_system_drive_stats().await;
@@ -646,16 +743,23 @@ pub fn collect_system_memory_stats() -> MemoryStats {
/// Collect node disk stats and drive stats from a single storage snapshot.
pub async fn collect_disk_and_system_drive_stats() -> (Vec<DiskStats>, Vec<DriveDetailedStats>, DriveCountStats) {
let (disk_stats, drive_stats, drive_count_stats) = collect_disk_and_system_drive_runtime_stats().await;
(disk_stats, drive_stats.into_iter().map(|stat| stat.stats).collect(), drive_count_stats)
}
pub(crate) async fn collect_disk_and_system_drive_runtime_stats()
-> (Vec<DiskStats>, Vec<DriveRuntimeDetailedStats>, DriveCountStats) {
let Some(store) = resolve_obs_object_store_handle() else {
return (Vec::new(), Vec::new(), DriveCountStats::default());
};
let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
let local_server = current_local_node_identity();
let disk_stats = storage_info
.disks
.iter()
.map(|disk| DiskStats {
server: disk.endpoint.clone(),
server: drive_server_label(&disk.endpoint, &local_server),
drive: disk.drive_path.clone(),
total_bytes: disk.total_space,
used_bytes: disk.used_space,
@@ -679,31 +783,61 @@ pub async fn collect_disk_and_system_drive_stats() -> (Vec<DiskStats>, Vec<Drive
} else {
offline_count += 1;
}
let (used_inodes, free_inodes, total_inodes) = drive_inode_stats(disk.used_inodes, disk.free_inodes);
DriveDetailedStats {
server: disk.endpoint.clone(),
drive: disk.drive_path.clone(),
total_bytes: disk.total_space,
used_bytes: disk.used_space,
free_bytes: disk.available_space,
capacity_observation_state,
capacity_observation_age_seconds,
used_inodes: None,
free_inodes: None,
total_inodes: None,
timeout_errors_total: None,
io_errors_total: None,
availability_errors_total: None,
waiting_io: None,
api_latency_micros: None,
health: if is_online { 1 } else { 0 },
reads_per_sec: None,
reads_kb_per_sec: None,
reads_await: None,
writes_per_sec: None,
writes_kb_per_sec: None,
writes_await: None,
perc_util: None,
DriveRuntimeDetailedStats {
pool_index: disk_topology_label(disk.pool_index),
set_index: disk_topology_label(disk.set_index),
drive_index: disk_topology_label(disk.disk_index),
disk_id: non_empty_disk_id(&disk.uuid),
runtime_state: Some(disk.runtime_state.as_deref().unwrap_or("unknown").to_ascii_lowercase()),
healing: disk.healing,
scanning: disk.scanning,
offline_duration_seconds: disk.offline_duration_seconds,
api_calls: disk
.metrics
.as_ref()
.map(|metrics| drive_api_calls(metrics.api_calls.iter()))
.unwrap_or_default(),
api_latency_by_api_micros: disk
.metrics
.as_ref()
.map(|metrics| {
drive_api_latency_by_api_micros(
metrics
.last_minute
.iter()
.map(|(api, action)| (api, action.count, action.acc_time)),
)
})
.unwrap_or_default(),
stats: DriveDetailedStats {
server: drive_server_label(&disk.endpoint, &local_server),
drive: disk.drive_path.clone(),
total_bytes: disk.total_space,
used_bytes: disk.used_space,
free_bytes: disk.available_space,
capacity_observation_state,
capacity_observation_age_seconds,
used_inodes,
free_inodes,
total_inodes,
timeout_errors_total: disk.metrics.as_ref().map(|metrics| metrics.total_errors_timeout),
io_errors_total: None,
availability_errors_total: disk.metrics.as_ref().map(|metrics| metrics.total_errors_availability),
waiting_io: disk.metrics.as_ref().map(|metrics| u64::from(metrics.total_waiting)),
api_latency_micros: disk.metrics.as_ref().and_then(|metrics| {
drive_api_latency_micros(metrics.last_minute.values().map(|action| (action.count, action.acc_time)))
}),
health: if is_online { 1 } else { 0 },
reads_per_sec: None,
reads_kb_per_sec: None,
reads_await: None,
writes_per_sec: None,
writes_kb_per_sec: None,
writes_await: None,
perc_util: None,
},
}
})
.collect();
@@ -1090,22 +1224,75 @@ async fn collect_cluster_usage_metric_stats_from_data_usage(
))
}
fn ilm_action_task_stats(ilm: &ObsIlmRuntimeSnapshot) -> Vec<IlmActionTaskStats> {
vec![
IlmActionTaskStats {
action: "expiry".to_string(),
state: "pending".to_string(),
value: ilm.expiry_pending_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "active".to_string(),
value: ilm.transition_active_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "pending".to_string(),
value: ilm.transition_pending_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "missed_immediate".to_string(),
value: ilm.transition_missed_immediate_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "queue_full".to_string(),
value: ilm.transition_queue_full_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "queue_send_timeout".to_string(),
value: ilm.transition_queue_send_timeout_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "compensation_scheduled".to_string(),
value: ilm.transition_compensation_scheduled_tasks,
},
IlmActionTaskStats {
action: "transition".to_string(),
state: "compensation_running".to_string(),
value: ilm.transition_compensation_running_tasks,
},
]
}
/// Collect ILM metrics from the current lifecycle runtime state.
pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
collect_ilm_runtime_metric_stats().await.map(|stats| stats.stats)
}
pub(crate) async fn collect_ilm_runtime_metric_stats() -> Option<IlmRuntimeStats> {
let ilm = obs_ilm_runtime_snapshot().await;
let metrics = global_metrics().report().await;
let versions_scanned = scanner_lifecycle_checked_versions(&metrics);
Some(IlmStats {
expiry_pending_tasks: ilm.expiry_pending_tasks,
transition_active_tasks: ilm.transition_active_tasks,
transition_pending_tasks: ilm.transition_pending_tasks,
transition_missed_immediate_tasks: ilm.transition_missed_immediate_tasks,
transition_queue_full_tasks: ilm.transition_queue_full_tasks,
transition_queue_send_timeout_tasks: ilm.transition_queue_send_timeout_tasks,
transition_compensation_scheduled_tasks: ilm.transition_compensation_scheduled_tasks,
transition_compensation_running_tasks: ilm.transition_compensation_running_tasks,
versions_scanned,
Some(IlmRuntimeStats {
server: current_local_node_identity(),
action_tasks: ilm_action_task_stats(&ilm),
stats: IlmStats {
expiry_pending_tasks: ilm.expiry_pending_tasks,
transition_active_tasks: ilm.transition_active_tasks,
transition_pending_tasks: ilm.transition_pending_tasks,
transition_missed_immediate_tasks: ilm.transition_missed_immediate_tasks,
transition_queue_full_tasks: ilm.transition_queue_full_tasks,
transition_queue_send_timeout_tasks: ilm.transition_queue_send_timeout_tasks,
transition_compensation_scheduled_tasks: ilm.transition_compensation_scheduled_tasks,
transition_compensation_running_tasks: ilm.transition_compensation_running_tasks,
versions_scanned,
},
})
}
@@ -1120,8 +1307,76 @@ fn scanner_bucket_scans_started(life_time_ops: &HashMap<String, u64>, bucket_sca
.unwrap_or(bucket_scans_finished)
}
fn scanner_source_work_stats(source_work: &[ScannerSourceWorkSnapshot]) -> Vec<ScannerSourceWorkStats> {
let mut stats = source_work
.iter()
.filter(|work| !work.source.is_empty())
.map(|work| ScannerSourceWorkStats {
source: work.source.clone(),
checked: work.checked,
queued: work.queued,
executed: work.executed,
failed: work.failed,
skipped: work.skipped,
missed: work.missed,
})
.collect::<Vec<_>>();
stats.sort_by(|left, right| left.source.cmp(&right.source));
stats
}
fn scanner_current_cycle_source_work_stats(metrics: &ScannerMetricsReport) -> Vec<ScannerSourceWorkStats> {
let current = scanner_source_work_stats(&metrics.current_cycle_source_work);
if !current.is_empty() {
return current;
}
let mut sources = scanner_source_work_stats(&metrics.last_cycle_source_work)
.into_iter()
.map(|work| work.source)
.chain(
scanner_source_work_stats(&metrics.source_work)
.into_iter()
.map(|work| work.source),
)
.collect::<Vec<_>>();
sources.sort();
sources.dedup();
sources
.into_iter()
.map(|source| ScannerSourceWorkStats {
source,
..Default::default()
})
.collect()
}
fn scanner_bucket_drive_result_stats(results: &[ScannerBucketDriveResultSnapshot]) -> Vec<ScannerBucketDriveResultStats> {
let mut stats = results
.iter()
.filter(|result| !result.bucket.is_empty() && !result.drive.is_empty() && !result.result.is_empty() && result.count > 0)
.map(|result| ScannerBucketDriveResultStats {
bucket: result.bucket.clone(),
drive: result.drive.clone(),
result: result.result.clone(),
count: result.count,
})
.collect::<Vec<_>>();
stats.sort_by(|left, right| {
left.bucket
.cmp(&right.bucket)
.then_with(|| left.drive.cmp(&right.drive))
.then_with(|| left.result.cmp(&right.result))
});
stats
}
pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
let metrics = global_metrics().report().await;
collect_scanner_runtime_metric_stats().await.map(|stats| stats.stats)
}
pub(crate) async fn collect_scanner_runtime_metric_stats() -> Option<ScannerRuntimeStats> {
let (metrics, runtime_details) = global_metrics().report_with_runtime_details().await;
let now = Utc::now();
let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default();
let bucket_scans_started = scanner_bucket_scans_started(&metrics.life_time_ops, bucket_scans_finished);
@@ -1147,88 +1402,102 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
let current_cycle_age = current_cycle_age_seconds as f64;
let last_cycle_duration = metrics.last_cycle_duration_seconds;
Some(ScannerStats {
bucket_scans_finished,
bucket_scans_started,
bucket_scans_failed,
directories_scanned,
objects_scanned,
versions_scanned,
last_activity_seconds,
active_paths,
oldest_active_path_age_seconds: metrics.oldest_active_path_age_seconds,
current_set_scan_concurrency_limit: metrics.current_set_scan_concurrency_limit,
current_set_scans_queued: metrics.current_set_scans_queued,
current_set_scans_active: metrics.current_set_scans_active,
current_disk_scan_concurrency_limit: metrics.current_disk_scan_concurrency_limit,
current_disk_bucket_scans_queued: metrics.current_disk_bucket_scans_queued,
current_disk_bucket_scans_active: metrics.current_disk_bucket_scans_active,
throttle_idle_mode_enabled: metrics.throttle_idle_mode_enabled,
throttle_sleep_factor: metrics.throttle_sleep_factor,
throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds,
yield_every_n_objects: metrics.yield_every_n_objects,
cycle_interval_seconds: metrics.cycle_interval_seconds,
cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
cycle_max_objects: metrics.cycle_max_objects,
cycle_max_directories: metrics.cycle_max_directories,
bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
current_cycle: metrics.current_cycle,
completed_cycles,
current_cycle_age_seconds,
current_cycle_objects_scanned: metrics.current_cycle_objects_scanned,
current_cycle_directories_scanned: metrics.current_cycle_directories_scanned,
current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans,
current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures,
current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age),
current_cycle_directories_per_second: scanner_work_rate_per_second(
metrics.current_cycle_directories_scanned,
current_cycle_age,
Some(ScannerRuntimeStats {
server: current_local_node_identity(),
source_work: scanner_source_work_stats(&metrics.source_work),
current_cycle_source_work: scanner_current_cycle_source_work_stats(&metrics),
last_cycle_source_work: scanner_source_work_stats(&metrics.last_cycle_source_work),
bucket_drive_results: scanner_bucket_drive_result_stats(&runtime_details.bucket_drive_results),
current_cycle_bucket_drive_results: scanner_bucket_drive_result_stats(
&runtime_details.current_cycle_bucket_drive_results,
),
current_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second(
metrics.current_cycle_bucket_drive_scans,
current_cycle_age,
),
current_cycle_yield_events: metrics.current_cycle_yield_events,
current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds,
current_cycle_throttle_sleep_events: metrics.current_cycle_throttle_sleep_events,
current_cycle_throttle_sleep_duration_seconds: metrics.current_cycle_throttle_sleep_duration_seconds,
current_cycle_ilm_actions: metrics.current_cycle_ilm_actions,
current_cycle_heal_objects: metrics.current_cycle_heal_objects,
current_cycle_replication_checks: metrics.current_cycle_replication_checks,
current_cycle_usage_saves: metrics.current_cycle_usage_saves,
current_scan_mode,
last_cycle_result: metrics.last_cycle_result_code,
last_cycle_partial_reason: metrics.last_cycle_partial_reason_code,
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans,
last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures,
last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration),
last_cycle_directories_per_second: scanner_work_rate_per_second(
metrics.last_cycle_directories_scanned,
last_cycle_duration,
),
last_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second(
metrics.last_cycle_bucket_drive_scans,
last_cycle_duration,
),
last_cycle_yield_events: metrics.last_cycle_yield_events,
last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds,
last_cycle_throttle_sleep_events: metrics.last_cycle_throttle_sleep_events,
last_cycle_throttle_sleep_duration_seconds: metrics.last_cycle_throttle_sleep_duration_seconds,
last_cycle_ilm_actions: metrics.last_cycle_ilm_actions,
last_cycle_heal_objects: metrics.last_cycle_heal_objects,
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
failed_cycles: metrics.failed_cycles,
superseded_cycles: metrics.superseded_cycles,
partial_cycles: metrics.partial_cycles,
partial_cycles_unknown: metrics.partial_cycles_unknown,
partial_cycles_runtime: metrics.partial_cycles_runtime,
partial_cycles_objects: metrics.partial_cycles_objects,
partial_cycles_directories: metrics.partial_cycles_directories,
last_cycle_bucket_drive_results: scanner_bucket_drive_result_stats(&runtime_details.last_cycle_bucket_drive_results),
stats: ScannerStats {
bucket_scans_finished,
bucket_scans_started,
bucket_scans_failed,
directories_scanned,
objects_scanned,
versions_scanned,
last_activity_seconds,
active_paths,
oldest_active_path_age_seconds: metrics.oldest_active_path_age_seconds,
current_set_scan_concurrency_limit: metrics.current_set_scan_concurrency_limit,
current_set_scans_queued: metrics.current_set_scans_queued,
current_set_scans_active: metrics.current_set_scans_active,
current_disk_scan_concurrency_limit: metrics.current_disk_scan_concurrency_limit,
current_disk_bucket_scans_queued: metrics.current_disk_bucket_scans_queued,
current_disk_bucket_scans_active: metrics.current_disk_bucket_scans_active,
throttle_idle_mode_enabled: metrics.throttle_idle_mode_enabled,
throttle_sleep_factor: metrics.throttle_sleep_factor,
throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds,
yield_every_n_objects: metrics.yield_every_n_objects,
cycle_interval_seconds: metrics.cycle_interval_seconds,
cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
cycle_max_objects: metrics.cycle_max_objects,
cycle_max_directories: metrics.cycle_max_directories,
bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
current_cycle: metrics.current_cycle,
completed_cycles,
current_cycle_age_seconds,
current_cycle_objects_scanned: metrics.current_cycle_objects_scanned,
current_cycle_directories_scanned: metrics.current_cycle_directories_scanned,
current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans,
current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures,
current_cycle_objects_per_second: scanner_work_rate_per_second(
metrics.current_cycle_objects_scanned,
current_cycle_age,
),
current_cycle_directories_per_second: scanner_work_rate_per_second(
metrics.current_cycle_directories_scanned,
current_cycle_age,
),
current_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second(
metrics.current_cycle_bucket_drive_scans,
current_cycle_age,
),
current_cycle_yield_events: metrics.current_cycle_yield_events,
current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds,
current_cycle_throttle_sleep_events: metrics.current_cycle_throttle_sleep_events,
current_cycle_throttle_sleep_duration_seconds: metrics.current_cycle_throttle_sleep_duration_seconds,
current_cycle_ilm_actions: metrics.current_cycle_ilm_actions,
current_cycle_heal_objects: metrics.current_cycle_heal_objects,
current_cycle_replication_checks: metrics.current_cycle_replication_checks,
current_cycle_usage_saves: metrics.current_cycle_usage_saves,
current_scan_mode,
last_cycle_result: metrics.last_cycle_result_code,
last_cycle_partial_reason: metrics.last_cycle_partial_reason_code,
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans,
last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures,
last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration),
last_cycle_directories_per_second: scanner_work_rate_per_second(
metrics.last_cycle_directories_scanned,
last_cycle_duration,
),
last_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second(
metrics.last_cycle_bucket_drive_scans,
last_cycle_duration,
),
last_cycle_yield_events: metrics.last_cycle_yield_events,
last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds,
last_cycle_throttle_sleep_events: metrics.last_cycle_throttle_sleep_events,
last_cycle_throttle_sleep_duration_seconds: metrics.last_cycle_throttle_sleep_duration_seconds,
last_cycle_ilm_actions: metrics.last_cycle_ilm_actions,
last_cycle_heal_objects: metrics.last_cycle_heal_objects,
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
failed_cycles: metrics.failed_cycles,
superseded_cycles: metrics.superseded_cycles,
partial_cycles: metrics.partial_cycles,
partial_cycles_unknown: metrics.partial_cycles_unknown,
partial_cycles_runtime: metrics.partial_cycles_runtime,
partial_cycles_objects: metrics.partial_cycles_objects,
partial_cycles_directories: metrics.partial_cycles_directories,
},
})
}
@@ -1480,6 +1749,60 @@ mod tests {
assert!(!disk_is_online_for_metrics(DRIVE_STATE_OK, Some("offline")));
}
#[test]
fn disk_topology_label_rejects_unknown_negative_index() {
assert_eq!(disk_topology_label(-1), None);
assert_eq!(disk_topology_label(3), Some("3".to_string()));
}
#[test]
fn non_empty_disk_id_rejects_blank_uuid() {
assert_eq!(non_empty_disk_id(" "), None);
assert_eq!(non_empty_disk_id("disk-1"), Some("disk-1".to_string()));
}
#[test]
fn drive_server_label_uses_node_identity_for_urls_and_local_paths() {
assert_eq!(drive_server_label("http://node1:9000/data", "local:9000"), "node1:9000");
assert_eq!(drive_server_label("https://node2:9443/export/d1", "local:9000"), "node2:9443");
assert_eq!(drive_server_label("/mnt/data1", "local:9000"), "local:9000");
}
#[test]
fn drive_inode_stats_skip_unknown_zero_inode_totals() {
assert_eq!(drive_inode_stats(0, 0), (None, None, None));
assert_eq!(drive_inode_stats(2, 3), (Some(2), Some(3), Some(5)));
}
#[test]
fn drive_api_metrics_are_sorted_and_average_latency() {
let last_minute = HashMap::from([
("write".to_string(), (2, 6_000)),
("read".to_string(), (1, 3_000)),
("zero".to_string(), (0, 9_000)),
]);
let api_calls = HashMap::from([("write".to_string(), 9), ("read".to_string(), 4)]);
assert_eq!(drive_api_latency_micros(last_minute.values().copied()), Some(3));
assert_eq!(
drive_api_latency_by_api_micros(last_minute.iter().map(|(api, (count, acc_time))| (api, *count, *acc_time))),
vec![("read".to_string(), 3), ("write".to_string(), 3), ("zero".to_string(), 0)]
);
assert_eq!(drive_api_calls(api_calls.iter()), vec![("read".to_string(), 4), ("write".to_string(), 9)]);
}
#[test]
fn drive_api_latency_skips_zero_denominators() {
let last_minute = HashMap::from([("zero".to_string(), (0, 9_000))]);
assert_eq!(drive_api_latency_micros(last_minute.values().copied()), Some(0));
assert_eq!(
drive_api_latency_by_api_micros(last_minute.iter().map(|(api, (count, acc_time))| (api, *count, *acc_time))),
vec![("zero".to_string(), 0)]
);
assert_eq!(drive_api_latency_micros([].into_iter()), None);
}
#[test]
fn derive_erasure_set_quorum_shape_handles_standard_layout() {
let shape = derive_erasure_set_quorum_shape(16, 4);
@@ -1580,6 +1903,35 @@ mod tests {
assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 5);
}
#[test]
fn ilm_action_task_stats_maps_runtime_states() {
let stats = ilm_action_task_stats(&ObsIlmRuntimeSnapshot {
expiry_pending_tasks: 1,
transition_active_tasks: 2,
transition_pending_tasks: 3,
transition_missed_immediate_tasks: 4,
transition_queue_full_tasks: 5,
transition_queue_send_timeout_tasks: 6,
transition_compensation_scheduled_tasks: 7,
transition_compensation_running_tasks: 8,
});
assert_eq!(stats.len(), 8);
assert_eq!(stats[0].action, "expiry");
assert_eq!(stats[0].state, "pending");
assert_eq!(stats[0].value, 1);
assert!(
stats
.iter()
.any(|task| { task.action == "transition" && task.state == "queue_send_timeout" && task.value == 6 })
);
assert!(
stats
.iter()
.any(|task| { task.action == "transition" && task.state == "compensation_running" && task.value == 8 })
);
}
#[test]
fn scanner_lifecycle_checked_versions_uses_lifecycle_checked_source_work() {
let report = ScannerMetricsReport {
@@ -1602,6 +1954,96 @@ mod tests {
assert_eq!(scanner_lifecycle_checked_versions(&report), 37);
}
#[test]
fn scanner_source_work_stats_sorts_and_skips_empty_source() {
let stats = scanner_source_work_stats(&[
ScannerSourceWorkSnapshot {
source: "usage".to_string(),
checked: 11,
queued: 2,
executed: 3,
failed: 4,
skipped: 5,
missed: 6,
},
ScannerSourceWorkSnapshot {
checked: 99,
queued: 99,
executed: 99,
failed: 99,
skipped: 99,
missed: 99,
..Default::default()
},
ScannerSourceWorkSnapshot {
source: "lifecycle".to_string(),
checked: 21,
queued: 7,
executed: 8,
failed: 9,
skipped: 10,
missed: 12,
},
]);
assert_eq!(stats.len(), 2);
assert_eq!(stats[0].source, "lifecycle");
assert_eq!(stats[0].checked, 21);
assert_eq!(stats[0].missed, 12);
assert_eq!(stats[1].source, "usage");
assert_eq!(stats[1].failed, 4);
}
#[test]
fn scanner_current_cycle_source_work_stats_zeroes_idle_sources() {
let report = ScannerMetricsReport {
source_work: vec![ScannerSourceWorkSnapshot {
source: "usage".to_string(),
checked: 11,
queued: 2,
..Default::default()
}],
last_cycle_source_work: vec![ScannerSourceWorkSnapshot {
source: "lifecycle".to_string(),
checked: 21,
queued: 7,
..Default::default()
}],
..Default::default()
};
let stats = scanner_current_cycle_source_work_stats(&report);
assert_eq!(stats.len(), 2);
assert_eq!(stats[0].source, "lifecycle");
assert_eq!(stats[0].checked, 0);
assert_eq!(stats[1].source, "usage");
assert_eq!(stats[1].queued, 0);
}
#[test]
fn scanner_current_cycle_source_work_stats_keeps_active_values() {
let report = ScannerMetricsReport {
source_work: vec![ScannerSourceWorkSnapshot {
source: "usage".to_string(),
checked: 11,
..Default::default()
}],
current_cycle_source_work: vec![ScannerSourceWorkSnapshot {
source: "usage".to_string(),
checked: 3,
..Default::default()
}],
..Default::default()
};
let stats = scanner_current_cycle_source_work_stats(&report);
assert_eq!(stats.len(), 1);
assert_eq!(stats[0].source, "usage");
assert_eq!(stats[0].checked, 3);
}
#[test]
fn scanner_lifecycle_checked_versions_defaults_to_zero_when_lifecycle_missing() {
let report = ScannerMetricsReport {
crates/obs/src/metrics/storage_api.rs
+120 -55
View File
@@ -18,9 +18,9 @@ use std::time::Duration;
pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
use rustfs_ecstore::api::bucket::replication::{
DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog,
durable_mrf_backlog_summary_snapshot, durable_mrf_target_backlog_snapshot, get_global_replication_stats,
mrf_backlog_observability_snapshot,
BucketReplicationStats as SourceBucketReplicationStats, DurableMrfBucketBacklog, DurableMrfTargetBacklog,
MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog, durable_mrf_backlog_summary_snapshot,
durable_mrf_target_backlog_snapshot, get_global_replication_stats, mrf_backlog_observability_snapshot,
};
pub(crate) use rustfs_ecstore::api::capacity::{
get_total_usable_capacity as obs_get_total_usable_capacity,
@@ -43,6 +43,14 @@ pub(crate) struct ObsBucketReplicationTargetStatsSnapshot {
pub(crate) bandwidth_limit_bytes_per_sec: u64,
pub(crate) current_bandwidth_bytes_per_sec: f64,
pub(crate) latency_ms: f64,
pub(crate) sent_bytes: u64,
pub(crate) sent_count: u64,
pub(crate) total_failed_bytes: u64,
pub(crate) total_failed_count: u64,
pub(crate) last_min_failed_bytes: u64,
pub(crate) last_min_failed_count: u64,
pub(crate) last_hour_failed_bytes: u64,
pub(crate) last_hour_failed_count: u64,
}
#[derive(Debug, Clone, PartialEq)]
@@ -173,6 +181,73 @@ fn replication_backlog_count(failed_counts: impl Iterator<Item = i64>, queued_co
failed_backlog.saturating_add(i64_to_u64_floor_zero(queued_count))
}
fn bucket_replication_runtime_snapshot_from_source(
bucket_stats: Option<&SourceBucketReplicationStats>,
) -> ObsBucketReplicationRuntimeSnapshot {
let mut runtime = ObsBucketReplicationRuntimeSnapshot {
targets: Vec::with_capacity(bucket_stats.map(|stats| stats.stats.len()).unwrap_or(0)),
..Default::default()
};
if let Some(bucket_stats) = bucket_stats {
for (target_arn, target_stats) in &bucket_stats.stats {
runtime.total_failed_bytes = runtime
.total_failed_bytes
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
runtime.total_failed_count = runtime
.total_failed_count
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
runtime.last_min_failed_bytes = runtime
.last_min_failed_bytes
.saturating_add(i64_to_u64_floor_zero(last_min.size));
runtime.last_min_failed_count = runtime
.last_min_failed_count
.saturating_add(i64_to_u64_floor_zero(last_min.count));
let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
runtime.last_hour_failed_bytes = runtime
.last_hour_failed_bytes
.saturating_add(i64_to_u64_floor_zero(last_hour.size));
runtime.last_hour_failed_count = runtime
.last_hour_failed_count
.saturating_add(i64_to_u64_floor_zero(last_hour.count));
runtime.sent_bytes = runtime
.sent_bytes
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
runtime.sent_count = runtime
.sent_count
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
runtime.targets.push(ObsBucketReplicationTargetStatsSnapshot {
target_arn: target_arn.clone(),
bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
latency_ms: target_stats.latency.curr,
sent_bytes: i64_to_u64_floor_zero(target_stats.replicated_size),
sent_count: i64_to_u64_floor_zero(target_stats.replicated_count),
total_failed_bytes: i64_to_u64_floor_zero(target_stats.fail_stats.size),
total_failed_count: i64_to_u64_floor_zero(target_stats.fail_stats.count),
last_min_failed_bytes: i64_to_u64_floor_zero(last_min.size),
last_min_failed_count: i64_to_u64_floor_zero(last_min.count),
last_hour_failed_bytes: i64_to_u64_floor_zero(last_hour.size),
last_hour_failed_count: i64_to_u64_floor_zero(last_hour.count),
});
}
runtime.resync_started_count = i64_to_u64_floor_zero(bucket_stats.resync_started_count);
runtime.resync_completed_count = i64_to_u64_floor_zero(bucket_stats.resync_completed_count);
runtime.resync_failed_count = i64_to_u64_floor_zero(bucket_stats.resync_failed_count);
runtime.resync_canceled_count = i64_to_u64_floor_zero(bucket_stats.resync_canceled_count);
runtime.resync_duration_ms = i64_to_u64_floor_zero(bucket_stats.resync_duration_ms);
runtime.current_backlog_count = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.count);
runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
}
runtime
}
fn bucket_replication_stats_snapshot_from_parts(
bucket: String,
runtime: ObsBucketReplicationRuntimeSnapshot,
@@ -357,58 +432,7 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
proxied_delete_tagging_requests_total: i64_to_u64_floor_zero(proxy.delete_tag_total),
proxied_delete_tagging_requests_failures: i64_to_u64_floor_zero(proxy.delete_tag_failed),
};
let mut runtime = ObsBucketReplicationRuntimeSnapshot {
targets: Vec::with_capacity(bucket_stats.map(|stats| stats.stats.len()).unwrap_or(0)),
..Default::default()
};
if let Some(bucket_stats) = bucket_stats {
for (target_arn, target_stats) in &bucket_stats.stats {
runtime.total_failed_bytes = runtime
.total_failed_bytes
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
runtime.total_failed_count = runtime
.total_failed_count
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
runtime.last_min_failed_bytes = runtime
.last_min_failed_bytes
.saturating_add(i64_to_u64_floor_zero(last_min.size));
runtime.last_min_failed_count = runtime
.last_min_failed_count
.saturating_add(i64_to_u64_floor_zero(last_min.count));
let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
runtime.last_hour_failed_bytes = runtime
.last_hour_failed_bytes
.saturating_add(i64_to_u64_floor_zero(last_hour.size));
runtime.last_hour_failed_count = runtime
.last_hour_failed_count
.saturating_add(i64_to_u64_floor_zero(last_hour.count));
runtime.sent_bytes = runtime
.sent_bytes
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
runtime.sent_count = runtime
.sent_count
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
runtime.targets.push(ObsBucketReplicationTargetStatsSnapshot {
target_arn: target_arn.clone(),
bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
latency_ms: target_stats.latency.curr,
});
}
runtime.resync_started_count = i64_to_u64_floor_zero(bucket_stats.resync_started_count);
runtime.resync_completed_count = i64_to_u64_floor_zero(bucket_stats.resync_completed_count);
runtime.resync_failed_count = i64_to_u64_floor_zero(bucket_stats.resync_failed_count);
runtime.resync_canceled_count = i64_to_u64_floor_zero(bucket_stats.resync_canceled_count);
runtime.resync_duration_ms = i64_to_u64_floor_zero(bucket_stats.resync_duration_ms);
runtime.current_backlog_count = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.count);
runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
}
let runtime = bucket_replication_runtime_snapshot_from_source(bucket_stats);
let durable_bucket = durable_buckets.get(&bucket).cloned().unwrap_or_default();
let runtime_targets = runtime_targets_by_bucket.remove(&bucket).unwrap_or_default();
let durable_targets = durable_targets_by_bucket.remove(&bucket).unwrap_or_default();
@@ -500,6 +524,47 @@ mod tests {
assert_eq!(replication_backlog_count([9].into_iter(), 0), 9);
}
#[test]
fn bucket_replication_runtime_snapshot_maps_target_flow_fields_from_source() {
let mut source = SourceBucketReplicationStats::new();
let target = source.stats.entry("arn:rustfs:replication:target-a".to_string()).or_default();
target.fail_stats.add_size::<()>(100, None);
target.fail_stats.add_size::<()>(200, None);
target.fail_stats.count = 7;
target.fail_stats.size = 900;
target.replicated_size = 1234;
target.replicated_count = 12;
target.bandwidth_limit_bytes_per_sec = 4096;
target.current_bandwidth_bytes_per_sec = 512.5;
target.latency.curr = 45.0;
let snapshot = bucket_replication_runtime_snapshot_from_source(Some(&source));
assert_eq!(snapshot.sent_bytes, 1234);
assert_eq!(snapshot.sent_count, 12);
assert_eq!(snapshot.total_failed_bytes, 900);
assert_eq!(snapshot.total_failed_count, 7);
assert_eq!(snapshot.last_min_failed_bytes, 300);
assert_eq!(snapshot.last_min_failed_count, 2);
assert_eq!(snapshot.last_hour_failed_bytes, 300);
assert_eq!(snapshot.last_hour_failed_count, 2);
assert_eq!(snapshot.targets.len(), 1);
let target = &snapshot.targets[0];
assert_eq!(target.target_arn, "arn:rustfs:replication:target-a");
assert_eq!(target.bandwidth_limit_bytes_per_sec, 4096);
assert_eq!(target.current_bandwidth_bytes_per_sec, 512.5);
assert_eq!(target.latency_ms, 45.0);
assert_eq!(target.sent_bytes, 1234);
assert_eq!(target.sent_count, 12);
assert_eq!(target.total_failed_bytes, 900);
assert_eq!(target.total_failed_count, 7);
assert_eq!(target.last_min_failed_bytes, 300);
assert_eq!(target.last_min_failed_count, 2);
assert_eq!(target.last_hour_failed_bytes, 300);
assert_eq!(target.last_hour_failed_count, 2);
}
#[test]
fn bucket_replication_snapshot_maps_runtime_and_durable_backlog() {
let snapshot = bucket_replication_stats_snapshot_from_parts(