|
|
|
@@ -83,6 +83,11 @@ const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
|
|
|
|
0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
// Keep millisecond resolution for fast requests and coarser buckets for long streams.
|
|
|
|
|
const HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
|
|
|
|
0.0001, 0.001, 0.002, 0.003, 0.005, 0.01, 0.025, 0.05, 0.25, 1.0, 5.0, 25.0, 120.0, 600.0, 2500.0, 10000.0,
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
const LOCAL_BATCH_DURATION_HISTOGRAM_METRICS: &[&str] = &[
|
|
|
|
|
"rustfs_io_get_object_local_batch_prepare_seconds",
|
|
|
|
|
"rustfs_io_get_object_local_batch_stage_seconds",
|
|
|
|
@@ -522,16 +527,15 @@ fn duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn duration_histogram_buckets(name: &str) -> Option<&'static [f64]> {
|
|
|
|
|
if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
|
|
|
|
Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS)
|
|
|
|
|
} else if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
|
|
|
|
Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS)
|
|
|
|
|
} else if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
|
|
|
|
Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS)
|
|
|
|
|
} else if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
|
|
|
|
Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS)
|
|
|
|
|
} else {
|
|
|
|
|
None
|
|
|
|
|
match name {
|
|
|
|
|
"rustfs_http_server_request_duration_seconds"
|
|
|
|
|
| "rustfs_http_server_response_body_chunk_latency_seconds"
|
|
|
|
|
| "rustfs_http_server_response_body_stream_duration_seconds" => Some(HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS),
|
|
|
|
|
_ if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS),
|
|
|
|
|
_ if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS),
|
|
|
|
|
_ if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS),
|
|
|
|
|
_ if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS),
|
|
|
|
|
_ => None,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -1156,11 +1160,79 @@ mod tests {
|
|
|
|
|
"rustfs_io_get_object_local_batch_shards",
|
|
|
|
|
"rustfs_object_lock_diag_slow_acquire_total",
|
|
|
|
|
"rustfs_object_lock_diag_acquire_duration_seconds_bucket",
|
|
|
|
|
"rustfs_http_server_response_body_chunk_size_bytes",
|
|
|
|
|
] {
|
|
|
|
|
assert!(duration_histogram_buckets(name).is_none());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn http_server_duration_histograms_export_seconds_buckets() {
|
|
|
|
|
use opentelemetry_sdk::metrics::{
|
|
|
|
|
InMemoryMetricExporter,
|
|
|
|
|
data::{AggregatedMetrics, MetricData},
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let exporter = InMemoryMetricExporter::default();
|
|
|
|
|
let reader = PeriodicReader::builder(exporter.clone())
|
|
|
|
|
.with_interval(Duration::from_secs(60))
|
|
|
|
|
.build();
|
|
|
|
|
let (provider, recorder) = Recorder::builder("http-server-duration-test")
|
|
|
|
|
.with_meter_provider(|builder| builder.with_reader(reader).with_view(duration_histogram_view))
|
|
|
|
|
.build();
|
|
|
|
|
let instruments = [
|
|
|
|
|
"rustfs_http_server_request_duration_seconds",
|
|
|
|
|
"rustfs_http_server_response_body_chunk_latency_seconds",
|
|
|
|
|
"rustfs_http_server_response_body_stream_duration_seconds",
|
|
|
|
|
];
|
|
|
|
|
let observations = [
|
|
|
|
|
(0.00008, 0.0001),
|
|
|
|
|
(0.0024, 0.003),
|
|
|
|
|
(0.037, 0.05),
|
|
|
|
|
(42.0, 120.0),
|
|
|
|
|
(4000.0, 10000.0),
|
|
|
|
|
(12000.0, f64::INFINITY),
|
|
|
|
|
];
|
|
|
|
|
metrics::with_local_recorder(&recorder, || {
|
|
|
|
|
for name in instruments {
|
|
|
|
|
let histogram = metrics::histogram!(name);
|
|
|
|
|
for (seconds, _) in observations {
|
|
|
|
|
histogram.record(seconds);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
provider.force_flush().expect("export HTTP duration observations");
|
|
|
|
|
let exported = exporter.get_finished_metrics().expect("read exported HTTP histograms");
|
|
|
|
|
let expected_sum = observations.iter().map(|(seconds, _)| seconds).sum::<f64>();
|
|
|
|
|
let expected_buckets = observations.map(|(_, bound)| (bound, 1));
|
|
|
|
|
for name in instruments {
|
|
|
|
|
let metric = exported
|
|
|
|
|
.iter()
|
|
|
|
|
.flat_map(|resource| resource.scope_metrics())
|
|
|
|
|
.flat_map(|scope| scope.metrics())
|
|
|
|
|
.find(|metric| metric.name() == name)
|
|
|
|
|
.unwrap_or_else(|| panic!("{name} must be exported"));
|
|
|
|
|
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
|
|
|
|
|
panic!("{name} must export as an f64 histogram");
|
|
|
|
|
};
|
|
|
|
|
let mut points = histogram.data_points();
|
|
|
|
|
let point = points.next().unwrap_or_else(|| panic!("{name} must export a point"));
|
|
|
|
|
assert!(points.next().is_none(), "{name} must export one point");
|
|
|
|
|
assert_eq!(point.count(), 6, "{name} must retain every observation");
|
|
|
|
|
assert!((point.sum() - expected_sum).abs() < 1e-9, "{name} must retain the sum in seconds");
|
|
|
|
|
assert_eq!(point.bucket_counts().count(), 17, "{name} must have 16 finite buckets and overflow");
|
|
|
|
|
let occupied: Vec<_> = point
|
|
|
|
|
.bounds()
|
|
|
|
|
.chain([f64::INFINITY])
|
|
|
|
|
.zip(point.bucket_counts())
|
|
|
|
|
.filter(|(_, count)| *count != 0)
|
|
|
|
|
.collect();
|
|
|
|
|
assert_eq!(occupied, expected_buckets, "{name}");
|
|
|
|
|
}
|
|
|
|
|
provider.shutdown().expect("stop HTTP histogram test provider");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn object_lock_histograms_export_millisecond_and_long_hold_buckets() {
|
|
|
|
|
use metrics::Recorder as _;
|
|
|
|
|