Compare commits

...

4 Commits

Author SHA1 Message Date
Hauser Zhi 7676140898 Merge branch 'main' into houseme/fix-http-duration-histograms 2026-11-18 21:50:01 +08:00
Hauser Zhi f3df5a6808 Merge origin/main into the HTTP histogram fix
Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-18 18:29:10 +08:00
Hauser Zhi 87a910e335 perf(obs): reduce HTTP duration histogram bucket overhead
Keep the 2–3 ms interval and long-stream range with 16 finite bounds.
Match the three instrument names directly in the existing view.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-18 18:16:53 +08:00
Hauser Zhi a4d4ce5522 fix(obs): use seconds-scale buckets for HTTP durations
Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-18 18:00:18 +08:00
3 changed files with 90 additions and 10 deletions
+4
View File
@@ -34,6 +34,10 @@ If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rust
- **High Performance**: Optimized configurations for batching, compression, and memory management.
- **Standardized Protocols**: Built entirely on OpenTelemetry standards.
## HTTP Request Latency Percentiles
The main RustFS dashboard shows request latency percentiles in milliseconds. The HTTP server request, response-body chunk, and response-body stream histograms export bucket boundaries in seconds. During a rolling upgrade or rollback that changes the bucket layout, compute percentiles from nodes that share one layout and from a query window entirely on that layout. The bundled dashboard and recording rules use a five-minute window. The mean from `_sum` divided by `_count` remains usable across the change.
## GET Performance Optimization Dashboards
Three pre-built Grafana dashboards are included for monitoring RustFS GET performance optimization rollout:
+4
View File
@@ -34,6 +34,10 @@
- **高性能**: 针对批处理、压缩和内存管理进行了优化配置。
- **标准化协议**: 完全基于 OpenTelemetry 标准构建。
## HTTP 请求延迟分位数
RustFS 主仪表盘以毫秒显示请求延迟分位数。HTTP 服务的请求耗时、响应体分块延迟和响应体流传输耗时直方图以秒为单位导出桶边界。滚动升级或回滚导致桶边界变化时,只对采用同一套桶边界的节点和时间窗口计算分位数。自带仪表盘和记录规则使用五分钟窗口。变更期间仍可用 `_sum` 除以 `_count` 计算平均值。
## GET 性能优化仪表盘
包含三个预构建的 Grafana 仪表盘,用于监控 RustFS GET 性能优化发布:
+82 -10
View File
@@ -83,6 +83,11 @@ const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
];
// Keep millisecond resolution for fast requests and coarser buckets for long streams.
const HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
0.0001, 0.001, 0.002, 0.003, 0.005, 0.01, 0.025, 0.05, 0.25, 1.0, 5.0, 25.0, 120.0, 600.0, 2500.0, 10000.0,
];
const LOCAL_BATCH_DURATION_HISTOGRAM_METRICS: &[&str] = &[
"rustfs_io_get_object_local_batch_prepare_seconds",
"rustfs_io_get_object_local_batch_stage_seconds",
@@ -522,16 +527,15 @@ fn duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
}
fn duration_histogram_buckets(name: &str) -> Option<&'static [f64]> {
if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS)
} else if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS)
} else if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS)
} else if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS)
} else {
None
match name {
"rustfs_http_server_request_duration_seconds"
| "rustfs_http_server_response_body_chunk_latency_seconds"
| "rustfs_http_server_response_body_stream_duration_seconds" => Some(HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS),
_ if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS),
_ if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS),
_ if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS),
_ if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS),
_ => None,
}
}
@@ -1156,11 +1160,79 @@ mod tests {
"rustfs_io_get_object_local_batch_shards",
"rustfs_object_lock_diag_slow_acquire_total",
"rustfs_object_lock_diag_acquire_duration_seconds_bucket",
"rustfs_http_server_response_body_chunk_size_bytes",
] {
assert!(duration_histogram_buckets(name).is_none());
}
}
#[test]
fn http_server_duration_histograms_export_seconds_buckets() {
use opentelemetry_sdk::metrics::{
InMemoryMetricExporter,
data::{AggregatedMetrics, MetricData},
};
let exporter = InMemoryMetricExporter::default();
let reader = PeriodicReader::builder(exporter.clone())
.with_interval(Duration::from_secs(60))
.build();
let (provider, recorder) = Recorder::builder("http-server-duration-test")
.with_meter_provider(|builder| builder.with_reader(reader).with_view(duration_histogram_view))
.build();
let instruments = [
"rustfs_http_server_request_duration_seconds",
"rustfs_http_server_response_body_chunk_latency_seconds",
"rustfs_http_server_response_body_stream_duration_seconds",
];
let observations = [
(0.00008, 0.0001),
(0.0024, 0.003),
(0.037, 0.05),
(42.0, 120.0),
(4000.0, 10000.0),
(12000.0, f64::INFINITY),
];
metrics::with_local_recorder(&recorder, || {
for name in instruments {
let histogram = metrics::histogram!(name);
for (seconds, _) in observations {
histogram.record(seconds);
}
}
});
provider.force_flush().expect("export HTTP duration observations");
let exported = exporter.get_finished_metrics().expect("read exported HTTP histograms");
let expected_sum = observations.iter().map(|(seconds, _)| seconds).sum::<f64>();
let expected_buckets = observations.map(|(_, bound)| (bound, 1));
for name in instruments {
let metric = exported
.iter()
.flat_map(|resource| resource.scope_metrics())
.flat_map(|scope| scope.metrics())
.find(|metric| metric.name() == name)
.unwrap_or_else(|| panic!("{name} must be exported"));
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
panic!("{name} must export as an f64 histogram");
};
let mut points = histogram.data_points();
let point = points.next().unwrap_or_else(|| panic!("{name} must export a point"));
assert!(points.next().is_none(), "{name} must export one point");
assert_eq!(point.count(), 6, "{name} must retain every observation");
assert!((point.sum() - expected_sum).abs() < 1e-9, "{name} must retain the sum in seconds");
assert_eq!(point.bucket_counts().count(), 17, "{name} must have 16 finite buckets and overflow");
let occupied: Vec<_> = point
.bounds()
.chain([f64::INFINITY])
.zip(point.bucket_counts())
.filter(|(_, count)| *count != 0)
.collect();
assert_eq!(occupied, expected_buckets, "{name}");
}
provider.shutdown().expect("stop HTTP histogram test provider");
}
#[test]
fn object_lock_histograms_export_millisecond_and_long_hold_buckets() {
use metrics::Recorder as _;