fix(obs): use seconds-scale buckets for HTTP durations (#9504)

* fix(obs): use seconds-scale buckets for HTTP durations

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* perf(obs): reduce HTTP duration histogram bucket overhead

Keep the 2–3 ms interval and long-stream range with 16 finite bounds.
Match the three instrument names directly in the existing view.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix: restore authorization and storage test invariants

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(notify): keep rule engine storage calls behind boundary

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix: satisfy current CI architecture and clippy guards

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

---------

Signed-off-by: Hauser Zhi <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
Hauser Zhi
2026-11-19 18:53:11 +08:00
committed by GitHub
parent ec8dc7d1ca
commit 88b90c1315
10 changed files with 153 additions and 32 deletions
+4
View File
@@ -34,6 +34,10 @@ If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rust
- **High Performance**: Optimized configurations for batching, compression, and memory management.
- **Standardized Protocols**: Built entirely on OpenTelemetry standards.
## HTTP Request Latency Percentiles
The main RustFS dashboard shows request latency percentiles in milliseconds. The HTTP server request, response-body chunk, and response-body stream histograms export bucket boundaries in seconds. During a rolling upgrade or rollback that changes the bucket layout, compute percentiles from nodes that share one layout and from a query window entirely on that layout. The bundled dashboard and recording rules use a five-minute window. The mean from `_sum` divided by `_count` remains usable across the change.
## GET Performance Optimization Dashboards
Three pre-built Grafana dashboards are included for monitoring RustFS GET performance optimization rollout:
+4
View File
@@ -34,6 +34,10 @@
- **高性能**: 针对批处理、压缩和内存管理进行了优化配置。
- **标准化协议**: 完全基于 OpenTelemetry 标准构建。
## HTTP 请求延迟分位数
RustFS 主仪表盘以毫秒显示请求延迟分位数。HTTP 服务的请求耗时、响应体分块延迟和响应体流传输耗时直方图以秒为单位导出桶边界。滚动升级或回滚导致桶边界变化时,只对采用同一套桶边界的节点和时间窗口计算分位数。自带仪表盘和记录规则使用五分钟窗口。变更期间仍可用 `_sum` 除以 `_count` 计算平均值。
## GET 性能优化仪表盘
包含三个预构建的 Grafana 仪表盘,用于监控 RustFS GET 性能优化发布:
@@ -1071,10 +1071,9 @@ mod tests {
}
fn signing_time(headers: &[(String, String)]) -> jiff::Timestamp {
jiff::civil::DateTime::strptime("%Y%m%dT%H%M%SZ", recorded_header(headers, "x-amz-date"))
.and_then(|datetime| datetime.to_zoned_datetime(jiff::tz::TimeZone::UTC))
.map(|datetime| datetime.timestamp())
.expect("SDK signing timestamp must use the UTC SigV4 format")
let value = recorded_header(headers, "x-amz-date").trim_end_matches('Z');
jiff::Timestamp::strptime("%Y%m%dT%H%M%S%z", format!("{value}+0000"))
.expect("SDK signing timestamp must use the SigV4 format")
}
impl SmithyHttpConnector for ClockSkewConnector {
+20 -7
View File
@@ -14854,13 +14854,9 @@ async fn batch_shard_pread_blocking(requests: Vec<(std::path::PathBuf, usize, us
metrics.as_deref(),
LocalBatchReadStage::Read,
|| {
// Announce the exact window before reading it. Kernel
// readahead otherwise sizes its first request from the
// read size: a small window at offset 0 pulls several
// times its size, and a multi-MiB read is split into
// serial readahead chunks. WILLNEED submits exactly the
// window at once. It is advisory, so an unsupported hint
// must not fail the read.
// Preload the requested window before reading it. Kernel
// readahead is advisory and may still extend past the
// window, so the tail is reclaimed after the read below.
#[cfg(target_os = "linux")]
if let (Ok(start), Ok(len)) = (u64::try_from(offset), u64::try_from(length)) {
let _ = rustix::fs::fadvise(
@@ -14885,6 +14881,23 @@ async fn batch_shard_pread_blocking(requests: Vec<(std::path::PathBuf, usize, us
}
total += nbytes;
}
// Keep a cold shard window from evicting unrelated data
// when the kernel over-reads. The requested tail page is
// retained because its final bytes may share a page with
// the readahead range.
#[cfg(target_os = "linux")]
if let Ok(page) = mmap_page_size() {
let aligned_end = end
.checked_add(page.saturating_sub(1))
.map(|value| value / page * page)
.unwrap_or(end);
if aligned_end < meta.len()
&& let Some(reclaim_len) = std::num::NonZeroU64::new(meta.len() - aligned_end)
{
let _ =
rustix::fs::fadvise(&file, aligned_end, Some(reclaim_len), rustix::fs::Advice::DontNeed);
}
}
Ok::<_, DiskError>(())
}
))?;
@@ -17380,6 +17380,9 @@ mod metadata_mutation_generation_tests {
version_id: old.version_id.map(|id| id.to_string()),
versioned: true,
mod_time: Some(old_time),
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
};
let dst_opts = ObjectOptions {
+3
View File
@@ -2854,6 +2854,9 @@ async fn evaluate_prepared_session_policy(policy: &PreparedSessionPolicy, args:
PreparedSessionPolicy::Policy(p) => {
let mut session_policy_args = args.clone();
session_policy_args.is_owner = false;
// Self-service checks ask whether an explicit session-policy Deny
// applies. They must not require an unrelated Allow statement.
session_policy_args.deny_only = args.deny_only;
Some(p.is_allowed(&session_policy_args).await)
}
}
+82 -10
View File
@@ -83,6 +83,11 @@ const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
];
// Keep millisecond resolution for fast requests and coarser buckets for long streams.
const HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
0.0001, 0.001, 0.002, 0.003, 0.005, 0.01, 0.025, 0.05, 0.25, 1.0, 5.0, 25.0, 120.0, 600.0, 2500.0, 10000.0,
];
const LOCAL_BATCH_DURATION_HISTOGRAM_METRICS: &[&str] = &[
"rustfs_io_get_object_local_batch_prepare_seconds",
"rustfs_io_get_object_local_batch_stage_seconds",
@@ -522,16 +527,15 @@ fn duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
}
fn duration_histogram_buckets(name: &str) -> Option<&'static [f64]> {
if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS)
} else if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS)
} else if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS)
} else if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS)
} else {
None
match name {
"rustfs_http_server_request_duration_seconds"
| "rustfs_http_server_response_body_chunk_latency_seconds"
| "rustfs_http_server_response_body_stream_duration_seconds" => Some(HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS),
_ if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS),
_ if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS),
_ if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS),
_ if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS),
_ => None,
}
}
@@ -1156,11 +1160,79 @@ mod tests {
"rustfs_io_get_object_local_batch_shards",
"rustfs_object_lock_diag_slow_acquire_total",
"rustfs_object_lock_diag_acquire_duration_seconds_bucket",
"rustfs_http_server_response_body_chunk_size_bytes",
] {
assert!(duration_histogram_buckets(name).is_none());
}
}
#[test]
fn http_server_duration_histograms_export_seconds_buckets() {
use opentelemetry_sdk::metrics::{
InMemoryMetricExporter,
data::{AggregatedMetrics, MetricData},
};
let exporter = InMemoryMetricExporter::default();
let reader = PeriodicReader::builder(exporter.clone())
.with_interval(Duration::from_secs(60))
.build();
let (provider, recorder) = Recorder::builder("http-server-duration-test")
.with_meter_provider(|builder| builder.with_reader(reader).with_view(duration_histogram_view))
.build();
let instruments = [
"rustfs_http_server_request_duration_seconds",
"rustfs_http_server_response_body_chunk_latency_seconds",
"rustfs_http_server_response_body_stream_duration_seconds",
];
let observations = [
(0.00008, 0.0001),
(0.0024, 0.003),
(0.037, 0.05),
(42.0, 120.0),
(4000.0, 10000.0),
(12000.0, f64::INFINITY),
];
metrics::with_local_recorder(&recorder, || {
for name in instruments {
let histogram = metrics::histogram!(name);
for (seconds, _) in observations {
histogram.record(seconds);
}
}
});
provider.force_flush().expect("export HTTP duration observations");
let exported = exporter.get_finished_metrics().expect("read exported HTTP histograms");
let expected_sum = observations.iter().map(|(seconds, _)| seconds).sum::<f64>();
let expected_buckets = observations.map(|(_, bound)| (bound, 1));
for name in instruments {
let metric = exported
.iter()
.flat_map(|resource| resource.scope_metrics())
.flat_map(|scope| scope.metrics())
.find(|metric| metric.name() == name)
.unwrap_or_else(|| panic!("{name} must be exported"));
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
panic!("{name} must export as an f64 histogram");
};
let mut points = histogram.data_points();
let point = points.next().unwrap_or_else(|| panic!("{name} must export a point"));
assert!(points.next().is_none(), "{name} must export one point");
assert_eq!(point.count(), 6, "{name} must retain every observation");
assert!((point.sum() - expected_sum).abs() < 1e-9, "{name} must retain the sum in seconds");
assert_eq!(point.bucket_counts().count(), 17, "{name} must have 16 finite buckets and overflow");
let occupied: Vec<_> = point
.bounds()
.chain([f64::INFINITY])
.zip(point.bucket_counts())
.filter(|(_, count)| *count != 0)
.collect();
assert_eq!(occupied, expected_buckets, "{name}");
}
provider.shutdown().expect("stop HTTP histogram test provider");
}
#[test]
fn object_lock_histograms_export_millisecond_and_long_hold_buckets() {
use metrics::Recorder as _;
+17 -7
View File
@@ -359,18 +359,22 @@ mod tests {
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, AtomicU64};
use time::OffsetDateTime;
/// A `Store` whose methods are never invoked by these tests: the gate only
/// reaches the persistence layer for non-owner principals, and the
/// `IamCache` we build starts with an empty in-memory snapshot, so
/// `get_user` resolves to `None` (unknown user) before any `api` call.
#[derive(Clone)]
struct EmptyStore;
/// An in-memory test store with the small authoritative revision surface
/// needed by derived-session authorization. All other persistence methods
/// remain unreachable in these gate tests.
#[derive(Clone, Default)]
struct EmptyStore {
regular_users: Arc<std::sync::RwLock<HashMap<String, (UserIdentity, String)>>>,
}
#[async_trait::async_trait]
impl Store for EmptyStore {
fn has_watcher(&self) -> bool {
false
}
async fn regular_user_revision(&self, name: &str) -> rustfs_iam::error::Result<Option<(UserIdentity, String)>> {
Ok(self.regular_users.read().expect("test store lock").get(name).cloned())
}
async fn save_iam_config<Item: Serialize + Send>(
&self,
_item: Item,
@@ -496,7 +500,7 @@ mod tests {
let (send_chan, _rx) = tokio::sync::mpsc::channel::<i64>(1);
IamCache {
cache: Cache::default(),
api: EmptyStore,
api: EmptyStore::default(),
state: Arc::new(AtomicU8::new(0)),
loading: Arc::new(AtomicBool::new(false)),
roles: HashMap::new(),
@@ -868,6 +872,12 @@ mod tests {
store
.cache
.add_or_update_user(&user.access_key, &UserIdentity::new(user.clone()), now);
store
.api
.regular_users
.write()
.expect("test store lock")
.insert(user.access_key.clone(), (UserIdentity::new(user.clone()), "self-service".into()));
store
.cache
.add_or_update_user_policy(&user.access_key, &MappedPolicy::new(""), now);
@@ -205,6 +205,12 @@ fn listen_notification_streams_close_after_identity_policy_or_bucket_revocation(
)
.await
.expect("listener");
let parent_credentials = rustfs_credentials::Credentials {
access_key: key.clone(),
secret_key: "listen-user-secret".into(),
status: "on".into(),
..Default::default()
};
let policy = serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow",
"Action":["s3:ListenNotification","s3:ListenBucketNotification"],"Resource":"arn:aws:s3:::*"}]});
iam.set_policy(
@@ -227,7 +233,7 @@ fn listen_notification_streams_close_after_identity_policy_or_bucket_revocation(
let mut cred = rustfs_policy::auth::get_new_credentials_with_metadata(&claims, "listen-root-secret")
.expect("signed temporary credentials");
cred.parent_user = key.clone();
iam.set_temp_user(&cred.access_key, &cred, None)
iam.set_temp_user_from_parent(&cred, &parent_credentials)
.await
.expect("temporary listener");
Some(cred)
+10 -3
View File
@@ -355,9 +355,16 @@ impl Operation for AddUser {
args.policy = None;
updated_at
} else {
if authorize_admin_envelope(req.envelope(), vec![Action::AdminAction(AdminAction::EnableUserAdminAction)])
.await
.is_ok()
if validate_admin_request(
req.headers(),
&cred,
owner,
false,
vec![Action::AdminAction(AdminAction::EnableUserAdminAction)],
req.policy_source_addr(),
)
.await
.is_ok()
{
iam_store.create_user(ak, &args).await
} else {