mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-20 06:53:44 +00:00
fix(obs): use seconds-scale buckets for HTTP durations (#9504)
* fix(obs): use seconds-scale buckets for HTTP durations Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * perf(obs): reduce HTTP duration histogram bucket overhead Keep the 2–3 ms interval and long-stream range with 16 finite bounds. Match the three instrument names directly in the existing view. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix: restore authorization and storage test invariants Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(notify): keep rule engine storage calls behind boundary Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix: satisfy current CI architecture and clippy guards Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> --------- Signed-off-by: Hauser Zhi <housemecn@gmail.com> Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
@@ -34,6 +34,10 @@ If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rust
|
||||
- **High Performance**: Optimized configurations for batching, compression, and memory management.
|
||||
- **Standardized Protocols**: Built entirely on OpenTelemetry standards.
|
||||
|
||||
## HTTP Request Latency Percentiles
|
||||
|
||||
The main RustFS dashboard shows request latency percentiles in milliseconds. The HTTP server request, response-body chunk, and response-body stream histograms export bucket boundaries in seconds. During a rolling upgrade or rollback that changes the bucket layout, compute percentiles from nodes that share one layout and from a query window entirely on that layout. The bundled dashboard and recording rules use a five-minute window. The mean from `_sum` divided by `_count` remains usable across the change.
|
||||
|
||||
## GET Performance Optimization Dashboards
|
||||
|
||||
Three pre-built Grafana dashboards are included for monitoring RustFS GET performance optimization rollout:
|
||||
|
||||
@@ -34,6 +34,10 @@
|
||||
- **高性能**: 针对批处理、压缩和内存管理进行了优化配置。
|
||||
- **标准化协议**: 完全基于 OpenTelemetry 标准构建。
|
||||
|
||||
## HTTP 请求延迟分位数
|
||||
|
||||
RustFS 主仪表盘以毫秒显示请求延迟分位数。HTTP 服务的请求耗时、响应体分块延迟和响应体流传输耗时直方图以秒为单位导出桶边界。滚动升级或回滚导致桶边界变化时,只对采用同一套桶边界的节点和时间窗口计算分位数。自带仪表盘和记录规则使用五分钟窗口。变更期间仍可用 `_sum` 除以 `_count` 计算平均值。
|
||||
|
||||
## GET 性能优化仪表盘
|
||||
|
||||
包含三个预构建的 Grafana 仪表盘,用于监控 RustFS GET 性能优化发布:
|
||||
|
||||
@@ -1071,10 +1071,9 @@ mod tests {
|
||||
}
|
||||
|
||||
fn signing_time(headers: &[(String, String)]) -> jiff::Timestamp {
|
||||
jiff::civil::DateTime::strptime("%Y%m%dT%H%M%SZ", recorded_header(headers, "x-amz-date"))
|
||||
.and_then(|datetime| datetime.to_zoned_datetime(jiff::tz::TimeZone::UTC))
|
||||
.map(|datetime| datetime.timestamp())
|
||||
.expect("SDK signing timestamp must use the UTC SigV4 format")
|
||||
let value = recorded_header(headers, "x-amz-date").trim_end_matches('Z');
|
||||
jiff::Timestamp::strptime("%Y%m%dT%H%M%S%z", format!("{value}+0000"))
|
||||
.expect("SDK signing timestamp must use the SigV4 format")
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for ClockSkewConnector {
|
||||
|
||||
@@ -14854,13 +14854,9 @@ async fn batch_shard_pread_blocking(requests: Vec<(std::path::PathBuf, usize, us
|
||||
metrics.as_deref(),
|
||||
LocalBatchReadStage::Read,
|
||||
|| {
|
||||
// Announce the exact window before reading it. Kernel
|
||||
// readahead otherwise sizes its first request from the
|
||||
// read size: a small window at offset 0 pulls several
|
||||
// times its size, and a multi-MiB read is split into
|
||||
// serial readahead chunks. WILLNEED submits exactly the
|
||||
// window at once. It is advisory, so an unsupported hint
|
||||
// must not fail the read.
|
||||
// Preload the requested window before reading it. Kernel
|
||||
// readahead is advisory and may still extend past the
|
||||
// window, so the tail is reclaimed after the read below.
|
||||
#[cfg(target_os = "linux")]
|
||||
if let (Ok(start), Ok(len)) = (u64::try_from(offset), u64::try_from(length)) {
|
||||
let _ = rustix::fs::fadvise(
|
||||
@@ -14885,6 +14881,23 @@ async fn batch_shard_pread_blocking(requests: Vec<(std::path::PathBuf, usize, us
|
||||
}
|
||||
total += nbytes;
|
||||
}
|
||||
// Keep a cold shard window from evicting unrelated data
|
||||
// when the kernel over-reads. The requested tail page is
|
||||
// retained because its final bytes may share a page with
|
||||
// the readahead range.
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Ok(page) = mmap_page_size() {
|
||||
let aligned_end = end
|
||||
.checked_add(page.saturating_sub(1))
|
||||
.map(|value| value / page * page)
|
||||
.unwrap_or(end);
|
||||
if aligned_end < meta.len()
|
||||
&& let Some(reclaim_len) = std::num::NonZeroU64::new(meta.len() - aligned_end)
|
||||
{
|
||||
let _ =
|
||||
rustix::fs::fadvise(&file, aligned_end, Some(reclaim_len), rustix::fs::Advice::DontNeed);
|
||||
}
|
||||
}
|
||||
Ok::<_, DiskError>(())
|
||||
}
|
||||
))?;
|
||||
|
||||
@@ -17380,6 +17380,9 @@ mod metadata_mutation_generation_tests {
|
||||
version_id: old.version_id.map(|id| id.to_string()),
|
||||
versioned: true,
|
||||
mod_time: Some(old_time),
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
};
|
||||
let dst_opts = ObjectOptions {
|
||||
|
||||
@@ -2854,6 +2854,9 @@ async fn evaluate_prepared_session_policy(policy: &PreparedSessionPolicy, args:
|
||||
PreparedSessionPolicy::Policy(p) => {
|
||||
let mut session_policy_args = args.clone();
|
||||
session_policy_args.is_owner = false;
|
||||
// Self-service checks ask whether an explicit session-policy Deny
|
||||
// applies. They must not require an unrelated Allow statement.
|
||||
session_policy_args.deny_only = args.deny_only;
|
||||
Some(p.is_allowed(&session_policy_args).await)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,6 +83,11 @@ const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
|
||||
];
|
||||
|
||||
// Keep millisecond resolution for fast requests and coarser buckets for long streams.
|
||||
const HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
0.0001, 0.001, 0.002, 0.003, 0.005, 0.01, 0.025, 0.05, 0.25, 1.0, 5.0, 25.0, 120.0, 600.0, 2500.0, 10000.0,
|
||||
];
|
||||
|
||||
const LOCAL_BATCH_DURATION_HISTOGRAM_METRICS: &[&str] = &[
|
||||
"rustfs_io_get_object_local_batch_prepare_seconds",
|
||||
"rustfs_io_get_object_local_batch_stage_seconds",
|
||||
@@ -522,16 +527,15 @@ fn duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
|
||||
}
|
||||
|
||||
fn duration_histogram_buckets(name: &str) -> Option<&'static [f64]> {
|
||||
if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
||||
Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS)
|
||||
} else if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
||||
Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS)
|
||||
} else if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
||||
Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS)
|
||||
} else if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
||||
Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS)
|
||||
} else {
|
||||
None
|
||||
match name {
|
||||
"rustfs_http_server_request_duration_seconds"
|
||||
| "rustfs_http_server_response_body_chunk_latency_seconds"
|
||||
| "rustfs_http_server_response_body_stream_duration_seconds" => Some(HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS),
|
||||
_ if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS),
|
||||
_ if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS),
|
||||
_ if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS),
|
||||
_ if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1156,11 +1160,79 @@ mod tests {
|
||||
"rustfs_io_get_object_local_batch_shards",
|
||||
"rustfs_object_lock_diag_slow_acquire_total",
|
||||
"rustfs_object_lock_diag_acquire_duration_seconds_bucket",
|
||||
"rustfs_http_server_response_body_chunk_size_bytes",
|
||||
] {
|
||||
assert!(duration_histogram_buckets(name).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_server_duration_histograms_export_seconds_buckets() {
|
||||
use opentelemetry_sdk::metrics::{
|
||||
InMemoryMetricExporter,
|
||||
data::{AggregatedMetrics, MetricData},
|
||||
};
|
||||
|
||||
let exporter = InMemoryMetricExporter::default();
|
||||
let reader = PeriodicReader::builder(exporter.clone())
|
||||
.with_interval(Duration::from_secs(60))
|
||||
.build();
|
||||
let (provider, recorder) = Recorder::builder("http-server-duration-test")
|
||||
.with_meter_provider(|builder| builder.with_reader(reader).with_view(duration_histogram_view))
|
||||
.build();
|
||||
let instruments = [
|
||||
"rustfs_http_server_request_duration_seconds",
|
||||
"rustfs_http_server_response_body_chunk_latency_seconds",
|
||||
"rustfs_http_server_response_body_stream_duration_seconds",
|
||||
];
|
||||
let observations = [
|
||||
(0.00008, 0.0001),
|
||||
(0.0024, 0.003),
|
||||
(0.037, 0.05),
|
||||
(42.0, 120.0),
|
||||
(4000.0, 10000.0),
|
||||
(12000.0, f64::INFINITY),
|
||||
];
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
for name in instruments {
|
||||
let histogram = metrics::histogram!(name);
|
||||
for (seconds, _) in observations {
|
||||
histogram.record(seconds);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
provider.force_flush().expect("export HTTP duration observations");
|
||||
let exported = exporter.get_finished_metrics().expect("read exported HTTP histograms");
|
||||
let expected_sum = observations.iter().map(|(seconds, _)| seconds).sum::<f64>();
|
||||
let expected_buckets = observations.map(|(_, bound)| (bound, 1));
|
||||
for name in instruments {
|
||||
let metric = exported
|
||||
.iter()
|
||||
.flat_map(|resource| resource.scope_metrics())
|
||||
.flat_map(|scope| scope.metrics())
|
||||
.find(|metric| metric.name() == name)
|
||||
.unwrap_or_else(|| panic!("{name} must be exported"));
|
||||
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
|
||||
panic!("{name} must export as an f64 histogram");
|
||||
};
|
||||
let mut points = histogram.data_points();
|
||||
let point = points.next().unwrap_or_else(|| panic!("{name} must export a point"));
|
||||
assert!(points.next().is_none(), "{name} must export one point");
|
||||
assert_eq!(point.count(), 6, "{name} must retain every observation");
|
||||
assert!((point.sum() - expected_sum).abs() < 1e-9, "{name} must retain the sum in seconds");
|
||||
assert_eq!(point.bucket_counts().count(), 17, "{name} must have 16 finite buckets and overflow");
|
||||
let occupied: Vec<_> = point
|
||||
.bounds()
|
||||
.chain([f64::INFINITY])
|
||||
.zip(point.bucket_counts())
|
||||
.filter(|(_, count)| *count != 0)
|
||||
.collect();
|
||||
assert_eq!(occupied, expected_buckets, "{name}");
|
||||
}
|
||||
provider.shutdown().expect("stop HTTP histogram test provider");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_lock_histograms_export_millisecond_and_long_hold_buckets() {
|
||||
use metrics::Recorder as _;
|
||||
|
||||
@@ -359,18 +359,22 @@ mod tests {
|
||||
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, AtomicU64};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// A `Store` whose methods are never invoked by these tests: the gate only
|
||||
/// reaches the persistence layer for non-owner principals, and the
|
||||
/// `IamCache` we build starts with an empty in-memory snapshot, so
|
||||
/// `get_user` resolves to `None` (unknown user) before any `api` call.
|
||||
#[derive(Clone)]
|
||||
struct EmptyStore;
|
||||
/// An in-memory test store with the small authoritative revision surface
|
||||
/// needed by derived-session authorization. All other persistence methods
|
||||
/// remain unreachable in these gate tests.
|
||||
#[derive(Clone, Default)]
|
||||
struct EmptyStore {
|
||||
regular_users: Arc<std::sync::RwLock<HashMap<String, (UserIdentity, String)>>>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Store for EmptyStore {
|
||||
fn has_watcher(&self) -> bool {
|
||||
false
|
||||
}
|
||||
async fn regular_user_revision(&self, name: &str) -> rustfs_iam::error::Result<Option<(UserIdentity, String)>> {
|
||||
Ok(self.regular_users.read().expect("test store lock").get(name).cloned())
|
||||
}
|
||||
async fn save_iam_config<Item: Serialize + Send>(
|
||||
&self,
|
||||
_item: Item,
|
||||
@@ -496,7 +500,7 @@ mod tests {
|
||||
let (send_chan, _rx) = tokio::sync::mpsc::channel::<i64>(1);
|
||||
IamCache {
|
||||
cache: Cache::default(),
|
||||
api: EmptyStore,
|
||||
api: EmptyStore::default(),
|
||||
state: Arc::new(AtomicU8::new(0)),
|
||||
loading: Arc::new(AtomicBool::new(false)),
|
||||
roles: HashMap::new(),
|
||||
@@ -868,6 +872,12 @@ mod tests {
|
||||
store
|
||||
.cache
|
||||
.add_or_update_user(&user.access_key, &UserIdentity::new(user.clone()), now);
|
||||
store
|
||||
.api
|
||||
.regular_users
|
||||
.write()
|
||||
.expect("test store lock")
|
||||
.insert(user.access_key.clone(), (UserIdentity::new(user.clone()), "self-service".into()));
|
||||
store
|
||||
.cache
|
||||
.add_or_update_user_policy(&user.access_key, &MappedPolicy::new(""), now);
|
||||
|
||||
@@ -205,6 +205,12 @@ fn listen_notification_streams_close_after_identity_policy_or_bucket_revocation(
|
||||
)
|
||||
.await
|
||||
.expect("listener");
|
||||
let parent_credentials = rustfs_credentials::Credentials {
|
||||
access_key: key.clone(),
|
||||
secret_key: "listen-user-secret".into(),
|
||||
status: "on".into(),
|
||||
..Default::default()
|
||||
};
|
||||
let policy = serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow",
|
||||
"Action":["s3:ListenNotification","s3:ListenBucketNotification"],"Resource":"arn:aws:s3:::*"}]});
|
||||
iam.set_policy(
|
||||
@@ -227,7 +233,7 @@ fn listen_notification_streams_close_after_identity_policy_or_bucket_revocation(
|
||||
let mut cred = rustfs_policy::auth::get_new_credentials_with_metadata(&claims, "listen-root-secret")
|
||||
.expect("signed temporary credentials");
|
||||
cred.parent_user = key.clone();
|
||||
iam.set_temp_user(&cred.access_key, &cred, None)
|
||||
iam.set_temp_user_from_parent(&cred, &parent_credentials)
|
||||
.await
|
||||
.expect("temporary listener");
|
||||
Some(cred)
|
||||
|
||||
@@ -355,9 +355,16 @@ impl Operation for AddUser {
|
||||
args.policy = None;
|
||||
updated_at
|
||||
} else {
|
||||
if authorize_admin_envelope(req.envelope(), vec![Action::AdminAction(AdminAction::EnableUserAdminAction)])
|
||||
.await
|
||||
.is_ok()
|
||||
if validate_admin_request(
|
||||
req.headers(),
|
||||
&cred,
|
||||
owner,
|
||||
false,
|
||||
vec![Action::AdminAction(AdminAction::EnableUserAdminAction)],
|
||||
req.policy_source_addr(),
|
||||
)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
iam_store.create_user(ak, &args).await
|
||||
} else {
|
||||
|
||||
Reference in New Issue
Block a user