gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 13:16:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(obs): improve metrics coverage and dashboard performance (#2682)

Browse Source
This commit is contained in:
houseme
2026-04-26 02:51:29 +08:00
committed by GitHub
parent 81854762d4
commit 59f41eb86a
42 changed files with 1108 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+14
View File
@@ -2578,6 +2578,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
}
};
let has_tagging_replication = !put_opts.user_tags.is_empty();
if let Some(err) = if is_multipart {
drop(gr);
let result = replicate_object_with_multipart(MultipartReplicationContext {
@@ -2593,6 +2594,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
})
.await;
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
if has_tagging_replication {
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
}
result.err()
} else {
gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
@@ -2602,6 +2606,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
.await
.map_err(|e| std::io::Error::other(e.to_string()));
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
if has_tagging_replication {
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
}
result.err()
} {
rinfo.replication_status = ReplicationStatusType::Failed;
@@ -2867,6 +2874,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
}
};
let has_tagging_replication = !put_opts.user_tags.is_empty();
if let Some(err) = if is_multipart {
drop(gr);
let result = replicate_object_with_multipart(MultipartReplicationContext {
@@ -2882,6 +2890,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
})
.await;
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
if has_tagging_replication {
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
}
result.err()
} else {
gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
@@ -2891,6 +2902,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
.await
.map_err(|e| std::io::Error::other(e.to_string()));
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
if has_tagging_replication {
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
}
result.err()
} {
rinfo.replication_status = ReplicationStatusType::Failed;
crates/ecstore/src/bucket/replication/replication_state.rs
+30
View File
@@ -489,8 +489,14 @@ impl QueueCache {
pub struct ProxyMetric {
pub get_total: i64,
pub get_failed: i64,
pub get_tag_total: i64,
pub get_tag_failed: i64,
pub put_total: i64,
pub put_failed: i64,
pub put_tag_total: i64,
pub put_tag_failed: i64,
pub delete_tag_total: i64,
pub delete_tag_failed: i64,
pub head_total: i64,
pub head_failed: i64,
}
@@ -499,8 +505,14 @@ impl ProxyMetric {
pub fn add(&mut self, other: &ProxyMetric) {
self.get_total += other.get_total;
self.get_failed += other.get_failed;
self.get_tag_total += other.get_tag_total;
self.get_tag_failed += other.get_tag_failed;
self.put_total += other.put_total;
self.put_failed += other.put_failed;
self.put_tag_total += other.put_tag_total;
self.put_tag_failed += other.put_tag_failed;
self.delete_tag_total += other.delete_tag_total;
self.delete_tag_failed += other.delete_tag_failed;
self.head_total += other.head_total;
self.head_failed += other.head_failed;
}
@@ -527,18 +539,36 @@ impl ProxyStatsCache {
metric.get_failed += 1;
}
}
"GetObjectTagging" => {
metric.get_tag_total += 1;
if is_err {
metric.get_tag_failed += 1;
}
}
"PutObject" => {
metric.put_total += 1;
if is_err {
metric.put_failed += 1;
}
}
"PutObjectTagging" => {
metric.put_tag_total += 1;
if is_err {
metric.put_tag_failed += 1;
}
}
"HeadObject" => {
metric.head_total += 1;
if is_err {
metric.head_failed += 1;
}
}
"DeleteObjectTagging" => {
metric.delete_tag_total += 1;
if is_err {
metric.delete_tag_failed += 1;
}
}
_ => {}
}
}
crates/ecstore/src/erasure_coding/encode.rs
+36 -1
View File
@@ -26,6 +26,20 @@ use tokio::io::AsyncRead;
use tokio::sync::mpsc;
use tracing::error;
const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: usize = 32 * 1024 * 1024;
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 8;
fn encode_channel_capacity(expanded_block_bytes: usize, max_inflight_bytes: usize) -> usize {
if expanded_block_bytes == 0 {
return 1;
}
max_inflight_bytes
.saturating_div(expanded_block_bytes)
.clamp(1, DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS)
}
pub(crate) struct MultiWriter<'a> {
writers: &'a mut [Option<BitrotWriterWrapper>],
write_quorum: usize,
@@ -185,7 +199,14 @@ impl Erasure {
where
R: AsyncRead + Send + Sync + Unpin + 'static,
{
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
// Bound queued encoded blocks by memory budget to avoid per-request spikes.
let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
let max_inflight_bytes = rustfs_utils::get_env_usize(
ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
);
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(inflight_blocks);
let task = tokio::spawn(async move {
let block_size = self.block_size;
@@ -310,4 +331,18 @@ mod tests {
assert_eq!(written, b"small payload".len());
assert!(!committed.lock().unwrap().is_empty());
}
#[test]
fn encode_channel_capacity_never_returns_zero() {
assert_eq!(encode_channel_capacity(0, 1024), 1);
assert_eq!(encode_channel_capacity(4096, 0), 1);
assert_eq!(encode_channel_capacity(4096, 1024), 1);
}
#[test]
fn encode_channel_capacity_respects_budget_and_hard_cap() {
assert_eq!(encode_channel_capacity(4 * 1024 * 1024, 32 * 1024 * 1024), 8);
assert_eq!(encode_channel_capacity(16 * 1024 * 1024, 32 * 1024 * 1024), 2);
assert_eq!(encode_channel_capacity(1, usize::MAX), DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS);
}
}