gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 08:36:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): split internal get metadata metrics (#5687)

Browse Source 
Classify expected metadata-missing errors separately from unknown get pipeline failures and attribute internal meta-bucket reader failures to an internal_meta path instead of legacy_duplex.

This keeps scanner/data-usage metadata probes from polluting user GET/mixed failure attribution while preserving the existing read error behavior.

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-04 16:37:09 +08:00
committed by GitHub
parent cebc28f678
commit b71483b1c8
4 changed files with 114 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/mod.rs
+5 -4
View File
@@ -61,10 +61,11 @@ use crate::data_usage::record_compression_total_memory;
use crate::diagnostics::get::{
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY,
GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION,
GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE, GET_STAGE_LOCK_ACQUIRE,
GET_STAGE_METADATA, GET_STAGE_OBJECT_INFO, GET_STAGE_PATH_DECISION, GET_STAGE_READER_SETUP, classify_storage_error,
get_stage_timer_if_enabled, record_get_object_pipeline_failure, record_get_stage_duration_if_enabled,
GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX,
GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE,
GET_STAGE_LOCK_ACQUIRE, GET_STAGE_METADATA, GET_STAGE_OBJECT_INFO, GET_STAGE_PATH_DECISION, GET_STAGE_READER_SETUP,
classify_storage_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
};
use crate::disk::error_reduce::{
BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs,
crates/ecstore/src/set_disk/ops/object.rs
+67 -2
View File
@@ -383,7 +383,12 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
Ok((fi, files, disks)) => (fi, files, disks, None),
Err(err) => {
rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_stage_start.elapsed().as_secs_f64());
record_get_object_pipeline_failure(GET_STAGE_METADATA, classify_storage_error(&err));
let failure_path = if is_meta_bucketname(bucket) {
GET_OBJECT_PATH_INTERNAL_META
} else {
GET_OBJECT_PATH_LEGACY_DUPLEX
};
record_get_object_pipeline_failure_for_path(failure_path, GET_STAGE_METADATA, classify_storage_error(&err));
return Err(to_object_err(err, vec![bucket, object]));
}
}
@@ -5488,7 +5493,8 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
mod get_object_downstream_close_accounting_tests {
use super::hermetic_set_disks_support::hermetic_set_disks;
use super::*;
use crate::diagnostics::get::{GET_STAGE_DECODE, GET_STAGE_EMIT, GetObjectFailureReason};
use crate::diagnostics::get::{GET_OBJECT_PATH_INTERNAL_META, GET_STAGE_DECODE, GET_STAGE_EMIT, GetObjectFailureReason};
use crate::disk::RUSTFS_META_BUCKET;
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use crate::test_metrics::CapturingRecorder;
@@ -5581,6 +5587,65 @@ mod get_object_downstream_close_accounting_tests {
assert!(decode_failures > 0, "the producer must expose the downstream close at decode");
assert_eq!(emit_failures, 0, "downstream closure must not be counted as an emit failure");
}
#[test]
#[serial_test::serial]
fn missing_internal_meta_reader_records_internal_meta_path() {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("current-thread runtime should build");
let recorder = CapturingRecorder::default();
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
let (internal_missing, legacy_unknown) = metrics::with_local_recorder(&recorder, || {
runtime.block_on(async {
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
let options = ObjectOptions {
no_lock: true,
..Default::default()
};
let result = set_disks
.get_object_reader(
RUSTFS_META_BUCKET,
"buckets/.usage-cache/nonexistent.bin",
None,
HeaderMap::new(),
&options,
)
.await;
assert!(result.is_err(), "missing internal metadata must still return an error");
(
recorder.counter_value(
"rustfs_io_get_object_pipeline_failures_total",
&[
("path", GET_OBJECT_PATH_INTERNAL_META),
("stage", GET_STAGE_METADATA),
("reason", GetObjectFailureReason::MetadataMissing.as_str()),
],
),
recorder.counter_value(
"rustfs_io_get_object_pipeline_failures_total",
&[
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
("stage", GET_STAGE_METADATA),
("reason", GetObjectFailureReason::Unknown.as_str()),
],
),
)
})
});
rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate);
assert!(internal_missing > 0, "internal metadata miss must use the internal_meta path");
assert_eq!(
legacy_unknown, 0,
"internal metadata miss must not be attributed to legacy_duplex/unknown"
);
}
}
#[cfg(test)]