gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-11 23:56:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(cluster): clarify peer health and listing timeouts (#5086)

Browse Source 
* fix(cluster): surface observed peer health

Refs rustfs/backlog#1387

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(admin): distinguish unreported peer health

Refs rustfs/backlog#1388

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ecstore): decouple metacache peek timeout

Refs rustfs/backlog#1389

Co-Authored-By: heihutu <heihutu@gmail.com>

* docs(ops): diagnose metacache listing timeouts

Refs rustfs/backlog#1390

Co-Authored-By: heihutu <heihutu@gmail.com>

* refactor(cluster): clarify observed peer health

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ecstore): route capability state through contract

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-21 19:11:24 +08:00
committed by GitHub
parent bb7bba3237
commit 7805cf5ae6
10 changed files with 349 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/log-analyzer/src/rules/seed/disk.rs
+17
View File
@@ -96,6 +96,23 @@ pub(super) fn rules() -> Vec<Rule> {
"定位具体盘(结合 samples 的 fields),检查硬件。",
)
},
Rule {
evidence_fields: strings(["bucket", "path", "state", "timeout_ms", "drive"]),
anchors: strings(["Metacache listing quorum failed", "Metacache reader peek timed out"]),
min_count: 2,
..base(
"metacache-listing-timeout",
P2Degraded,
"disk",
"metacache listing 多盘等待超时",
any([
all([contains("Metacache listing quorum failed"), field("state", "quorum_failed")]),
all([contains("Metacache reader peek timed out"), field("state", "peek_timed_out")]),
]),
"对象 listing/scanner 期间 metacache reader 等待超时并可能导致 listing quorum failure;常见于宽前缀、高延迟盘或慢远端 reader。",
"先确认是否存在宽前缀/慢盘/高延迟远端 reader;再参考 docs/operations/drive-timeout-tuning.md 调整 RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS、RUSTFS_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS 或 RUSTFS_DRIVE_TIMEOUT_PROFILE=high_latency。",
)
},
Rule {
anchors: strings(["Unformatted disk"]),
..base(
crates/log-analyzer/src/rules/seed/tests.rs
+26 -1
View File
@@ -73,7 +73,7 @@ fn msg(message: &'static str) -> Sample {
#[test]
fn seed_rule_set_is_valid_and_complete() {
let set = seed_rule_set();
assert_eq!(set.rules().len(), 68);
assert_eq!(set.rules().len(), 69);
// Every message-matching rule carries at least one anchor for the CI
// guard; the only anchor-less rules are the two structural matchers
// (panic kind, scanner target+level).
@@ -113,6 +113,14 @@ fn every_rule_has_a_positive_sample() {
msg("reporting peer disks offline after consecutive storage_info failures"),
),
("drive-faulty-error", msg("remote drive is faulty")),
(
"metacache-listing-timeout",
Sample {
message: "Metacache listing quorum failed",
fields: &[("state", "quorum_failed"), ("bucket", "photos"), ("path", "wide/")],
..Default::default()
},
),
("unformatted-disk", msg("Unformatted disk found")),
("disk-access-denied", msg("disk access denied: /data/disk1")),
("inconsistent-drive", msg("inconsistent drive found")),
@@ -308,6 +316,23 @@ fn smoke_samples_hit_exact_rule_sets() {
&["disk-marked-faulty"],
);
exact(&msg("erasure write quorum (required=8, achieved=5)"), &["ec-write-quorum"]);
exact(
&Sample {
message: "Metacache listing quorum failed",
fields: &[("state", "quorum_failed"), ("bucket", "photos"), ("path", "wide/")],
..Default::default()
},
&["metacache-listing-timeout"],
);
exact(
&Sample {
message: "S3Error InvalidAccessKeyId: Access Key Id you provided does not exist",
fields: &[("code", "InvalidAccessKeyId")],
..Default::default()
},
&["unknown-access-key"],
);
exact(&msg("HTTP transport failed: BrokenPipe while streaming response body"), &[]);
// Internode auth failures legitimately hit both the internode rule and
// the generic client signature rule.
exact(