fix(log-analyzer): match structured heal writer failures (#8469)

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
Hauser
2026-10-11 20:46:44 +08:00
committed by GitHub
parent ba2047b184
commit 38263ba896
2 changed files with 60 additions and 5 deletions
+20 -5
View File
@@ -15,7 +15,7 @@
//! Self-heal rules.
use super::super::model::{Rule, Severity::*};
use super::{any, base, contains, prefix, strings};
use super::{all, any, base, contains, field, prefix, strings};
pub(super) fn rules() -> Vec<Rule> {
vec![
@@ -33,14 +33,29 @@ pub(super) fn rules() -> Vec<Rule> {
)
},
Rule {
anchors: strings(["all drives had write errors, unable to heal"]),
evidence_fields: strings([
"bucket",
"object",
"version_id",
"target_count",
"writer_failure_count",
"error",
]),
anchors: strings(["Set disk object heal writer failures"]),
..base(
"heal-all-writes-failed",
P0DataRisk,
"heal",
"heal 结果无法写入任何盘",
contains("all drives had write errors, unable to heal"),
"heal 结果无法写入任何盘(全盘故障/满盘)。",
"所有自愈目标不可写",
any([
contains("all drives had write errors, unable to heal"),
all([
contains("Set disk object heal writer failures"),
field("state", "writer_unavailable"),
field("result", "all_targets_unavailable"),
]),
]),
"所有自愈目标不可写,修复无法继续;健康源盘可能仍可读。",
"先解决容量/盘故障 finding,再重跑 heal。",
)
},
@@ -297,6 +297,46 @@ fn every_rule_has_a_positive_sample() {
}
}
#[test]
fn heal_writer_rule_requires_all_targets_and_retains_legacy_messages() {
let engine = engine();
for (sample, expected) in [
(
Sample {
message: "Set disk object heal writer failures",
level: Some(LogLevel::Warn),
fields: &[("state", "writer_unavailable"), ("result", "all_targets_unavailable")],
..Default::default()
},
true,
),
(
Sample {
message: "Set disk object heal writer failures",
level: Some(LogLevel::Warn),
fields: &[("state", "writer_unavailable"), ("result", "partial_targets_unavailable")],
..Default::default()
},
false,
),
(
Sample {
message: "Set disk object heal writer failures",
fields: &[("result", "all_targets_unavailable")],
..Default::default()
},
false,
),
(msg("all drives had write errors, unable to heal b/o"), true),
] {
let matched = engine
.matches(&event(&sample))
.iter()
.any(|&index| engine.rules()[index].id == "heal-all-writes-failed");
assert_eq!(matched, expected);
}
}
#[test]
fn smoke_samples_hit_exact_rule_sets() {
let engine = engine();