Files
rustfs/crates/madmin/src/service_commands.rs
T
houseme 360bceafce feat(heal): add progress and trace observability (#6179)
* feat(heal): track erasure set progress baseline

Record erasure-set heal byte progress from per-object results and seed progress totals from complete usage-cache snapshots when available.

Keep usage-cache failures observational so heal execution continues without a baseline.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(heal): skip filtered erasure set versions

Skip erasure-set versions written after the durable heal start time, and queue lifecycle-expired versions for expiry before skipping them.

Track new-version and ILM-expired skips separately so progress can explain completed baseline work without treating these skips as retry-blocking failures.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(heal): wire abandoned data-dir cleanup check

Connect check_abandoned_parts through ECStore, pool, and set layers so heal can invoke the existing orphan data-dir reclaim path instead of returning NotImplemented.

Add dry-run support to the reclaim scan and cover dry-run plus scoped set behavior with regression tests.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): add heal scanner trace bus

Introduce an in-process broadcast trace bus with typed heal and scanner events, lazy event construction, and bounded lagged-subscriber behavior.

Cover zero-subscriber publishing, subscription delivery, drop accounting, and lagged receivers with focused common-crate tests.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): stream heal trace events from admin API

Wire the admin trace endpoint to the common trace bus for heal/scanner events, including kind, regex, and threshold filtering.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): emit heal trace events

Publish heal task lifecycle and abandoned-parts cleanup events through the common trace bus so the admin trace stream has live heal diagnostics.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): emit scanner trace events

Publish scanner folder, lifecycle action, and heal-candidate events through the common trace bus for live admin scanner diagnostics.

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(heal): route data usage loader through storage api

Keep ECStore data-usage facade access behind the heal storage_api boundary so architecture migration guards can validate the heal progress path.

Co-Authored-By: heihutu <heihutu@gmail.com>

* perf(heal): avoid lifecycle snapshots on ordinary heal pages

Only request lifecycle object snapshots when the heal pass has lifecycle expiry context. This keeps ordinary listing and disk-walk pages from cloning FileInfo/ObjectInfo payloads while preserving the skip path that queues expired versions.

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(heal): update bug-fix mocks for lifecycle snapshots

Carry the lifecycle snapshot opt-in argument through the remaining heal bug-fix test mocks so all-targets clippy covers the updated storage trait.

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(rustfs): sync heal storage mock signature

Update the rustfs storage RPC test mock for the lifecycle snapshot opt-in argument and cover it with rustfs all-targets clippy.

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(e2e): allocate smoke ports across nextest processes

Serialize E2E port selection with a small /tmp allocator so nextest workers do not reuse the same just-released ephemeral port before RustFS binds it.

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
2026-08-18 08:29:29 +08:00

147 lines
5.3 KiB
Rust

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{collections::HashMap, time::Duration};
use hyper::Uri;
use crate::{trace::TraceType, utils::parse_duration};
#[derive(Debug, Default)]
#[allow(dead_code)]
pub struct ServiceTraceOpts {
s3: bool,
internal: bool,
storage: bool,
os: bool,
scanner: bool,
decommission: bool,
healing: bool,
batch_replication: bool,
batch_key_rotation: bool,
batch_expire: bool,
batch_all: bool,
rebalance: bool,
replication_resync: bool,
bootstrap: bool,
ftp: bool,
ilm: bool,
only_errors: bool,
threshold: Duration,
}
#[allow(dead_code)]
impl ServiceTraceOpts {
pub fn trace_types(&self) -> TraceType {
let mut tt = TraceType::default();
tt.set_if(self.s3, &TraceType::S3);
tt.set_if(self.internal, &TraceType::INTERNAL);
tt.set_if(self.storage, &TraceType::STORAGE);
tt.set_if(self.os, &TraceType::OS);
tt.set_if(self.scanner, &TraceType::SCANNER);
tt.set_if(self.decommission, &TraceType::DECOMMISSION);
tt.set_if(self.healing, &TraceType::HEALING);
if self.batch_all {
tt.set_if(true, &TraceType::BATCH_REPLICATION);
tt.set_if(true, &TraceType::BATCH_KEY_ROTATION);
tt.set_if(true, &TraceType::BATCH_EXPIRE);
} else {
tt.set_if(self.batch_replication, &TraceType::BATCH_REPLICATION);
tt.set_if(self.batch_key_rotation, &TraceType::BATCH_KEY_ROTATION);
tt.set_if(self.batch_expire, &TraceType::BATCH_EXPIRE);
}
tt.set_if(self.rebalance, &TraceType::REBALANCE);
tt.set_if(self.replication_resync, &TraceType::REPLICATION_RESYNC);
tt.set_if(self.bootstrap, &TraceType::BOOTSTRAP);
tt.set_if(self.ftp, &TraceType::FTP);
tt.set_if(self.ilm, &TraceType::ILM);
tt
}
pub fn only_errors(&self) -> bool {
self.only_errors
}
pub fn threshold(&self) -> Duration {
self.threshold
}
pub fn parse_params(&mut self, uri: &Uri) -> Result<(), String> {
let query_pairs: HashMap<_, _> = uri
.query()
.unwrap_or("")
.split('&')
.filter_map(|pair| {
let mut split = pair.split('=');
let key = split.next()?.to_string();
let value = split.next().map(|v| v.to_string()).unwrap_or_else(|| "false".to_string());
Some((key, value))
})
.collect();
self.s3 = query_pairs.get("s3").is_some_and(|v| v == "true");
self.os = query_pairs.get("os").is_some_and(|v| v == "true");
self.scanner = query_pairs.get("scanner").is_some_and(|v| v == "true");
self.decommission = query_pairs.get("decommission").is_some_and(|v| v == "true");
self.healing = query_pairs.get("healing").is_some_and(|v| v == "true");
self.batch_replication = query_pairs.get("batch-replication").is_some_and(|v| v == "true");
self.batch_key_rotation = query_pairs.get("batch-keyrotation").is_some_and(|v| v == "true");
self.batch_expire = query_pairs.get("batch-expire").is_some_and(|v| v == "true");
self.rebalance = query_pairs.get("rebalance").is_some_and(|v| v == "true");
self.storage = query_pairs.get("storage").is_some_and(|v| v == "true");
self.internal = query_pairs.get("internal").is_some_and(|v| v == "true");
self.only_errors = query_pairs.get("err").is_some_and(|v| v == "true");
self.replication_resync = query_pairs.get("replication-resync").is_some_and(|v| v == "true");
self.bootstrap = query_pairs.get("bootstrap").is_some_and(|v| v == "true");
self.ftp = query_pairs.get("ftp").is_some_and(|v| v == "true");
self.ilm = query_pairs.get("ilm").is_some_and(|v| v == "true");
if query_pairs.get("all").is_some_and(|v| v == "true") {
self.s3 = true;
self.internal = true;
self.storage = true;
self.os = true;
}
if let Some(threshold) = query_pairs.get("threshold") {
let duration = parse_duration(threshold)?;
self.threshold = duration;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_params_all_true_keeps_internal_and_storage_enabled() {
let uri: Uri = "/trace?all=true".parse().expect("valid uri");
let mut opts = ServiceTraceOpts::default();
opts.parse_params(&uri).expect("all=true should parse");
let trace_types = opts.trace_types();
assert!(trace_types.contains(&TraceType::S3));
assert!(trace_types.contains(&TraceType::INTERNAL));
assert!(trace_types.contains(&TraceType::STORAGE));
assert!(trace_types.contains(&TraceType::OS));
}
}