gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 21:46:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add timing and metrics for query execution phases (#2419)

Browse Source 
Signed-off-by: Shreyan Jain <shreyan11@duck.com>
Co-authored-by: Shreyan Jain <shreyan11@duck.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
2026-04-07 20:32:32 +08:00
committed by GitHub
parent 9cd1d6b4d5
commit ec8f059506
7 changed files with 309 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1
View File
@@ -8434,6 +8434,7 @@ dependencies = [
"futures", "futures",
"futures-core", "futures-core",
"http 1.4.0", "http 1.4.0",
"metrics",
"object_store", "object_store",
"parking_lot 0.12.5", "parking_lot 0.12.5",
"pin-project-lite", "pin-project-lite",
crates/common/src/metrics.rs
+222 -210
View File
@@ -22,12 +22,12 @@ use std::{
future::Future, future::Future,
pin::Pin, pin::Pin,
sync::{ sync::{
Arc, OnceLock, Arc, Mutex, OnceLock,
atomic::{AtomicU64, Ordering}, atomic::{AtomicU64, Ordering},
}, },
time::{Duration, SystemTime}, time::{Duration, SystemTime},
}; };
use tokio::sync::{Mutex, RwLock}; use tokio::sync::RwLock;
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IlmAction { pub enum IlmAction {
@@ -110,7 +110,7 @@ pub fn global_metrics() -> &'static Arc<Metrics> {
GLOBAL_METRICS.get_or_init(|| Arc::new(Metrics::new())) GLOBAL_METRICS.get_or_init(|| Arc::new(Metrics::new()))
} }
#[derive(Clone, Debug, PartialEq, PartialOrd)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Metric { pub enum Metric {
// START Realtime metrics, that only records // START Realtime metrics, that only records
// last minute latencies and total operation count. // last minute latencies and total operation count.
@@ -188,7 +188,6 @@ impl Metric {
if index >= Self::Last as usize { if index >= Self::Last as usize {
return None; return None;
} }
// Safe conversion using match instead of unsafe transmute
match index { match index {
0 => Some(Self::ReadMetadata), 0 => Some(Self::ReadMetadata),
1 => Some(Self::CheckMissing), 1 => Some(Self::CheckMissing),
@@ -220,16 +219,48 @@ impl Metric {
} }
} }
/// Thread-safe wrapper for LastMinuteLatency with atomic operations // ---------------------------------------------------------------------------
#[derive(Default)] // LockedLastMinuteLatency
// ---------------------------------------------------------------------------
//
// Uses std::sync::Mutex instead of tokio::sync::Mutex.
//
// Rationale: the critical section is a handful of integer additions inside
// LastMinuteLatency::add_all / get_total — no I/O, no blocking syscalls, no
// awaiting. A std blocking mutex is cheaper (no task-wakeup overhead) and,
// crucially, lets every caller be *synchronous*. That eliminates the need to
// spawn a background task just to record a duration, which was the only reason
// tokio::spawn appeared in log/time/time_size/time_n/time_ilm.
//
// Note: vec![LockedLastMinuteLatency::default(); N] with the old Arc-based
// Clone made every element share the *same* inner mutex — a latent bug where
// all metrics slots wrote to one counter. The new Clone creates a fresh
// independent Mutex per element, matching the intent.
/// Thread-safe wrapper for LastMinuteLatency backed by a std blocking mutex.
pub struct LockedLastMinuteLatency { pub struct LockedLastMinuteLatency {
// Arc so Clone is cheap *and* each cloned value stays independent (its own
// allocation). We never hand out the Arc to two Metrics at once; the Arc
// is purely a convenience for the Clone impl below.
latency: Arc<Mutex<LastMinuteLatency>>, latency: Arc<Mutex<LastMinuteLatency>>,
} }
impl Default for LockedLastMinuteLatency {
fn default() -> Self {
Self::new()
}
}
// Produce a fresh, independent slot — *not* a shared alias. This is what
// vec![val; N] and #[derive(Clone)] on the parent struct both need.
impl Clone for LockedLastMinuteLatency { impl Clone for LockedLastMinuteLatency {
fn clone(&self) -> Self { fn clone(&self) -> Self {
let inner = match self.latency.lock() {
Ok(guard) => guard.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
};
Self { Self {
latency: Arc::clone(&self.latency), latency: Arc::new(Mutex::new(inner)),
} }
} }
} }
@@ -241,14 +272,13 @@ impl LockedLastMinuteLatency {
} }
} }
/// Add a duration measurement /// Record a duration sample (no size).
pub async fn add(&self, duration: Duration) { pub fn add(&self, duration: Duration) {
self.add_size(duration, 0).await; self.add_size(duration, 0);
} }
/// Add a duration measurement with size /// Record a duration sample with an associated byte count.
pub async fn add_size(&self, duration: Duration, size: u64) { pub fn add_size(&self, duration: Duration, size: u64) {
let mut latency = self.latency.lock().await;
let now = SystemTime::now() let now = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH) .duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default() .unwrap_or_default()
@@ -259,17 +289,27 @@ impl LockedLastMinuteLatency {
total: duration.as_secs(), total: duration.as_secs(),
size, size,
}; };
latency.add_all(now, &elem);
match self.latency.lock() {
Ok(mut guard) => guard.add_all(now, &elem),
Err(poisoned) => poisoned.into_inner().add_all(now, &elem),
}
} }
/// Get total accumulated metrics for the last minute /// Return accumulated totals for the last minute window.
pub async fn total(&self) -> AccElem { pub fn total(&self) -> AccElem {
let mut latency = self.latency.lock().await; match self.latency.lock() {
latency.get_total() Ok(mut latency) => latency.get_total(),
Err(poisoned) => poisoned.into_inner().get_total(),
}
} }
} }
/// Current path tracker for monitoring active scan paths // ---------------------------------------------------------------------------
// CurrentPathTracker — unchanged, still uses tokio::sync::RwLock because it
// lives inside async path-update callbacks.
// ---------------------------------------------------------------------------
struct CurrentPathTracker { struct CurrentPathTracker {
current_path: Arc<RwLock<String>>, current_path: Arc<RwLock<String>>,
} }
@@ -290,17 +330,16 @@ impl CurrentPathTracker {
} }
} }
/// Main scanner metrics structure // ---------------------------------------------------------------------------
// Metrics
// ---------------------------------------------------------------------------
pub struct Metrics { pub struct Metrics {
// All fields must be accessed atomically and aligned.
operations: Vec<AtomicU64>, operations: Vec<AtomicU64>,
latency: Vec<LockedLastMinuteLatency>, latency: Vec<LockedLastMinuteLatency>,
actions: Vec<AtomicU64>, actions: Vec<AtomicU64>,
actions_latency: Vec<LockedLastMinuteLatency>, actions_latency: Vec<LockedLastMinuteLatency>,
// Current paths contains disk -> tracker mappings
current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>, current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
// Cycle information
cycle_info: Arc<RwLock<Option<CurrentCycle>>>, cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
} }
@@ -331,8 +370,6 @@ const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds"; const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds"; const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
/// Emit an OTEL counter increment for the given scanner metric.
/// ScanCycle and ScanBucketDrive are handled by dedicated emit functions with labels.
fn emit_otel_counter(metric: usize, count: u64) { fn emit_otel_counter(metric: usize, count: u64) {
match Metric::from_index(metric) { match Metric::from_index(metric) {
Some(Metric::ScanObject) => { Some(Metric::ScanObject) => {
@@ -345,8 +382,6 @@ fn emit_otel_counter(metric: usize, count: u64) {
} }
} }
/// Emit OTel metrics for a completed scan cycle.
/// Counter with result label + gauge for last successful cycle duration.
pub fn emit_scan_cycle_complete(success: bool, duration: Duration) { pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
let result = if success { "success" } else { "error" }; let result = if success { "success" } else { "error" };
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => result).increment(1); metrics::counter!(OTEL_SCANNER_CYCLES, "result" => result).increment(1);
@@ -355,8 +390,6 @@ pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
} }
} }
/// Emit OTel metrics for a completed bucket-drive scan.
/// Counter with result/bucket/disk labels + histogram for duration.
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) { pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
let result = if success { "success" } else { "error" }; let result = if success { "success" } else { "error" };
metrics::counter!( metrics::counter!(
@@ -376,226 +409,202 @@ pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str,
impl Metrics { impl Metrics {
pub fn new() -> Self { pub fn new() -> Self {
let operations = (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect();
let latency = (0..Metric::LastRealtime as usize)
.map(|_| LockedLastMinuteLatency::new())
.collect();
Self { Self {
operations, operations: (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
latency, // Each slot gets its own fresh LockedLastMinuteLatency so that
// different metrics never accidentally share state.
latency: (0..Metric::LastRealtime as usize)
.map(|_| LockedLastMinuteLatency::new())
.collect(),
actions: (0..IlmAction::ActionCount as usize).map(|_| AtomicU64::new(0)).collect(), actions: (0..IlmAction::ActionCount as usize).map(|_| AtomicU64::new(0)).collect(),
actions_latency: vec![LockedLastMinuteLatency::default(); IlmAction::ActionCount as usize], actions_latency: (0..IlmAction::ActionCount as usize)
.map(|_| LockedLastMinuteLatency::new())
.collect(),
current_paths: Arc::new(RwLock::new(HashMap::new())), current_paths: Arc::new(RwLock::new(HashMap::new())),
cycle_info: Arc::new(RwLock::new(None)), cycle_info: Arc::new(RwLock::new(None)),
} }
} }
/// Log scanner action with custom metadata - compatible with existing usage // -----------------------------------------------------------------------
// Metric recording helpers
//
// All of these are now pure sync closures. No tokio::spawn, no heap-
// allocated future, no scheduler overhead — just an atomic increment and a
// std-mutex lock for fewer than ~10 ns of work.
// -----------------------------------------------------------------------
/// Return a closure that records one observation of `metric` (with
/// optional caller-supplied metadata). Call it once the operation ends.
pub fn log(metric: Metric) -> impl Fn(&HashMap<String, String>) { pub fn log(metric: Metric) -> impl Fn(&HashMap<String, String>) {
let metric = metric as usize; let metric_idx = metric as usize;
let start_time = SystemTime::now(); let start = SystemTime::now();
move |_custom: &HashMap<String, String>| { move |_custom: &HashMap<String, String>| {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); let duration = SystemTime::now().duration_since(start).unwrap_or_default();
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
// Update operation count emit_otel_counter(metric_idx, 1);
global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed); if metric_idx < Metric::LastRealtime as usize {
emit_otel_counter(metric, 1); global_metrics().latency[metric_idx].add(duration);
// Update latency for realtime metrics (spawn async task for this)
if (metric) < Metric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
global_metrics().latency[metric_index].add(duration).await;
});
}
// Log trace metrics
if metric as u8 > Metric::StartTrace as u8 {
//debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric");
} }
} }
} }
/// Time scanner action with size - returns function that takes size /// Return a closure that records one observation of `metric` together with
/// a byte count. Call `done(size_bytes)` when the operation ends.
pub fn time_size(metric: Metric) -> impl Fn(u64) { pub fn time_size(metric: Metric) -> impl Fn(u64) {
let metric = metric as usize; let metric_idx = metric as usize;
let start_time = SystemTime::now(); let start = SystemTime::now();
move |size: u64| { move |size: u64| {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); let duration = SystemTime::now().duration_since(start).unwrap_or_default();
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
// Update operation count emit_otel_counter(metric_idx, 1);
global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed); if metric_idx < Metric::LastRealtime as usize {
emit_otel_counter(metric, 1); global_metrics().latency[metric_idx].add_size(duration, size);
// Update latency for realtime metrics with size (spawn async task)
if (metric) < Metric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
global_metrics().latency[metric_index].add_size(duration, size).await;
});
} }
} }
} }
/// Time a scanner action - returns a closure to call when done /// Return a closure that records one observation of `metric`.
/// Call `done()` when the operation ends.
pub fn time(metric: Metric) -> impl Fn() { pub fn time(metric: Metric) -> impl Fn() {
let metric = metric as usize; let metric_idx = metric as usize;
let start_time = SystemTime::now(); let start = SystemTime::now();
move || { move || {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); let duration = SystemTime::now().duration_since(start).unwrap_or_default();
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
// Update operation count emit_otel_counter(metric_idx, 1);
global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed); if metric_idx < Metric::LastRealtime as usize {
emit_otel_counter(metric, 1); global_metrics().latency[metric_idx].add(duration);
// Update latency for realtime metrics (spawn async task)
if (metric) < Metric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
global_metrics().latency[metric_index].add(duration).await;
});
} }
} }
} }
/// Time N scanner actions - returns function that takes count, then returns completion function /// Return a two-stage closure: first call takes an item count, second call
/// (returned closure) fires when the batch of `count` operations ends.
pub fn time_n(metric: Metric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> { pub fn time_n(metric: Metric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
let metric = metric as usize; let metric_idx = metric as usize;
let start_time = SystemTime::now(); let start = SystemTime::now();
Box::new(move |count: usize| { Box::new(move |count: usize| {
Box::new(move || { Box::new(move || {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); let duration = SystemTime::now().duration_since(start).unwrap_or_default();
global_metrics().operations[metric_idx].fetch_add(count as u64, Ordering::Relaxed);
// Update operation count emit_otel_counter(metric_idx, count as u64);
global_metrics().operations[metric].fetch_add(count as u64, Ordering::Relaxed); if metric_idx < Metric::LastRealtime as usize {
emit_otel_counter(metric, count as u64); global_metrics().latency[metric_idx].add(duration);
// Update latency for realtime metrics (spawn async task)
if (metric) < Metric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
global_metrics().latency[metric_index].add(duration).await;
});
} }
}) })
}) })
} }
/// Time ILM action with versions - returns function that takes versions, then returns completion function /// Return a two-stage closure for ILM actions: first call takes a version
/// count, second call fires when the action completes.
pub fn time_ilm(a: IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> { pub fn time_ilm(a: IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
let a_clone = a as usize; let a_idx = a as usize;
if a_clone == IlmAction::NoneAction as usize || a_clone >= IlmAction::ActionCount as usize { if a_idx == IlmAction::NoneAction as usize || a_idx >= IlmAction::ActionCount as usize {
return Box::new(move |_: u64| Box::new(move || {})); return Box::new(|_| Box::new(|| {}));
} }
let start = SystemTime::now(); let start = SystemTime::now();
Box::new(move |versions: u64| { Box::new(move |versions: u64| {
Box::new(move || { Box::new(move || {
let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0)); let duration = SystemTime::now().duration_since(start).unwrap_or_default();
tokio::spawn(async move { global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed);
global_metrics().actions[a_clone].fetch_add(versions, Ordering::Relaxed); global_metrics().actions_latency[a_idx].add(duration);
global_metrics().actions_latency[a_clone].add(duration).await;
});
}) })
}) })
} }
/// Increment time with specific duration /// Record a single observation of `metric` with a caller-supplied duration.
pub async fn inc_time(metric: Metric, duration: Duration) { /// No longer async — nothing inside requires it.
let metric = metric as usize; pub fn inc_time(metric: Metric, duration: Duration) {
// Update operation count let metric_idx = metric as usize;
global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed); global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
emit_otel_counter(metric, 1); emit_otel_counter(metric_idx, 1);
if metric_idx < Metric::LastRealtime as usize {
// Update latency for realtime metrics global_metrics().latency[metric_idx].add(duration);
if (metric) < Metric::LastRealtime as usize {
global_metrics().latency[metric].add(duration).await;
} }
} }
/// Get lifetime operation count for a metric // -----------------------------------------------------------------------
// Read-side helpers
// -----------------------------------------------------------------------
/// Lifetime operation count for `metric`.
pub fn lifetime(&self, metric: Metric) -> u64 { pub fn lifetime(&self, metric: Metric) -> u64 {
let metric = metric as usize; let idx = metric as usize;
if (metric) >= Metric::Last as usize { if idx >= Metric::Last as usize {
return 0; return 0;
} }
self.operations[metric].load(Ordering::Relaxed) self.operations[idx].load(Ordering::Relaxed)
} }
/// Get last minute statistics for a metric /// Last-minute accumulated stats for a realtime metric.
pub async fn last_minute(&self, metric: Metric) -> AccElem { /// No longer async — LockedLastMinuteLatency::total() is now synchronous.
let metric = metric as usize; pub fn last_minute(&self, metric: Metric) -> AccElem {
if (metric) >= Metric::LastRealtime as usize { let idx = metric as usize;
if idx >= Metric::LastRealtime as usize {
return AccElem::default(); return AccElem::default();
} }
self.latency[metric].total().await self.latency[idx].total()
} }
/// Set current cycle information /// Replace the current cycle record.
pub async fn set_cycle(&self, cycle: Option<CurrentCycle>) { pub async fn set_cycle(&self, cycle: Option<CurrentCycle>) {
*self.cycle_info.write().await = cycle; *self.cycle_info.write().await = cycle;
} }
/// Get current cycle information /// Read the current cycle record.
pub async fn get_cycle(&self) -> Option<CurrentCycle> { pub async fn get_cycle(&self) -> Option<CurrentCycle> {
self.cycle_info.read().await.clone() self.cycle_info.read().await.clone()
} }
/// Get current active paths /// Snapshot of every path currently being scanned.
pub async fn get_current_paths(&self) -> Vec<String> { pub async fn get_current_paths(&self) -> Vec<String> {
let mut result = Vec::new();
let paths = self.current_paths.read().await; let paths = self.current_paths.read().await;
let mut result = Vec::with_capacity(paths.len());
for (disk, tracker) in paths.iter() { for (disk, tracker) in paths.iter() {
let path = tracker.get_path().await; result.push(format!("{disk}/{}", tracker.get_path().await));
result.push(format!("{disk}/{path}"));
} }
result result
} }
/// Get number of active drives /// Number of drives with an active scan in progress.
pub async fn active_drives(&self) -> usize { pub async fn active_drives(&self) -> usize {
self.current_paths.read().await.len() self.current_paths.read().await.len()
} }
/// Generate metrics report /// Build a full metrics report snapshot.
pub async fn report(&self) -> M_ScannerMetrics { pub async fn report(&self) -> M_ScannerMetrics {
let mut metrics = M_ScannerMetrics::default(); let mut m = M_ScannerMetrics::default();
// Set cycle information
if let Some(cycle) = self.get_cycle().await { if let Some(cycle) = self.get_cycle().await {
metrics.current_cycle = cycle.current; m.current_cycle = cycle.current;
metrics.cycles_completed_at = cycle.cycle_completed; m.cycles_completed_at = cycle.cycle_completed;
metrics.current_started = cycle.started; m.current_started = cycle.started;
} }
// Replace default start time with global init time if it's the placeholder
if let Some(init_time) = crate::get_global_init_time().await { if let Some(init_time) = crate::get_global_init_time().await {
metrics.current_started = init_time; m.current_started = init_time;
} }
metrics.collected_at = Utc::now(); m.collected_at = Utc::now();
metrics.active_paths = self.get_current_paths().await; m.active_paths = self.get_current_paths().await;
// Lifetime operations // Lifetime operation counts
for i in 0..Metric::Last as usize { for i in 0..Metric::Last as usize {
let count = self.operations[i].load(Ordering::Relaxed); let count = self.operations[i].load(Ordering::Relaxed);
if count > 0 if count > 0
&& let Some(metric) = Metric::from_index(i) && let Some(metric) = Metric::from_index(i)
{ {
metrics.life_time_ops.insert(metric.as_str().to_string(), count); m.life_time_ops.insert(metric.as_str().to_string(), count);
} }
} }
// Last minute statistics for realtime metrics // Last-minute stats for realtime metrics — now plain sync calls
for i in 0..Metric::LastRealtime as usize { for i in 0..Metric::LastRealtime as usize {
let last_min = self.latency[i].total().await; let last_min = self.latency[i].total();
if last_min.n > 0 if last_min.n > 0
&& let Some(metric) = Metric::from_index(i) && let Some(metric) = Metric::from_index(i)
{ {
metrics.last_minute.actions.insert( m.last_minute.actions.insert(
metric.as_str().to_string(), metric.as_str().to_string(),
TimedAction { TimedAction {
count: last_min.n, count: last_min.n,
@@ -606,23 +615,23 @@ impl Metrics {
} }
} }
// Lifetime ILM operations // Lifetime ILM counts
for i in 0..IlmAction::ActionCount as usize { for i in 0..IlmAction::ActionCount as usize {
let count = self.actions[i].load(Ordering::Relaxed); let count = self.actions[i].load(Ordering::Relaxed);
if count > 0 if count > 0
&& let Some(action) = IlmAction::from_index(i) && let Some(action) = IlmAction::from_index(i)
{ {
metrics.life_time_ilm.insert(action.as_str().to_string(), count); m.life_time_ilm.insert(action.as_str().to_string(), count);
} }
} }
// Last minute ILM latency // Last-minute ILM latency — plain sync calls
for i in 0..IlmAction::ActionCount as usize { for i in 0..IlmAction::ActionCount as usize {
let last_min = self.actions_latency[i].total().await; let last_min = self.actions_latency[i].total();
if last_min.n > 0 if last_min.n > 0
&& let Some(action) = IlmAction::from_index(i) && let Some(action) = IlmAction::from_index(i)
{ {
metrics.last_minute.ilm.insert( m.last_minute.ilm.insert(
action.as_str().to_string(), action.as_str().to_string(),
TimedAction { TimedAction {
count: last_min.n, count: last_min.n,
@@ -633,56 +642,65 @@ impl Metrics {
} }
} }
metrics m
} }
} }
// Type aliases for compatibility with existing code
pub type UpdateCurrentPathFn = Arc<dyn Fn(&str) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
/// Create a current path updater for tracking scan progress
pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
let disk_name = disk.to_string();
// Store the tracker in global metrics
let tracker_clone = Arc::clone(&tracker);
let disk_clone = disk_name.clone();
tokio::spawn(async move {
global_metrics().current_paths.write().await.insert(disk_clone, tracker_clone);
});
let update_fn = {
let tracker = Arc::clone(&tracker);
Arc::new(move |path: &str| -> Pin<Box<dyn Future<Output = ()> + Send>> {
let tracker = Arc::clone(&tracker);
let path = path.to_string();
Box::pin(async move {
tracker.update_path(path).await;
})
})
};
let done_fn = {
let disk_name = disk_name.clone();
Arc::new(move || -> Pin<Box<dyn Future<Output = ()> + Send>> {
let disk_name = disk_name.clone();
Box::pin(async move {
global_metrics().current_paths.write().await.remove(&disk_name);
})
})
};
(update_fn, done_fn)
}
impl Default for Metrics { impl Default for Metrics {
fn default() -> Self { fn default() -> Self {
Self::new() Self::new()
} }
} }
// ---------------------------------------------------------------------------
// Path tracking helpers
// ---------------------------------------------------------------------------
pub type UpdateCurrentPathFn = Arc<dyn Fn(&str) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
/// Register a new disk in the global path tracker and return two callbacks:
/// one to update the current path and one to deregister the disk when done.
pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
let disk_name = disk.to_string();
let tracker_clone = Arc::clone(&tracker);
let disk_insert = disk_name.clone();
tokio::spawn(async move {
global_metrics()
.current_paths
.write()
.await
.insert(disk_insert, tracker_clone);
});
let update_fn: UpdateCurrentPathFn = {
let tracker = Arc::clone(&tracker);
Arc::new(move |path: &str| {
let tracker = Arc::clone(&tracker);
let path = path.to_string();
Box::pin(async move { tracker.update_path(path).await })
})
};
let done_fn: CloseDiskFn = {
let disk = disk_name.clone();
Arc::new(move || {
let disk = disk.clone();
Box::pin(async move {
global_metrics().current_paths.write().await.remove(&disk);
})
})
};
(update_fn, done_fn)
}
// ---------------------------------------------------------------------------
// CloseDiskGuard
// ---------------------------------------------------------------------------
pub struct CloseDiskGuard(CloseDiskFn); pub struct CloseDiskGuard(CloseDiskFn);
impl CloseDiskGuard { impl CloseDiskGuard {
@@ -697,16 +715,10 @@ impl CloseDiskGuard {
impl Drop for CloseDiskGuard { impl Drop for CloseDiskGuard {
fn drop(&mut self) { fn drop(&mut self) {
// Drop cannot be async, so we spawn the async cleanup task
// The task will run in the background and complete asynchronously
if let Ok(handle) = tokio::runtime::Handle::try_current() { if let Ok(handle) = tokio::runtime::Handle::try_current() {
let close_fn = self.0.clone(); let close_fn = self.0.clone();
handle.spawn(async move { handle.spawn(async move { close_fn().await });
close_fn().await;
});
} else {
// If we're not in a tokio runtime context, we can't spawn
// This is a best-effort cleanup, so we just skip it
} }
// If there is no runtime we are in a test or shutdown path; skip cleanup.
} }
} }
crates/ecstore/src/bucket/quota/checker.rs
+4 -3
View File
@@ -107,9 +107,10 @@ impl QuotaChecker {
}; };
let duration = start_time.elapsed(); let duration = start_time.elapsed();
rustfs_common::metrics::Metrics::inc_time(Metric::QuotaCheck, duration).await; // inc_time is now a plain fn (not async) — no .await needed.
rustfs_common::metrics::Metrics::inc_time(Metric::QuotaCheck, duration);
if !allowed { if !allowed {
rustfs_common::metrics::Metrics::inc_time(Metric::QuotaViolation, duration).await; rustfs_common::metrics::Metrics::inc_time(Metric::QuotaViolation, duration);
} }
Ok(result) Ok(result)
@@ -146,7 +147,7 @@ impl QuotaChecker {
.await .await
.map_err(QuotaError::StorageError)?; .map_err(QuotaError::StorageError)?;
rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed()).await; rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
Ok(()) Ok(())
} }
crates/kms/src/api_types.rs
+2 -7
View File
@@ -122,11 +122,7 @@ pub enum ConfigureKmsRequest {
)] )]
VaultKv2(ConfigureVaultKmsRequest), VaultKv2(ConfigureVaultKmsRequest),
/// Configure with Vault Transit backend /// Configure with Vault Transit backend
#[serde( #[serde(rename = "VaultTransit", alias = "vault-transit", alias = "vault_transit")]
rename = "VaultTransit",
alias = "vault-transit",
alias = "vault_transit"
)]
VaultTransit(ConfigureVaultTransitKmsRequest), VaultTransit(ConfigureVaultTransitKmsRequest),
} }
@@ -472,8 +468,7 @@ mod tests {
"mount_path": "transit", "mount_path": "transit",
"default_key_id": "rustfs-master-key" "default_key_id": "rustfs-master-key"
}); });
let request: ConfigureKmsRequest = let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("vault-transit request should deserialize");
serde_json::from_value(raw).expect("vault-transit request should deserialize");
let config = request.to_kms_config(); let config = request.to_kms_config();
assert_eq!(config.backend, KmsBackend::VaultTransit); assert_eq!(config.backend, KmsBackend::VaultTransit);
let vault = config.vault_transit_config().expect("vault-transit config should be present"); let vault = config.vault_transit_config().expect("vault-transit config should be present");
crates/s3select-api/Cargo.toml
+1
View File
@@ -26,6 +26,7 @@ categories = ["web-programming", "development-tools", "asynchronous"]
documentation = "https://docs.rs/rustfs-s3select-api/latest/rustfs_s3select_api/" documentation = "https://docs.rs/rustfs-s3select-api/latest/rustfs_s3select_api/"
[dependencies] [dependencies]
metrics = { workspace = true }
async-trait.workspace = true async-trait.workspace = true
bytes.workspace = true bytes.workspace = true
chrono.workspace = true chrono.workspace = true
crates/s3select-api/src/query/execution.rs
+59 -15
View File
@@ -18,13 +18,13 @@ use std::sync::Arc;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use parking_lot::RwLock;
use async_trait::async_trait; use async_trait::async_trait;
use datafusion::arrow::datatypes::{Schema, SchemaRef}; use datafusion::arrow::datatypes::{Schema, SchemaRef};
use datafusion::arrow::record_batch::RecordBatch; use datafusion::arrow::record_batch::RecordBatch;
use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::physical_plan::SendableRecordBatchStream;
use futures::{Stream, StreamExt, TryStreamExt}; use futures::{Stream, StreamExt, TryStreamExt};
use parking_lot::RwLock;
use tracing::debug;
use crate::{QueryError, QueryResult}; use crate::{QueryError, QueryResult};
@@ -32,6 +32,36 @@ use super::Query;
use super::logical_planner::Plan; use super::logical_planner::Plan;
use super::session::SessionCtx; use super::session::SessionCtx;
pub struct PhaseTimer {
phase_name: &'static str,
start_time: Instant,
}
impl PhaseTimer {
pub fn new(phase_name: &'static str) -> Self {
Self {
phase_name,
start_time: Instant::now(),
}
}
}
impl Drop for PhaseTimer {
fn drop(&mut self) {
let duration = self.start_time.elapsed();
if !std::thread::panicking() {
metrics::histogram!(
"rustfs.s3select.phase.duration.seconds",
"phase" => self.phase_name
)
.record(duration.as_secs_f64());
}
debug!("Phase '{}' took {:?}", self.phase_name, duration);
}
}
pub type QueryExecutionRef = Arc<dyn QueryExecution>; pub type QueryExecutionRef = Arc<dyn QueryExecution>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -94,13 +124,13 @@ impl Output {
} }
} }
/// Returns the number of records affected by the query operation /// Returns the number of records affected by the query operation
/// ///
/// If it is a select statement, returns the number of rows in the result set /// If it is a select statement, returns the number of rows in the result set
/// ///
/// -1 means unknown /// -1 means unknown
/// ///
/// panic! when StreamData's number of records greater than i64::Max /// panic! when StreamData's number of records greater than i64::Max
pub async fn affected_rows(self) -> i64 { pub async fn affected_rows(self) -> i64 {
self.num_rows().await as i64 self.num_rows().await as i64
} }
@@ -138,6 +168,20 @@ pub struct QueryStateMachine {
} }
impl QueryStateMachine { impl QueryStateMachine {
fn record_phase_timestamp(&self, phase: &'static str, event: &'static str) {
let elapsed = self.start.elapsed();
metrics::histogram!(
"rustfs.s3select.phase.timestamp.seconds",
"phase" => phase,
"event" => event
)
.record(elapsed.as_secs_f64());
}
#[must_use]
pub fn time_phase(&self, phase_name: &'static str) -> PhaseTimer {
PhaseTimer::new(phase_name)
}
pub fn begin(query: Query, session: SessionCtx) -> Self { pub fn begin(query: Query, session: SessionCtx) -> Self {
Self { Self {
session, session,
@@ -148,30 +192,30 @@ impl QueryStateMachine {
} }
pub fn begin_analyze(&self) { pub fn begin_analyze(&self) {
// TODO record time self.record_phase_timestamp("analyze", "start");
self.translate_to(QueryState::RUNNING(RUNNING::ANALYZING)); self.translate_to(QueryState::RUNNING(RUNNING::ANALYZING));
} }
pub fn end_analyze(&self) { pub fn end_analyze(&self) {
// TODO record time self.record_phase_timestamp("analyze", "end");
} }
pub fn begin_optimize(&self) { pub fn begin_optimize(&self) {
// TODO record time self.record_phase_timestamp("optimize", "start");
self.translate_to(QueryState::RUNNING(RUNNING::OPTIMIZING)); self.translate_to(QueryState::RUNNING(RUNNING::OPTIMIZING));
} }
pub fn end_optimize(&self) { pub fn end_optimize(&self) {
// TODO self.record_phase_timestamp("optimize", "end");
} }
pub fn begin_schedule(&self) { pub fn begin_schedule(&self) {
// TODO self.record_phase_timestamp("schedule", "start");
self.translate_to(QueryState::RUNNING(RUNNING::SCHEDULING)); self.translate_to(QueryState::RUNNING(RUNNING::SCHEDULING));
} }
pub fn end_schedule(&self) { pub fn end_schedule(&self) {
// TODO self.record_phase_timestamp("schedule", "end");
} }
pub fn finish(&self) { pub fn finish(&self) {
crates/s3select-query/src/execution/query.rs
+20 -14
View File
@@ -50,21 +50,27 @@ impl SqlQueryExecution {
} }
async fn start(&self) -> QueryResult<Output> { async fn start(&self) -> QueryResult<Output> {
// begin optimize let physical_plan = {
self.query_state_machine.begin_optimize(); // Time optimize phase - dropped at end of this block
let physical_plan = self.optimizer.optimize(&self.plan, &self.query_state_machine.session).await?; let _optimize_timer = self.query_state_machine.time_phase("optimize");
self.query_state_machine.end_optimize(); self.query_state_machine.begin_optimize();
let plan = self.optimizer.optimize(&self.plan, &self.query_state_machine.session).await?;
self.query_state_machine.end_optimize();
plan
};
// begin schedule let stream = {
self.query_state_machine.begin_schedule(); // Time schedule phase - dropped at end of this block
let stream = self let _schedule_timer = self.query_state_machine.time_phase("schedule");
.scheduler self.query_state_machine.begin_schedule();
.schedule(physical_plan.clone(), self.query_state_machine.session.inner().task_ctx()) let stream = self
.await? .scheduler
.stream(); .schedule(physical_plan.clone(), self.query_state_machine.session.inner().task_ctx())
.await?
debug!("Success build result stream."); .stream();
self.query_state_machine.end_schedule(); self.query_state_machine.end_schedule();
stream
};
Ok(Output::StreamData(stream)) Ok(Output::StreamData(stream))
} }