mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
refactor(capacity): optimize capacity management module (#2325)
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
@@ -32,73 +32,239 @@ use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
// ============================================================================
|
||||
// Configuration Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Cached capacity configuration to avoid repeated environment variable reads
|
||||
#[derive(Clone, Debug)]
|
||||
struct CachedCapacityConfig {
|
||||
/// Scheduled update interval
|
||||
scheduled_update_interval: Duration,
|
||||
/// Write trigger delay
|
||||
write_trigger_delay: Duration,
|
||||
/// Write frequency threshold
|
||||
write_frequency_threshold: usize,
|
||||
/// Fast update threshold
|
||||
fast_update_threshold: Duration,
|
||||
/// Max files threshold for sampling
|
||||
max_files_threshold: usize,
|
||||
/// Stat timeout
|
||||
stat_timeout: Duration,
|
||||
/// Sample rate
|
||||
sample_rate: usize,
|
||||
/// Follow symlinks flag
|
||||
follow_symlinks: bool,
|
||||
/// Max symlink depth
|
||||
max_symlink_depth: u8,
|
||||
/// Enable dynamic timeout flag
|
||||
enable_dynamic_timeout: bool,
|
||||
/// Min timeout
|
||||
min_timeout: Duration,
|
||||
/// Max timeout
|
||||
max_timeout: Duration,
|
||||
/// Stall timeout
|
||||
stall_timeout: Duration,
|
||||
}
|
||||
|
||||
impl CachedCapacityConfig {
|
||||
/// Build configuration from environment variables
|
||||
fn from_env() -> Self {
|
||||
Self {
|
||||
scheduled_update_interval: Duration::from_secs(get_env_u64(
|
||||
ENV_CAPACITY_SCHEDULED_INTERVAL,
|
||||
DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS,
|
||||
)),
|
||||
write_trigger_delay: Duration::from_secs(get_env_u64(
|
||||
ENV_CAPACITY_WRITE_TRIGGER_DELAY,
|
||||
DEFAULT_WRITE_TRIGGER_DELAY_SECS,
|
||||
)),
|
||||
write_frequency_threshold: get_env_usize(ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, DEFAULT_WRITE_FREQUENCY_THRESHOLD),
|
||||
fast_update_threshold: Duration::from_secs(get_env_u64(
|
||||
ENV_CAPACITY_FAST_UPDATE_THRESHOLD,
|
||||
DEFAULT_FAST_UPDATE_THRESHOLD_SECS,
|
||||
)),
|
||||
max_files_threshold: get_env_usize(ENV_CAPACITY_MAX_FILES_THRESHOLD, DEFAULT_MAX_FILES_THRESHOLD),
|
||||
stat_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_STAT_TIMEOUT, DEFAULT_STAT_TIMEOUT_SECS)),
|
||||
sample_rate: get_env_usize(ENV_CAPACITY_SAMPLE_RATE, DEFAULT_SAMPLE_RATE),
|
||||
follow_symlinks: get_env_bool(ENV_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_FOLLOW_SYMLINKS),
|
||||
max_symlink_depth: get_env_u64(ENV_CAPACITY_MAX_SYMLINK_DEPTH, DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH as u64) as u8,
|
||||
enable_dynamic_timeout: get_env_bool(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT),
|
||||
min_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_MIN_TIMEOUT, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS)),
|
||||
max_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_MAX_TIMEOUT, DEFAULT_CAPACITY_MAX_TIMEOUT_SECS)),
|
||||
stall_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_STALL_TIMEOUT, DEFAULT_CAPACITY_STALL_TIMEOUT_SECS)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get cached capacity configuration (reads environment variables once)
|
||||
#[cfg(not(test))]
|
||||
fn get_cached_config() -> &'static CachedCapacityConfig {
|
||||
static CONFIG: std::sync::OnceLock<CachedCapacityConfig> = std::sync::OnceLock::new();
|
||||
CONFIG.get_or_init(CachedCapacityConfig::from_env)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn get_cached_config() -> CachedCapacityConfig {
|
||||
// Don't cache in tests to allow temp_env::with_var to work
|
||||
CachedCapacityConfig::from_env()
|
||||
}
|
||||
|
||||
/// Get scheduled update interval from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_scheduled_update_interval() -> Duration {
|
||||
Duration::from_secs(get_env_u64(ENV_CAPACITY_SCHEDULED_INTERVAL, DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS))
|
||||
get_cached_config().scheduled_update_interval
|
||||
}
|
||||
|
||||
/// Get scheduled update interval from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_scheduled_update_interval() -> Duration {
|
||||
get_cached_config().scheduled_update_interval
|
||||
}
|
||||
|
||||
/// Get write trigger delay from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_write_trigger_delay() -> Duration {
|
||||
Duration::from_secs(get_env_u64(ENV_CAPACITY_WRITE_TRIGGER_DELAY, DEFAULT_WRITE_TRIGGER_DELAY_SECS))
|
||||
get_cached_config().write_trigger_delay
|
||||
}
|
||||
|
||||
/// Get write trigger delay from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_write_trigger_delay() -> Duration {
|
||||
get_cached_config().write_trigger_delay
|
||||
}
|
||||
|
||||
/// Get write frequency threshold from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_write_frequency_threshold() -> usize {
|
||||
get_env_usize(ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, DEFAULT_WRITE_FREQUENCY_THRESHOLD)
|
||||
get_cached_config().write_frequency_threshold
|
||||
}
|
||||
|
||||
/// Get write frequency threshold from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_write_frequency_threshold() -> usize {
|
||||
get_cached_config().write_frequency_threshold
|
||||
}
|
||||
|
||||
/// Get fast update threshold from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_fast_update_threshold() -> Duration {
|
||||
Duration::from_secs(get_env_u64(ENV_CAPACITY_FAST_UPDATE_THRESHOLD, DEFAULT_FAST_UPDATE_THRESHOLD_SECS))
|
||||
get_cached_config().fast_update_threshold
|
||||
}
|
||||
|
||||
/// Get fast update threshold from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_fast_update_threshold() -> Duration {
|
||||
get_cached_config().fast_update_threshold
|
||||
}
|
||||
|
||||
/// Get max files threshold from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_max_files_threshold() -> usize {
|
||||
get_env_usize(ENV_CAPACITY_MAX_FILES_THRESHOLD, DEFAULT_MAX_FILES_THRESHOLD)
|
||||
get_cached_config().max_files_threshold
|
||||
}
|
||||
|
||||
/// Get max files threshold from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_max_files_threshold() -> usize {
|
||||
get_cached_config().max_files_threshold
|
||||
}
|
||||
|
||||
/// Get stat timeout from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_stat_timeout() -> Duration {
|
||||
Duration::from_secs(get_env_u64(ENV_CAPACITY_STAT_TIMEOUT, DEFAULT_STAT_TIMEOUT_SECS))
|
||||
get_cached_config().stat_timeout
|
||||
}
|
||||
|
||||
/// Get stat timeout from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_stat_timeout() -> Duration {
|
||||
get_cached_config().stat_timeout
|
||||
}
|
||||
|
||||
/// Get sample rate from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_sample_rate() -> usize {
|
||||
get_env_usize(ENV_CAPACITY_SAMPLE_RATE, DEFAULT_SAMPLE_RATE)
|
||||
get_cached_config().sample_rate
|
||||
}
|
||||
|
||||
/// Get sample rate from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_sample_rate() -> usize {
|
||||
get_cached_config().sample_rate
|
||||
}
|
||||
|
||||
/// Get follow symlinks flag from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_follow_symlinks() -> bool {
|
||||
get_env_bool(ENV_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_FOLLOW_SYMLINKS)
|
||||
get_cached_config().follow_symlinks
|
||||
}
|
||||
|
||||
/// Get follow symlinks flag from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_follow_symlinks() -> bool {
|
||||
get_cached_config().follow_symlinks
|
||||
}
|
||||
|
||||
/// Get max symlink depth from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_max_symlink_depth() -> u8 {
|
||||
get_env_u64(ENV_CAPACITY_MAX_SYMLINK_DEPTH, DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH as u64) as u8
|
||||
get_cached_config().max_symlink_depth
|
||||
}
|
||||
|
||||
/// Get max symlink depth from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_max_symlink_depth() -> u8 {
|
||||
get_cached_config().max_symlink_depth
|
||||
}
|
||||
|
||||
/// Get enable dynamic timeout flag from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_enable_dynamic_timeout() -> bool {
|
||||
get_env_bool(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT)
|
||||
get_cached_config().enable_dynamic_timeout
|
||||
}
|
||||
|
||||
/// Get enable dynamic timeout flag from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_enable_dynamic_timeout() -> bool {
|
||||
get_cached_config().enable_dynamic_timeout
|
||||
}
|
||||
|
||||
/// Get min timeout from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_min_timeout() -> Duration {
|
||||
Duration::from_secs(get_env_u64(ENV_CAPACITY_MIN_TIMEOUT, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS))
|
||||
get_cached_config().min_timeout
|
||||
}
|
||||
|
||||
/// Get min timeout from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_min_timeout() -> Duration {
|
||||
get_cached_config().min_timeout
|
||||
}
|
||||
|
||||
/// Get max timeout from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_max_timeout() -> Duration {
|
||||
Duration::from_secs(get_env_u64(ENV_CAPACITY_MAX_TIMEOUT, DEFAULT_CAPACITY_MAX_TIMEOUT_SECS))
|
||||
get_cached_config().max_timeout
|
||||
}
|
||||
|
||||
/// Get max timeout from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_max_timeout() -> Duration {
|
||||
get_cached_config().max_timeout
|
||||
}
|
||||
|
||||
/// Get stall timeout from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_stall_timeout() -> Duration {
|
||||
Duration::from_secs(get_env_u64(ENV_CAPACITY_STALL_TIMEOUT, DEFAULT_CAPACITY_STALL_TIMEOUT_SECS))
|
||||
get_cached_config().stall_timeout
|
||||
}
|
||||
|
||||
/// Get stall timeout from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_stall_timeout() -> Duration {
|
||||
get_cached_config().stall_timeout
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -107,22 +273,22 @@ pub fn get_stall_timeout() -> Duration {
|
||||
|
||||
/// Cached capacity data
|
||||
#[derive(Clone, Debug)]
|
||||
#[allow(dead_code)]
|
||||
pub struct CachedCapacity {
|
||||
/// Total used capacity in bytes
|
||||
pub total_used: u64,
|
||||
/// Last update time
|
||||
pub last_update: Instant,
|
||||
/// File count (optional)
|
||||
#[allow(dead_code)]
|
||||
pub file_count: usize,
|
||||
/// Whether it's an estimated value
|
||||
#[allow(dead_code)]
|
||||
pub is_estimated: bool,
|
||||
/// Data source
|
||||
pub source: DataSource,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Copy, Eq)]
|
||||
#[allow(dead_code)]
|
||||
pub enum DataSource {
|
||||
/// Real-time statistics
|
||||
RealTime,
|
||||
@@ -131,6 +297,7 @@ pub enum DataSource {
|
||||
/// Write triggered
|
||||
WriteTriggered,
|
||||
/// Fallback value
|
||||
#[allow(dead_code)]
|
||||
Fallback,
|
||||
}
|
||||
|
||||
@@ -342,15 +509,31 @@ impl HybridCapacityManager {
|
||||
}
|
||||
|
||||
/// Global capacity manager instance
|
||||
static CAPACITY_MANAGER: std::sync::OnceLock<Arc<HybridCapacityManager>> = std::sync::OnceLock::new();
|
||||
static GLOBAL_CAPACITY_MANAGER: std::sync::OnceLock<Arc<HybridCapacityManager>> = std::sync::OnceLock::new();
|
||||
|
||||
/// Get or initialize the global capacity manager
|
||||
pub fn get_capacity_manager() -> Arc<HybridCapacityManager> {
|
||||
CAPACITY_MANAGER
|
||||
GLOBAL_CAPACITY_MANAGER
|
||||
.get_or_init(|| Arc::new(HybridCapacityManager::from_env()))
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Create an isolated capacity manager instance for testing
|
||||
///
|
||||
/// This factory function allows tests to create independent instances
|
||||
/// without affecting the global singleton, avoiding test pollution.
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
/// manager.update_capacity(1000, DataSource::RealTime).await;
|
||||
/// ```
|
||||
#[cfg(test)]
|
||||
#[allow(dead_code)]
|
||||
pub fn create_isolated_manager(config: HybridStrategyConfig) -> Arc<HybridCapacityManager> {
|
||||
Arc::new(HybridCapacityManager::new(config))
|
||||
}
|
||||
|
||||
/// Start background update task
|
||||
pub async fn start_background_task(disks: Vec<rustfs_madmin::Disk>) {
|
||||
let manager = get_capacity_manager();
|
||||
|
||||
@@ -20,6 +20,88 @@ use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
use tracing::info;
|
||||
|
||||
// ============================================================================
|
||||
// Metric Name Constants (following existing naming convention)
|
||||
// ============================================================================
|
||||
|
||||
/// Cache hit counter
|
||||
const CAPACITY_CACHE_HIT: &str = "rustfs.capacity.cache.hits";
|
||||
|
||||
/// Cache miss counter
|
||||
const CAPACITY_CACHE_MISS: &str = "rustfs.capacity.cache.misses";
|
||||
|
||||
/// Cache hit rate gauge
|
||||
const CAPACITY_CACHE_HIT_RATE: &str = "rustfs.capacity.cache.hit_rate";
|
||||
|
||||
/// Cache hits total gauge
|
||||
const CAPACITY_CACHE_HITS_TOTAL: &str = "rustfs.capacity.cache.hits_total";
|
||||
|
||||
/// Cache misses total gauge
|
||||
const CAPACITY_CACHE_MISSES_TOTAL: &str = "rustfs.capacity.cache.misses_total";
|
||||
|
||||
/// Scheduled update counter
|
||||
const CAPACITY_UPDATE_SCHEDULED: &str = "rustfs.capacity.update.scheduled";
|
||||
|
||||
/// Write-triggered update counter
|
||||
const CAPACITY_UPDATE_WRITE_TRIGGERED: &str = "rustfs.capacity.update.write_triggered";
|
||||
|
||||
/// Update failure counter
|
||||
const CAPACITY_UPDATE_FAILURES: &str = "rustfs.capacity.update.failures";
|
||||
|
||||
/// Current capacity in bytes gauge
|
||||
#[allow(dead_code)]
|
||||
const CAPACITY_CURRENT_BYTES: &str = "rustfs.capacity.current";
|
||||
|
||||
/// Write operations counter
|
||||
const CAPACITY_WRITE_OPERATIONS: &str = "rustfs.capacity.write.operations";
|
||||
|
||||
/// Write frequency gauge
|
||||
#[allow(dead_code)]
|
||||
const CAPACITY_WRITE_FREQUENCY: &str = "rustfs.capacity.write.frequency";
|
||||
|
||||
/// Update duration in microseconds histogram
|
||||
const CAPACITY_UPDATE_DURATION_US: &str = "rustfs.capacity.update.duration_us";
|
||||
|
||||
/// Scheduled updates total gauge
|
||||
const CAPACITY_UPDATE_SCHEDULED_TOTAL: &str = "rustfs.capacity.update.scheduled_total";
|
||||
|
||||
/// Write-triggered updates total gauge
|
||||
const CAPACITY_UPDATE_WRITE_TRIGGERED_TOTAL: &str = "rustfs.capacity.update.write_triggered_total";
|
||||
|
||||
/// Update failures total gauge
|
||||
const CAPACITY_UPDATE_FAILURES_TOTAL: &str = "rustfs.capacity.update.failures_total";
|
||||
|
||||
/// Symlinks encountered counter
|
||||
const CAPACITY_SYMLINKS_ENCOUNTERED: &str = "rustfs.capacity.symlinks.encountered";
|
||||
|
||||
/// Symlinks total size gauge
|
||||
const CAPACITY_SYMLINKS_SIZE: &str = "rustfs.capacity.symlinks.total_size";
|
||||
|
||||
/// Symlinks count gauge
|
||||
const CAPACITY_SYMLINKS_COUNT: &str = "rustfs.capacity.symlinks.count";
|
||||
|
||||
/// Dynamic timeout counter
|
||||
const CAPACITY_TIMEOUT_DYNAMIC: &str = "rustfs.capacity.timeout.dynamic";
|
||||
|
||||
/// Timeout fallback counter
|
||||
const CAPACITY_TIMEOUT_FALLBACK: &str = "rustfs.capacity.timeout.fallback";
|
||||
|
||||
/// Stall detected counter
|
||||
const CAPACITY_TIMEOUT_STALL: &str = "rustfs.capacity.timeout.stall";
|
||||
|
||||
/// Dynamic timeout total gauge
|
||||
const CAPACITY_TIMEOUT_DYNAMIC_TOTAL: &str = "rustfs.capacity.timeout.dynamic_total";
|
||||
|
||||
/// Timeout fallback total gauge
|
||||
const CAPACITY_TIMEOUT_FALLBACK_TOTAL: &str = "rustfs.capacity.timeout.fallback_total";
|
||||
|
||||
/// Stall detected total gauge
|
||||
const CAPACITY_TIMEOUT_STALL_TOTAL: &str = "rustfs.capacity.timeout.stall_total";
|
||||
|
||||
// ============================================================================
|
||||
// Capacity Metrics
|
||||
// ============================================================================
|
||||
|
||||
/// Capacity metrics for monitoring
|
||||
#[derive(Debug, Default)]
|
||||
pub struct CapacityMetrics {
|
||||
@@ -58,66 +140,66 @@ impl CapacityMetrics {
|
||||
/// Record cache hit
|
||||
pub fn record_cache_hit(&self) {
|
||||
self.cache_hits.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs.capacity.cache.hits").increment(1);
|
||||
counter!(CAPACITY_CACHE_HIT).increment(1);
|
||||
}
|
||||
|
||||
/// Record cache miss
|
||||
pub fn record_cache_miss(&self) {
|
||||
self.cache_misses.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs.capacity.cache.misses").increment(1);
|
||||
counter!(CAPACITY_CACHE_MISS).increment(1);
|
||||
}
|
||||
|
||||
/// Record scheduled update
|
||||
#[allow(dead_code)]
|
||||
pub fn record_scheduled_update(&self) {
|
||||
self.scheduled_updates.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs.capacity.update.scheduled").increment(1);
|
||||
counter!(CAPACITY_UPDATE_SCHEDULED).increment(1);
|
||||
}
|
||||
|
||||
/// Record write triggered update
|
||||
#[allow(dead_code)]
|
||||
pub fn record_write_triggered_update(&self) {
|
||||
self.write_triggered_updates.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs.capacity.update.write_triggered").increment(1);
|
||||
counter!(CAPACITY_UPDATE_WRITE_TRIGGERED).increment(1);
|
||||
}
|
||||
|
||||
/// Record update failure
|
||||
#[allow(dead_code)]
|
||||
pub fn record_update_failure(&self) {
|
||||
self.update_failures.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs.capacity.update.failures").increment(1);
|
||||
counter!(CAPACITY_UPDATE_FAILURES).increment(1);
|
||||
}
|
||||
|
||||
/// Record write operation
|
||||
#[allow(dead_code)]
|
||||
pub fn record_write_operation(&self) {
|
||||
counter!("rustfs.capacity.write.operations").increment(1);
|
||||
counter!(CAPACITY_WRITE_OPERATIONS).increment(1);
|
||||
}
|
||||
|
||||
/// Record symlink encountered
|
||||
pub fn record_symlink(&self, size: u64) {
|
||||
self.symlink_count.fetch_add(1, Ordering::Relaxed);
|
||||
self.symlink_size.fetch_add(size, Ordering::Relaxed);
|
||||
counter!("rustfs.capacity.symlinks.encountered").increment(1);
|
||||
gauge!("rustfs.capacity.symlinks.total_size").set(size as f64);
|
||||
let total_size = self.symlink_size.fetch_add(size, Ordering::Relaxed) + size;
|
||||
counter!(CAPACITY_SYMLINKS_ENCOUNTERED).increment(1);
|
||||
gauge!(CAPACITY_SYMLINKS_SIZE).set(total_size as f64);
|
||||
}
|
||||
|
||||
/// Record dynamic timeout usage
|
||||
pub fn record_dynamic_timeout(&self) {
|
||||
self.dynamic_timeout_count.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs.capacity.timeout.dynamic").increment(1);
|
||||
counter!(CAPACITY_TIMEOUT_DYNAMIC).increment(1);
|
||||
}
|
||||
|
||||
/// Record timeout fallback to sampling
|
||||
pub fn record_timeout_fallback(&self) {
|
||||
self.timeout_fallback_count.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs.capacity.timeout.fallback").increment(1);
|
||||
counter!(CAPACITY_TIMEOUT_FALLBACK).increment(1);
|
||||
}
|
||||
|
||||
/// Record stall detection
|
||||
pub fn record_stall_detected(&self) {
|
||||
self.stall_detected_count.fetch_add(1, Ordering::Relaxed);
|
||||
counter!("rustfs.capacity.timeout.stall").increment(1);
|
||||
counter!(CAPACITY_TIMEOUT_STALL).increment(1);
|
||||
}
|
||||
|
||||
/// Get symlink statistics
|
||||
@@ -143,7 +225,7 @@ impl CapacityMetrics {
|
||||
self.total_update_duration_us.fetch_add(duration_us, Ordering::Relaxed);
|
||||
self.update_count.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
histogram!("rustfs.capacity.update.duration_us").record(duration_us as f64);
|
||||
histogram!(CAPACITY_UPDATE_DURATION_US).record(duration_us as f64);
|
||||
}
|
||||
|
||||
/// Get cache hit rate
|
||||
@@ -187,18 +269,18 @@ impl CapacityMetrics {
|
||||
pub fn log_summary(&self) {
|
||||
let summary = self.get_summary();
|
||||
|
||||
// Update gauges for current values
|
||||
gauge!("rustfs.capacity.cache.hit_rate").set(summary.cache_hit_rate);
|
||||
gauge!("rustfs.capacity.cache.hits_total").set(summary.cache_hits as f64);
|
||||
gauge!("rustfs.capacity.cache.misses_total").set(summary.cache_misses as f64);
|
||||
gauge!("rustfs.capacity.update.scheduled_total").set(summary.scheduled_updates as f64);
|
||||
gauge!("rustfs.capacity.update.write_triggered_total").set(summary.write_triggered_updates as f64);
|
||||
gauge!("rustfs.capacity.update.failures_total").set(summary.update_failures as f64);
|
||||
gauge!("rustfs.capacity.symlinks.count").set(summary.symlink_count as f64);
|
||||
gauge!("rustfs.capacity.symlinks.size").set(summary.symlink_size as f64);
|
||||
gauge!("rustfs.capacity.timeout.dynamic_total").set(summary.dynamic_timeout_count as f64);
|
||||
gauge!("rustfs.capacity.timeout.fallback_total").set(summary.timeout_fallback_count as f64);
|
||||
gauge!("rustfs.capacity.timeout.stall_total").set(summary.stall_detected_count as f64);
|
||||
// Update gauges for current values using constant names
|
||||
gauge!(CAPACITY_CACHE_HIT_RATE).set(summary.cache_hit_rate);
|
||||
gauge!(CAPACITY_CACHE_HITS_TOTAL).set(summary.cache_hits as f64);
|
||||
gauge!(CAPACITY_CACHE_MISSES_TOTAL).set(summary.cache_misses as f64);
|
||||
gauge!(CAPACITY_UPDATE_SCHEDULED_TOTAL).set(summary.scheduled_updates as f64);
|
||||
gauge!(CAPACITY_UPDATE_WRITE_TRIGGERED_TOTAL).set(summary.write_triggered_updates as f64);
|
||||
gauge!(CAPACITY_UPDATE_FAILURES_TOTAL).set(summary.update_failures as f64);
|
||||
gauge!(CAPACITY_SYMLINKS_COUNT).set(summary.symlink_count as f64);
|
||||
gauge!(CAPACITY_SYMLINKS_SIZE).set(summary.symlink_size as f64);
|
||||
gauge!(CAPACITY_TIMEOUT_DYNAMIC_TOTAL).set(summary.dynamic_timeout_count as f64);
|
||||
gauge!(CAPACITY_TIMEOUT_FALLBACK_TOTAL).set(summary.timeout_fallback_count as f64);
|
||||
gauge!(CAPACITY_TIMEOUT_STALL_TOTAL).set(summary.stall_detected_count as f64);
|
||||
|
||||
info!(
|
||||
"Capacity Metrics: cache_hit_rate={:.2}%, cache_hits={}, cache_misses={}, scheduled_updates={}, write_triggered_updates={}, update_failures={}, avg_update_duration={:?}, symlinks={}, symlink_size={}, dynamic_timeouts={}, timeout_fallbacks={}, stalls={}",
|
||||
@@ -278,6 +360,10 @@ pub fn record_global_cache_miss() {
|
||||
metrics.record_cache_miss();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -12,6 +12,63 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! # Capacity Management Module
|
||||
//!
|
||||
//! This module provides hybrid capacity management for RustFS with:
|
||||
//! - Scheduled background updates (configurable interval)
|
||||
//! - Write-triggered updates for high-frequency write scenarios
|
||||
//! - Configurable caching thresholds and smart update strategies
|
||||
//! - Comprehensive metrics collection for monitoring
|
||||
//!
|
||||
//! ## Configuration
|
||||
//!
|
||||
//! All configuration is via environment variables (see `rustfs_config`):
|
||||
//! - `RUSTFS_CAPACITY_SCHEDULED_INTERVAL` - Update interval in seconds (default: 300)
|
||||
//! - `RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY` - Write trigger delay (default: 10s)
|
||||
//! - `RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD` - Write frequency threshold (default: 10 writes/min)
|
||||
//! - `RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD` - Fast update threshold (default: 60s)
|
||||
//! - `RUSTFS_CAPACITY_MAX_FILES_THRESHOLD` - Max files before sampling (default: 1,000,000)
|
||||
//! - `RUSTFS_CAPACITY_STAT_TIMEOUT` - Stat operation timeout (default: 5s)
|
||||
//! - `RUSTFS_CAPACITY_SAMPLE_RATE` - Sampling rate for metrics (default: 100)
|
||||
//! - `RUSTFS_CAPACITY_FOLLOW_SYMLINKS` - Follow symlinks during traversal (default: false)
|
||||
//! - `RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH` - Max symlink depth (default: 8)
|
||||
//! - `RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT` - Enable dynamic timeout (default: false)
|
||||
//! - `RUSTFS_CAPACITY_MIN_TIMEOUT` - Minimum timeout (default: 1s)
|
||||
//! - `RUSTFS_CAPACITY_MAX_TIMEOUT` - Maximum timeout (default: 300s)
|
||||
//! - `RUSTFS_CAPACITY_STALL_TIMEOUT` - Stall detection timeout (default: 30s)
|
||||
//!
|
||||
//! ## Architecture
|
||||
//!
|
||||
//! The capacity management system uses a hybrid strategy:
|
||||
//! 1. **Real-time updates**: Triggered by write operations above threshold
|
||||
//! 2. **Scheduled updates**: Periodic background updates
|
||||
//! 3. **Cached responses**: Returns cached data when fresh
|
||||
//! 4. **Timeout protection**: Dynamic timeouts prevent hangs on large directories
|
||||
//!
|
||||
//! ## Metrics
|
||||
//!
|
||||
//! Metrics are automatically recorded via the `metrics` crate and accessible
|
||||
//! through the `rustfs-metrics` collection system. Key metrics include:
|
||||
//! - `rustfs.capacity.cache.{hits,misses}` - Cache hit/miss tracking
|
||||
//! - `rustfs.capacity.current` - Current capacity in bytes
|
||||
//! - `rustfs.capacity.write.operations` - Write operation count
|
||||
//! - `rustfs.capacity.update.{scheduled,write_triggered,failures}` - Update statistics
|
||||
//! - `rustfs.capacity.symlinks.*` - Symlink tracking statistics
|
||||
//! - `rustfs.capacity.timeout.*` - Timeout and stall detection
|
||||
//!
|
||||
//! ## Testing
|
||||
//!
|
||||
//! For isolated tests, use `create_isolated_manager()` to create independent
|
||||
//! instances instead of the global singleton:
|
||||
//!
|
||||
//! ```ignore
|
||||
//! use crate::capacity::create_isolated_manager;
|
||||
//!
|
||||
//! let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
//! // Test without affecting global state
|
||||
//! ```
|
||||
//!
|
||||
|
||||
pub mod capacity_integration;
|
||||
pub mod capacity_manager;
|
||||
#[cfg(test)]
|
||||
|
||||
Reference in New Issue
Block a user