mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-10 15:16:56 +00:00
0ebe00b383
Progress/timeout checks only ran when file_count advanced, so trees of directory-only or error-dense entries (dirs, traversal errors, symlinks never increment file_count) bypassed the entire cooperative time budget and held the blocking thread for unbounded time, while each error entry logged its own warn — permission-dense trees flooded the log (S13). The stall detector was structurally unreachable: it fired on 'no file progress between checks', but checks themselves only ran when file progress happened, so RUSTFS_CAPACITY_STALL_TIMEOUT never did anything since its introduction (S09). - Progress checks (timeout + early-sampling entry) are now driven by a visited-entry counter that also advances on directories and errors, so every tree shape reaches the budget checks. - Remove the unreachable stall detector and its plumbing end to end (ProgressMonitor fields, ScanLimits, config getter, env const, stall metric). Genuine walker wedges are handled by the hard outer wall-clock budget from backlog#1017; no decorative protection is left. - Cap per-entry error warns at 10 per scan with an explicit suppression notice; had_partial_errors still records the condition. Ref: rustfs/backlog#1016 (S09+S13 from audit rustfs/backlog#1010)
141 lines
5.8 KiB
Rust
141 lines
5.8 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
//! Capacity calculation configuration constants
|
|
|
|
// ============================================================================
|
|
// Environment Variable Names
|
|
// ============================================================================
|
|
|
|
/// Environment variable for scheduled update interval
|
|
pub const ENV_CAPACITY_SCHEDULED_INTERVAL: &str = "RUSTFS_CAPACITY_SCHEDULED_INTERVAL";
|
|
|
|
/// Environment variable for write trigger delay
|
|
pub const ENV_CAPACITY_WRITE_TRIGGER_DELAY: &str = "RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY";
|
|
|
|
/// Environment variable for write frequency threshold
|
|
pub const ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD: &str = "RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD";
|
|
|
|
/// Environment variable for fast update threshold
|
|
pub const ENV_CAPACITY_FAST_UPDATE_THRESHOLD: &str = "RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD";
|
|
|
|
/// Environment variable for max files threshold
|
|
pub const ENV_CAPACITY_MAX_FILES_THRESHOLD: &str = "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD";
|
|
|
|
/// Environment variable for statistics timeout
|
|
pub const ENV_CAPACITY_STAT_TIMEOUT: &str = "RUSTFS_CAPACITY_STAT_TIMEOUT";
|
|
|
|
/// Environment variable for sample rate
|
|
pub const ENV_CAPACITY_SAMPLE_RATE: &str = "RUSTFS_CAPACITY_SAMPLE_RATE";
|
|
|
|
/// Environment variable for metrics logging interval
|
|
pub const ENV_CAPACITY_METRICS_INTERVAL: &str = "RUSTFS_CAPACITY_METRICS_INTERVAL";
|
|
|
|
/// Environment variable for following symbolic links during capacity calculation
|
|
pub const ENV_CAPACITY_FOLLOW_SYMLINKS: &str = "RUSTFS_CAPACITY_FOLLOW_SYMLINKS";
|
|
|
|
/// Environment variable for maximum symlink follow depth
|
|
pub const ENV_CAPACITY_MAX_SYMLINK_DEPTH: &str = "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH";
|
|
|
|
/// Environment variable for enabling dynamic timeout calculation
|
|
pub const ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT: &str = "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT";
|
|
|
|
/// Environment variable for minimum capacity calculation timeout
|
|
pub const ENV_CAPACITY_MIN_TIMEOUT: &str = "RUSTFS_CAPACITY_MIN_TIMEOUT";
|
|
|
|
/// Environment variable for maximum capacity calculation timeout
|
|
pub const ENV_CAPACITY_MAX_TIMEOUT: &str = "RUSTFS_CAPACITY_MAX_TIMEOUT";
|
|
|
|
// ============================================================================
|
|
// Default Values
|
|
// ============================================================================
|
|
|
|
/// Scheduled update interval in seconds
|
|
/// Default: 120 seconds (2 minutes)
|
|
pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 120;
|
|
|
|
/// Write trigger delay in seconds
|
|
/// Default: 5 seconds
|
|
pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 5;
|
|
|
|
/// Write frequency threshold (writes per minute)
|
|
/// Default: 5 writes/minute
|
|
pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 5;
|
|
|
|
/// Fast update threshold in seconds
|
|
/// Default: 30 seconds
|
|
pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 30;
|
|
|
|
/// Maximum files threshold for sampling
|
|
/// Default: 200,000 files
|
|
pub const DEFAULT_MAX_FILES_THRESHOLD: usize = 200_000;
|
|
|
|
/// Statistics timeout in seconds
|
|
/// Default: 3 seconds
|
|
pub const DEFAULT_STAT_TIMEOUT_SECS: u64 = 3;
|
|
|
|
/// Sampling rate (1 in every N files)
|
|
/// Default: 200
|
|
pub const DEFAULT_SAMPLE_RATE: usize = 200;
|
|
|
|
/// Capacity metrics logging interval in seconds
|
|
/// Default: 600 seconds (10 minutes)
|
|
pub const DEFAULT_CAPACITY_METRICS_INTERVAL_SECS: u64 = 600;
|
|
|
|
/// Follow symbolic links during capacity calculation
|
|
/// Default: false (disabled for safety)
|
|
pub const DEFAULT_CAPACITY_FOLLOW_SYMLINKS: bool = false;
|
|
|
|
/// Maximum symlink follow depth
|
|
/// Default: 3 levels
|
|
pub const DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH: u8 = 3;
|
|
|
|
/// Enable dynamic timeout calculation based on directory characteristics
|
|
/// Default: true (enabled)
|
|
pub const DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT: bool = true;
|
|
|
|
/// Minimum capacity calculation timeout in seconds
|
|
/// Default: 2 seconds
|
|
pub const DEFAULT_CAPACITY_MIN_TIMEOUT_SECS: u64 = 2;
|
|
|
|
/// Maximum capacity calculation timeout in seconds
|
|
/// Default: 15 seconds
|
|
pub const DEFAULT_CAPACITY_MAX_TIMEOUT_SECS: u64 = 15;
|
|
|
|
// ============================================================================
|
|
// Tests
|
|
// ============================================================================
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_env_var_names() {
|
|
assert_eq!(ENV_CAPACITY_SCHEDULED_INTERVAL, "RUSTFS_CAPACITY_SCHEDULED_INTERVAL");
|
|
assert_eq!(ENV_CAPACITY_WRITE_TRIGGER_DELAY, "RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY");
|
|
assert_eq!(ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, "RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD");
|
|
assert_eq!(ENV_CAPACITY_FAST_UPDATE_THRESHOLD, "RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD");
|
|
assert_eq!(ENV_CAPACITY_MAX_FILES_THRESHOLD, "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD");
|
|
assert_eq!(ENV_CAPACITY_STAT_TIMEOUT, "RUSTFS_CAPACITY_STAT_TIMEOUT");
|
|
assert_eq!(ENV_CAPACITY_SAMPLE_RATE, "RUSTFS_CAPACITY_SAMPLE_RATE");
|
|
assert_eq!(ENV_CAPACITY_METRICS_INTERVAL, "RUSTFS_CAPACITY_METRICS_INTERVAL");
|
|
assert_eq!(ENV_CAPACITY_FOLLOW_SYMLINKS, "RUSTFS_CAPACITY_FOLLOW_SYMLINKS");
|
|
assert_eq!(ENV_CAPACITY_MAX_SYMLINK_DEPTH, "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH");
|
|
assert_eq!(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT");
|
|
assert_eq!(ENV_CAPACITY_MIN_TIMEOUT, "RUSTFS_CAPACITY_MIN_TIMEOUT");
|
|
assert_eq!(ENV_CAPACITY_MAX_TIMEOUT, "RUSTFS_CAPACITY_MAX_TIMEOUT");
|
|
}
|
|
}
|