fix(disk): Fix Usage Report Capacity Calculation (#2274)

Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: heihutu <heihutu@gmail.com>

crates/config/src/constants/capacity.rs

@@ -0,0 +1,155 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Capacity calculation configuration constants
+
+// ============================================================================
+// Environment Variable Names
+// ============================================================================
+
+/// Environment variable for scheduled update interval
+pub const ENV_CAPACITY_SCHEDULED_INTERVAL: &str = "RUSTFS_CAPACITY_SCHEDULED_INTERVAL";
+
+/// Environment variable for write trigger delay
+pub const ENV_CAPACITY_WRITE_TRIGGER_DELAY: &str = "RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY";
+
+/// Environment variable for write frequency threshold
+pub const ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD: &str = "RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD";
+
+/// Environment variable for fast update threshold
+pub const ENV_CAPACITY_FAST_UPDATE_THRESHOLD: &str = "RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD";
+
+/// Environment variable for max files threshold
+pub const ENV_CAPACITY_MAX_FILES_THRESHOLD: &str = "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD";
+
+/// Environment variable for statistics timeout
+pub const ENV_CAPACITY_STAT_TIMEOUT: &str = "RUSTFS_CAPACITY_STAT_TIMEOUT";
+
+/// Environment variable for sample rate
+pub const ENV_CAPACITY_SAMPLE_RATE: &str = "RUSTFS_CAPACITY_SAMPLE_RATE";
+
+/// Environment variable for following symbolic links during capacity calculation
+pub const ENV_CAPACITY_FOLLOW_SYMLINKS: &str = "RUSTFS_CAPACITY_FOLLOW_SYMLINKS";
+
+/// Environment variable for maximum symlink follow depth
+pub const ENV_CAPACITY_MAX_SYMLINK_DEPTH: &str = "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH";
+
+/// Environment variable for enabling dynamic timeout calculation
+pub const ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT: &str = "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT";
+
+/// Environment variable for minimum capacity calculation timeout
+pub const ENV_CAPACITY_MIN_TIMEOUT: &str = "RUSTFS_CAPACITY_MIN_TIMEOUT";
+
+/// Environment variable for maximum capacity calculation timeout
+pub const ENV_CAPACITY_MAX_TIMEOUT: &str = "RUSTFS_CAPACITY_MAX_TIMEOUT";
+
+/// Environment variable for progress stall detection timeout
+pub const ENV_CAPACITY_STALL_TIMEOUT: &str = "RUSTFS_CAPACITY_STALL_TIMEOUT";
+
+// ============================================================================
+// Default Values
+// ============================================================================
+
+/// Scheduled update interval in seconds
+/// Default: 300 seconds (5 minutes)
+pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 300;
+
+/// Write trigger delay in seconds
+/// Default: 10 seconds
+pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 10;
+
+/// Write frequency threshold (writes per minute)
+/// Default: 10 writes/minute
+pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 10;
+
+/// Fast update threshold in seconds
+/// Default: 60 seconds
+pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 60;
+
+/// Maximum files threshold for sampling
+/// Default: 1,000,000 files
+pub const DEFAULT_MAX_FILES_THRESHOLD: usize = 1_000_000;
+
+/// Statistics timeout in seconds
+/// Default: 5 seconds
+pub const DEFAULT_STAT_TIMEOUT_SECS: u64 = 5;
+
+/// Sampling rate (1 in every N files)
+/// Default: 100
+pub const DEFAULT_SAMPLE_RATE: usize = 100;
+
+/// Follow symbolic links during capacity calculation
+/// Default: false (disabled for safety)
+pub const DEFAULT_CAPACITY_FOLLOW_SYMLINKS: bool = false;
+
+/// Maximum symlink follow depth
+/// Default: 3 levels
+pub const DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH: u8 = 3;
+
+/// Enable dynamic timeout calculation based on directory characteristics
+/// Default: true (enabled)
+pub const DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT: bool = true;
+
+/// Minimum capacity calculation timeout in seconds
+/// Default: 5 seconds
+pub const DEFAULT_CAPACITY_MIN_TIMEOUT_SECS: u64 = 5;
+
+/// Maximum capacity calculation timeout in seconds
+/// Default: 60 seconds
+pub const DEFAULT_CAPACITY_MAX_TIMEOUT_SECS: u64 = 60;
+
+/// Progress stall detection timeout in seconds
+/// Default: 1 second (no progress for 1 second = stall)
+pub const DEFAULT_CAPACITY_STALL_TIMEOUT_SECS: u64 = 1;
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_env_var_names() {
+        assert_eq!(ENV_CAPACITY_SCHEDULED_INTERVAL, "RUSTFS_CAPACITY_SCHEDULED_INTERVAL");
+        assert_eq!(ENV_CAPACITY_WRITE_TRIGGER_DELAY, "RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY");
+        assert_eq!(ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, "RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD");
+        assert_eq!(ENV_CAPACITY_FAST_UPDATE_THRESHOLD, "RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD");
+        assert_eq!(ENV_CAPACITY_MAX_FILES_THRESHOLD, "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD");
+        assert_eq!(ENV_CAPACITY_STAT_TIMEOUT, "RUSTFS_CAPACITY_STAT_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_SAMPLE_RATE, "RUSTFS_CAPACITY_SAMPLE_RATE");
+        assert_eq!(ENV_CAPACITY_FOLLOW_SYMLINKS, "RUSTFS_CAPACITY_FOLLOW_SYMLINKS");
+        assert_eq!(ENV_CAPACITY_MAX_SYMLINK_DEPTH, "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH");
+        assert_eq!(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_MIN_TIMEOUT, "RUSTFS_CAPACITY_MIN_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_MAX_TIMEOUT, "RUSTFS_CAPACITY_MAX_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_STALL_TIMEOUT, "RUSTFS_CAPACITY_STALL_TIMEOUT");
+    }
+
+    #[test]
+    fn test_default_values() {
+        assert_eq!(DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, 300);
+        assert_eq!(DEFAULT_WRITE_TRIGGER_DELAY_SECS, 10);
+        assert_eq!(DEFAULT_WRITE_FREQUENCY_THRESHOLD, 10);
+        assert_eq!(DEFAULT_FAST_UPDATE_THRESHOLD_SECS, 60);
+        assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 1_000_000);
+        assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 5);
+        assert_eq!(DEFAULT_SAMPLE_RATE, 100);
+        assert_eq!(DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH, 3);
+        assert_eq!(DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, 5);
+        assert_eq!(DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, 60);
+        assert_eq!(DEFAULT_CAPACITY_STALL_TIMEOUT_SECS, 1);
+    }
+}

crates/config/src/constants/mod.rs

@@ -14,6 +14,7 @@
 
 pub(crate) mod app;
 pub(crate) mod body_limits;
+pub(crate) mod capacity;
 pub(crate) mod compress;
 pub(crate) mod console;
 pub(crate) mod env;

crates/config/src/constants/object.rs

@@ -214,6 +214,54 @@ pub const ENV_OBJECT_DISK_READ_TIMEOUT: &str = "RUSTFS_OBJECT_DISK_READ_TIMEOUT"
 /// Default disk read timeout in seconds.
 pub const DEFAULT_OBJECT_DISK_READ_TIMEOUT: u64 = 10;
 
+/// Environment variable for minimum GetObject timeout in seconds.
+///
+/// When dynamic timeout calculation is enabled, this is the minimum timeout
+/// that will be used regardless of object size. This prevents excessively
+/// short timeouts for very small objects.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_MIN_TIMEOUT`).
+pub const ENV_OBJECT_MIN_TIMEOUT: &str = "RUSTFS_OBJECT_MIN_TIMEOUT";
+
+/// Default minimum GetObject timeout: 5 seconds.
+pub const DEFAULT_OBJECT_MIN_TIMEOUT: u64 = 5;
+
+/// Environment variable for maximum GetObject timeout in seconds.
+///
+/// When dynamic timeout calculation is enabled, this is the maximum timeout
+/// that will be used regardless of object size. This prevents excessively
+/// long timeouts for very large objects.
+///
+/// Default: 300 seconds (5 minutes, can be overridden by `RUSTFS_OBJECT_MAX_TIMEOUT`).
+pub const ENV_OBJECT_MAX_TIMEOUT: &str = "RUSTFS_OBJECT_MAX_TIMEOUT";
+
+/// Default maximum GetObject timeout: 300 seconds (5 minutes).
+pub const DEFAULT_OBJECT_MAX_TIMEOUT: u64 = 300;
+
+/// Environment variable for default bytes per second for timeout estimation.
+///
+/// This value is used to estimate timeout duration based on object size when
+/// dynamic timeout calculation is enabled. The timeout is calculated as:
+/// (object_size / bytes_per_second) * buffer_factor
+///
+/// Default: 1048576 (1 MB/s, can be overridden by `RUSTFS_OBJECT_BYTES_PER_SECOND`).
+pub const ENV_OBJECT_BYTES_PER_SECOND: &str = "RUSTFS_OBJECT_BYTES_PER_SECOND";
+
+/// Default bytes per second for timeout estimation: 1 MB/s.
+pub const DEFAULT_OBJECT_BYTES_PER_SECOND: u64 = 1024 * 1024;
+
+/// Environment variable to enable dynamic timeout calculation.
+///
+/// When enabled, timeout is calculated based on object size and transfer speed
+/// rather than using a fixed timeout value. This provides better timeout
+/// handling for objects of varying sizes.
+///
+/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_DYNAMIC_TIMEOUT_ENABLE`).
+pub const ENV_OBJECT_DYNAMIC_TIMEOUT_ENABLE: &str = "RUSTFS_OBJECT_DYNAMIC_TIMEOUT_ENABLE";
+
+/// Default: dynamic timeout calculation is enabled.
+pub const DEFAULT_OBJECT_DYNAMIC_TIMEOUT_ENABLE: bool = true;
+
 /// Environment variable for duplex pipe buffer size in bytes.
 ///
 /// The duplex pipe connects the disk read task to the HTTP response stream.

crates/config/src/lib.rs

@@ -19,6 +19,8 @@ pub use constants::app::*;
 #[cfg(feature = "constants")]
 pub use constants::body_limits::*;
 #[cfg(feature = "constants")]
+pub use constants::capacity::*;
+#[cfg(feature = "constants")]
 pub use constants::compress::*;
 #[cfg(feature = "constants")]
 pub use constants::console::*;

crates/protocols/src/swift/handler.rs

@@ -1013,7 +1013,7 @@ async fn handle_authenticated_request(
 type SymlinkResolutionFuture<'a> =
     Pin<Box<dyn std::future::Future<Output = Result<(String, String, String, Option<String>), SwiftError>> + Send + 'a>>;
 
-/// Resolve symlink chain recursively
+/// Resolve symlink chain recursively with circular reference detection
 ///
 /// Returns (final_account, final_container, final_object, symlink_target_header)
 /// where symlink_target_header is Some(target) if the original object was a symlink
@@ -1023,12 +1023,17 @@ fn resolve_symlink_chain<'a>(
     object: &'a str,
     credentials: &'a Option<Credentials>,
     depth: u8,
+    visited: std::collections::HashSet<crate::swift::symlink::SymlinkPath>,
 ) -> SymlinkResolutionFuture<'a> {
     Box::pin(async move {
         use super::symlink;
 
-        // Validate depth to prevent infinite loops
-        symlink::validate_symlink_depth(depth)?;
+        // Validate both depth and circular references
+        symlink::validate_symlink_access(&visited, depth, account, container, object)?;
+
+        // Add current path to visited set
+        let mut new_visited = visited;
+        new_visited.insert(symlink::SymlinkPath::new(account, container, object));
 
         // Get object metadata
         let info = if let Some(creds) = credentials {
@@ -1041,14 +1046,14 @@ fn resolve_symlink_chain<'a>(
         // Check if this object is a symlink
         if let Some(target) = symlink::get_symlink_target(&info.user_defined)? {
             let target_container = target.resolve_container(container);
-            let target_object = &target.object;
+            let target_object = target.object.clone();
 
             // Store the original target for the response header
             let target_header = target.to_header_value(container);
 
             // Recursively resolve the target (it might also be a symlink)
             let (final_account, final_container, final_object, _) =
-                resolve_symlink_chain(account, target_container, target_object, credentials, depth + 1).await?;
+                resolve_symlink_chain(account, target_container, &target_object, credentials, depth + 1, new_visited).await?;
 
             // Return the final target, but keep the first-level symlink target for the header
             Ok((final_account, final_container, final_object, Some(target_header)))
@@ -1059,6 +1064,18 @@ fn resolve_symlink_chain<'a>(
     })
 }
 
+/// Helper function to start symlink resolution with an empty visited set
+fn resolve_symlink_chain_wrapper<'a>(
+    account: &'a str,
+    container: &'a str,
+    object: &'a str,
+    credentials: &'a Option<Credentials>,
+) -> SymlinkResolutionFuture<'a> {
+    Box::pin(
+        async move { resolve_symlink_chain(account, container, object, credentials, 0, std::collections::HashSet::new()).await },
+    )
+}
+
 /// Helper function for object GET operations (used by both authenticated and TempURL requests)
 async fn handle_object_get(
     account: &str,
@@ -1072,7 +1089,7 @@ async fn handle_object_get(
 
     // Resolve symlinks first (with loop detection)
     let (final_account, final_container, final_object, symlink_target) =
-        resolve_symlink_chain(account, container, object, credentials, 0).await?;
+        resolve_symlink_chain_wrapper(account, container, object, credentials).await?;
 
     // Check if object is SLO (via metadata)
     if slo::is_slo_object(&final_account, &final_container, &final_object, credentials).await? {
@@ -1213,7 +1230,7 @@ async fn handle_object_head(
 ) -> Result<Response<Body>, SwiftError> {
     // Resolve symlinks first (with loop detection)
     let (final_account, final_container, final_object, symlink_target) =
-        resolve_symlink_chain(account, container, object, credentials, 0).await?;
+        resolve_symlink_chain_wrapper(account, container, object, credentials).await?;
 
     let info = if let Some(creds) = credentials {
         object::head_object(&final_account, &final_container, &final_object, creds).await?
@@ -1437,8 +1454,7 @@ fn swift_error_to_response(error: SwiftError) -> Response<Body> {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-
+    use super::parse_range_header;
     #[test]
     fn test_parse_range_header_start_end() {
         // bytes=100-199

crates/protocols/src/swift/symlink.rs

@@ -59,11 +59,34 @@
 //! ```
 
 use super::{SwiftError, SwiftResult};
-use tracing::debug;
+use std::collections::HashSet;
+use tracing::{debug, warn};
 
 /// Maximum symlink follow depth to prevent infinite loops
 const MAX_SYMLINK_DEPTH: u8 = 5;
 
+/// Symlink path used for loop detection
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct SymlinkPath {
+    pub account: String,
+    pub container: String,
+    pub object: String,
+}
+
+impl SymlinkPath {
+    pub fn new(account: &str, container: &str, object: &str) -> Self {
+        Self {
+            account: account.to_string(),
+            container: container.to_string(),
+            object: object.to_string(),
+        }
+    }
+
+    pub fn from_strs(account: &str, container: &str, object: &str) -> Self {
+        Self::new(account, container, object)
+    }
+}
+
 /// Parsed symlink target
 #[derive(Debug, Clone, PartialEq)]
 pub struct SymlinkTarget {
@@ -167,6 +190,43 @@ pub fn validate_symlink_depth(depth: u8) -> SwiftResult<()> {
     Ok(())
 }
 
+/// Check if a symlink path has been visited before (circular reference detection)
+pub fn check_circular_reference(visited: &HashSet<SymlinkPath>, account: &str, container: &str, object: &str) -> SwiftResult<()> {
+    let path = SymlinkPath::new(account, container, object);
+
+    if visited.contains(&path) {
+        warn!(
+            account = %account,
+            container = %container,
+            object = %object,
+            "Circular symlink reference detected"
+        );
+        return Err(SwiftError::Conflict(format!(
+            "Circular symlink reference detected: {}/{}/{}",
+            account, container, object
+        )));
+    }
+
+    Ok(())
+}
+
+/// Validate symlink depth and check for circular references
+pub fn validate_symlink_access(
+    visited: &HashSet<SymlinkPath>,
+    depth: u8,
+    account: &str,
+    container: &str,
+    object: &str,
+) -> SwiftResult<()> {
+    // Check depth limit first
+    validate_symlink_depth(depth)?;
+
+    // Check for circular references
+    check_circular_reference(visited, account, container, object)?;
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -310,4 +370,94 @@ mod tests {
         assert!(validate_symlink_depth(5).is_err());
         assert!(validate_symlink_depth(10).is_err());
     }
+
+    #[test]
+    fn test_symlink_path_creation() {
+        let path = SymlinkPath::new("account1", "container1", "object1");
+        assert_eq!(path.account, "account1");
+        assert_eq!(path.container, "container1");
+        assert_eq!(path.object, "object1");
+    }
+
+    #[test]
+    fn test_symlink_path_equality() {
+        let path1 = SymlinkPath::new("account1", "container1", "object1");
+        let path2 = SymlinkPath::new("account1", "container1", "object1");
+        let path3 = SymlinkPath::new("account2", "container1", "object1");
+
+        assert_eq!(path1, path2);
+        assert_ne!(path1, path3);
+    }
+
+    #[test]
+    fn test_check_circular_reference_not_visited() {
+        let visited = HashSet::new();
+        assert!(check_circular_reference(&visited, "acc", "cont", "obj").is_ok());
+    }
+
+    #[test]
+    fn test_check_circular_reference_visited() {
+        let mut visited = HashSet::new();
+        visited.insert(SymlinkPath::new("acc", "cont", "obj"));
+
+        let result = check_circular_reference(&visited, "acc", "cont", "obj");
+        assert!(result.is_err());
+
+        if let Err(SwiftError::Conflict(msg)) = result {
+            assert!(msg.contains("Circular symlink reference detected"));
+            assert!(msg.contains("acc/cont/obj"));
+        } else {
+            panic!("Expected Conflict error");
+        }
+    }
+
+    #[test]
+    fn test_check_circular_reference_different_path() {
+        let mut visited = HashSet::new();
+        visited.insert(SymlinkPath::new("acc1", "cont1", "obj1"));
+
+        // Different path should not trigger circular reference error
+        assert!(check_circular_reference(&visited, "acc2", "cont2", "obj2").is_ok());
+    }
+
+    #[test]
+    fn test_validate_symlink_access_success() {
+        let visited = HashSet::new();
+        assert!(validate_symlink_access(&visited, 0, "acc", "cont", "obj").is_ok());
+        assert!(validate_symlink_access(&visited, 4, "acc", "cont", "obj").is_ok());
+    }
+
+    #[test]
+    fn test_validate_symlink_access_depth_exceeded() {
+        let visited = HashSet::new();
+        assert!(validate_symlink_access(&visited, 5, "acc", "cont", "obj").is_err());
+        assert!(validate_symlink_access(&visited, 10, "acc", "cont", "obj").is_err());
+    }
+
+    #[test]
+    fn test_validate_symlink_access_circular_reference() {
+        let mut visited = HashSet::new();
+        visited.insert(SymlinkPath::new("acc", "cont", "obj"));
+
+        let result = validate_symlink_access(&visited, 0, "acc", "cont", "obj");
+        assert!(result.is_err());
+
+        if let Err(SwiftError::Conflict(msg)) = result {
+            assert!(msg.contains("Circular symlink reference detected"));
+        } else {
+            panic!("Expected Conflict error");
+        }
+    }
+
+    #[test]
+    fn test_validate_symlink_access_both_checks() {
+        let mut visited = HashSet::new();
+        visited.insert(SymlinkPath::new("acc", "cont", "obj"));
+
+        // Should fail due to circular reference even though depth is OK
+        assert!(validate_symlink_access(&visited, 0, "acc", "cont", "obj").is_err());
+
+        // Should fail due to depth even though no circular reference
+        assert!(validate_symlink_access(&visited, 6, "acc2", "cont2", "obj2").is_err());
+    }
 }