Optimize log cleanup and rotation, update dependencies (#2032)

Co-authored-by: heihutu <heihutu@gmail.com>

.gitignore

@@ -43,3 +43,4 @@ docs
 
 # nix stuff
 result*
+*.gz

Cargo.lock

@@ -4529,9 +4529,9 @@ dependencies = [
 
 [[package]]
 name = "jiff"
-version = "0.2.21"
+version = "0.2.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3e3d65f018c6ae946ab16e80944b97096ed73c35b221d1c478a6c81d8f57940"
+checksum = "819b44bc7c87d9117eb522f14d46e918add69ff12713c475946b0a29363ed1c2"
 dependencies = [
  "jiff-static",
  "jiff-tzdb-platform",
@@ -4544,9 +4544,9 @@ dependencies = [
 
 [[package]]
 name = "jiff-static"
-version = "0.2.21"
+version = "0.2.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a17c2b211d863c7fde02cbea8a3c1a439b98e109286554f2860bdded7ff83818"
+checksum = "470252db18ecc35fd766c0891b1e3ec6cbbcd62507e85276c01bf75d8e94d4a1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -8001,8 +8001,9 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 
 [[package]]
 name = "s3s"
-version = "0.13.0-alpha.3"
+version = "0.13.0"
-source = "git+https://github.com/s3s-project/s3s.git?rev=3600aad5bcd691683e99d1e5fa17bae3bc21791d#3600aad5bcd691683e99d1e5fa17bae3bc21791d"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcf5572ca9bae5fd92d4e5a3e934c73793ab743bcbc761cd5e86a622c2a5e98a"
 dependencies = [
  "arc-swap",
  "arrayvec",

Cargo.toml

@@ -171,7 +171,7 @@ zeroize = { version = "1.8.2", features = ["derive"] }
 # Time and Date
 chrono = { version = "0.4.44", features = ["serde"] }
 humantime = "2.3.0"
-jiff = { version = "0.2.21", features = ["serde"] }
+jiff = { version = "0.2.22", features = ["serde"] }
 time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
 
 # Utilities and Tools
@@ -235,7 +235,7 @@ rumqttc = { version = "0.25.1" }
 rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.1" }
-s3s = { version = "0.13.0-alpha.3", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", rev = "3600aad5bcd691683e99d1e5fa17bae3bc21791d" }
+s3s = { version = "0.13.0", features = ["minio"] }
 serial_test = "3.4.0"
 shadow-rs = { version = "1.7.0", default-features = false }
 siphasher = "1.0.2"

crates/config/src/observability/mod.rs

@@ -53,6 +53,7 @@ pub const ENV_OBS_LOG_DELETE_EMPTY_FILES: &str = "RUSTFS_OBS_LOG_DELETE_EMPTY_FI
 pub const ENV_OBS_LOG_MIN_FILE_AGE_SECONDS: &str = "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS";
 pub const ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS: &str = "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS";
 pub const ENV_OBS_LOG_DRY_RUN: &str = "RUSTFS_OBS_LOG_DRY_RUN";
+pub const ENV_OBS_LOG_MATCH_MODE: &str = "RUSTFS_OBS_LOG_MATCH_MODE";
 
 /// Default values for log cleanup
 pub const DEFAULT_OBS_LOG_KEEP_COUNT: usize = 10;
@@ -65,6 +66,7 @@ pub const DEFAULT_OBS_LOG_DELETE_EMPTY_FILES: bool = true;
 pub const DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS: u64 = 3600; // 1 hour
 pub const DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS: u64 = 6 * 3600; // 6 hours
 pub const DEFAULT_OBS_LOG_DRY_RUN: bool = false;
+pub const DEFAULT_OBS_LOG_MATCH_MODE: &str = "suffix";
 
 /// Default values for observability configuration
 // ### Supported Environment Values
@@ -118,6 +120,7 @@ mod tests {
         assert_eq!(ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS");
         assert_eq!(ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS");
         assert_eq!(ENV_OBS_LOG_DRY_RUN, "RUSTFS_OBS_LOG_DRY_RUN");
+        assert_eq!(ENV_OBS_LOG_MATCH_MODE, "RUSTFS_OBS_LOG_MATCH_MODE");
     }
 
     #[test]
@@ -126,5 +129,6 @@ mod tests {
         assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development");
         assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test");
         assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging");
+        assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE, "suffix");
     }
 }

crates/obs/src/cleaner/core.rs

@@ -25,7 +25,7 @@
 
 use super::compress::compress_file;
 use super::scanner::{collect_expired_compressed_files, collect_log_files};
-use super::types::FileInfo;
+use super::types::{FileInfo, FileMatchMode};
 use std::path::PathBuf;
 use tracing::{debug, error, info};
 
@@ -41,8 +41,10 @@ use tracing::{debug, error, info};
 pub struct LogCleaner {
     /// Directory containing the managed log files.
     pub(super) log_dir: PathBuf,
-    /// Filename prefix that identifies managed files (e.g. `"rustfs.log."`).
+    /// Pattern string to match files (used as prefix or suffix).
-    pub(super) file_prefix: String,
+    pub(super) file_pattern: String,
+    /// Whether to match by prefix or suffix.
+    pub(super) match_mode: FileMatchMode,
     /// The cleaner will never delete files if doing so would leave fewer than
     /// this many files in the directory.
     pub(super) keep_count: usize,
@@ -77,7 +79,8 @@ impl LogCleaner {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
         log_dir: PathBuf,
-        file_prefix: String,
+        file_pattern: String,
+        match_mode: FileMatchMode,
         keep_count: usize,
         max_total_size_bytes: u64,
         max_single_file_size_bytes: u64,
@@ -96,7 +99,8 @@ impl LogCleaner {
 
         Self {
             log_dir,
-            file_prefix,
+            file_pattern,
+            match_mode,
             keep_count,
             max_total_size_bytes,
             max_single_file_size_bytes,
@@ -136,7 +140,8 @@ impl LogCleaner {
         // ── 1. Discover active log files ──────────────────────────────────────
         let mut files = collect_log_files(
             &self.log_dir,
-            &self.file_prefix,
+            &self.file_pattern,
+            self.match_mode,
             &self.exclude_patterns,
             self.min_file_age_seconds,
             self.delete_empty_files,
@@ -156,7 +161,15 @@ impl LogCleaner {
             );
 
             // ── 2. Select + compress + delete ─────────────────────────────────
-            let to_delete = self.select_files_to_delete(&files, total_size);
+            let (to_delete, to_rotate) = self.select_files_to_process(&files, total_size);
+
+            // Handle rotation for active file if needed
+            if let Some(active_file) = to_rotate
+                && let Err(e) = self.rotate_active_file(&active_file)
+            {
+                error!("Failed to rotate active file {:?}: {}", active_file.path, e);
+            }
+
             if !to_delete.is_empty() {
                 let (d, f) = self.compress_and_delete(&to_delete)?;
                 total_deleted += d;
@@ -165,7 +178,12 @@ impl LogCleaner {
         }
 
         // ── 3. Remove expired compressed archives ─────────────────────────────
-        let expired_gz = collect_expired_compressed_files(&self.log_dir, &self.file_prefix, self.compressed_file_retention_days)?;
+        let expired_gz = collect_expired_compressed_files(
+            &self.log_dir,
+            &self.file_pattern,
+            self.match_mode,
+            self.compressed_file_retention_days,
+        )?;
         if !expired_gz.is_empty() {
             let (d, f) = self.delete_files(&expired_gz)?;
             total_deleted += d;
@@ -186,51 +204,180 @@ impl LogCleaner {
 
     // ─── Selection ────────────────────────────────────────────────────────────
 
-    /// Choose which files from `files` (sorted oldest-first) should be deleted.
+    /// Choose which files from `files` (sorted oldest-first) should be deleted or rotated.
     ///
     /// The algorithm respects three constraints in order:
     /// 1. Always keep at least `keep_count` files.
     /// 2. Delete old files while the total size exceeds `max_total_size_bytes`.
     /// 3. Delete any file whose individual size exceeds `max_single_file_size_bytes`.
-    pub(super) fn select_files_to_delete(&self, files: &[FileInfo], total_size: u64) -> Vec<FileInfo> {
+    ///
+    /// **Note**: The most recent file (assumed to be the active log) is exempt
+    /// from size-based deletion. If it exceeds the size limit, it is returned
+    /// as `to_rotate`.
+    pub(super) fn select_files_to_process(&self, files: &[FileInfo], total_size: u64) -> (Vec<FileInfo>, Option<FileInfo>) {
         let mut to_delete = Vec::new();
+        let mut to_rotate = None;
 
-        if files.len() <= self.keep_count {
+        if files.is_empty() {
-            return to_delete;
+            return (to_delete, to_rotate);
         }
 
+        // Identify the index of the most recent file (last in the sorted list).
+        // We will protect this file from size-based deletion.
+        let active_file_idx = files.len() - 1;
+
+        // The number of files we are allowed to delete.
+        // Any file with index >= max_deletable_count is protected by keep_count.
+        let max_deletable_count = files.len().saturating_sub(self.keep_count);
+
         let mut current_size = total_size;
-        let deletable = files.len() - self.keep_count;
 
         for (idx, file) in files.iter().enumerate() {
-            if idx >= deletable {
+            // If we are in the protected range, we stop deleting.
-                break;
+            if idx >= max_deletable_count {
+                // However, if the active file is too large, we might rotate it.
+                if idx == active_file_idx {
+                    let over_single = self.max_single_file_size_bytes > 0 && file.size > self.max_single_file_size_bytes;
+                    if over_single {
+                        to_rotate = Some(file.clone());
+                    }
+                }
+                continue;
             }
 
+            // We are in the deletable range. Check if we *should* delete.
+
+            // Condition 2: Enforce max_total_size_bytes.
             let over_total = self.max_total_size_bytes > 0 && current_size > self.max_total_size_bytes;
+
+            // Condition 3: Enforce max_single_file_size_bytes.
             let over_single = self.max_single_file_size_bytes > 0 && file.size > self.max_single_file_size_bytes;
 
-            if over_total || over_single {
+            if over_total {
-                if over_single {
+                // If we are over total size, we delete unless it's the active file.
+                if idx == active_file_idx {
+                    debug!(
+                        "Active log file contributes to total size limit overflow, but skipping deletion to preserve current logs."
+                    );
+                } else {
+                    current_size = current_size.saturating_sub(file.size);
+                    to_delete.push(file.clone());
+                }
+            } else if over_single {
+                // For single file limits, we MUST NOT delete the active file.
+                if idx == active_file_idx {
+                    // Mark active file for rotation instead of deletion
+                    to_rotate = Some(file.clone());
+                } else {
                     debug!(
                         "File exceeds single-file size limit: {:?} ({} > {} bytes)",
                         file.path, file.size, self.max_single_file_size_bytes
                     );
+                    current_size = current_size.saturating_sub(file.size);
+                    to_delete.push(file.clone());
                 }
-                current_size = current_size.saturating_sub(file.size);
-                to_delete.push(file.clone());
-            } else {
-                // Neither limit is breached; stop scanning.
-                break;
             }
         }
 
-        to_delete
+        (to_delete, to_rotate)
+    }
+
+    // ─── Rotation ─────────────────────────────────────────────────────────────
+
+    /// Rotate the active file by renaming it with a timestamp suffix.
+    /// The original filename will be recreated by the logging appender on next write.
+    fn rotate_active_file(&self, file: &FileInfo) -> Result<(), std::io::Error> {
+        if self.dry_run {
+            info!("[DRY RUN] Would rotate active file: {:?} ({} bytes)", file.path, file.size);
+            return Ok(());
+        }
+
+        // Generate timestamp: unix timestamp in seconds
+        let timestamp = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .map_err(std::io::Error::other)?
+            .as_secs();
+
+        let file_name = file
+            .path
+            .file_name()
+            .and_then(|n| n.to_str())
+            .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid filename"))?;
+
+        // Construct the rotated filename.
+        // We must ensure the new filename still matches the file_pattern so it can be discovered
+        // by the scanner in future runs (and eventually deleted).
+        //
+        // Suffix mode: Insert timestamp BEFORE the suffix.
+        //   Example: "2026-03-01.rustfs.log" (pattern="rustfs.log")
+        //   -> "2026-03-01.1740810000.rustfs.log"
+        //
+        // Prefix mode: Append timestamp at the end.
+        //   Example: "app.log" (pattern="app")
+        //   -> "app.log.1740810000"
+        let rotated_name = match self.match_mode {
+            FileMatchMode::Suffix => {
+                if let Some(base) = file_name.strip_suffix(&self.file_pattern) {
+                    let mut new_name = String::with_capacity(file_name.len() + 20);
+                    new_name.push_str(base);
+
+                    // Ensure separator between base and timestamp
+                    if !base.is_empty() && !base.ends_with('.') {
+                        new_name.push('.');
+                    }
+                    new_name.push_str(&timestamp.to_string());
+
+                    // Ensure separator between timestamp and suffix
+                    if !self.file_pattern.starts_with('.') {
+                        new_name.push('.');
+                    }
+                    new_name.push_str(&self.file_pattern);
+
+                    new_name
+                } else {
+                    // Should not happen if scanner works correctly, but fallback safely
+                    format!("{}.{}", file_name, timestamp)
+                }
+            }
+            FileMatchMode::Prefix => {
+                // For prefix matching, appending to the end preserves the prefix.
+                format!("{}.{}", file_name, timestamp)
+            }
+        };
+
+        let rotated_path = file.path.with_file_name(&rotated_name);
+
+        // Check if target already exists to avoid overwriting (unlikely with timestamp but possible)
+        if rotated_path.exists() {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::AlreadyExists,
+                format!("Rotated file already exists: {:?}", rotated_path),
+            ));
+        }
+
+        info!("Rotating active log file: {:?} -> {:?}", file.path, rotated_path);
+
+        // Rename the current active file to the rotated name.
+        // The logging appender (tracing-appender) will automatically create a new file
+        // with the original name when it next attempts to write.
+        // Note: On Linux/Unix, this rename is atomic and safe even if the file is open.
+        if let Err(e) = std::fs::rename(&file.path, &rotated_path) {
+            // Add context to the error
+            return Err(std::io::Error::new(
+                e.kind(),
+                format!("Failed to rename {:?} to {:?}: {}", file.path, rotated_path, e),
+            ));
+        }
+
+        Ok(())
     }
 
     // ─── Compression + deletion ───────────────────────────────────────────────
 
     /// Optionally compress and then delete the given files.
+    ///
+    /// This function is synchronous and blocking. It should be called within a
+    /// `spawn_blocking` task if running in an async context.
     fn compress_and_delete(&self, files: &[FileInfo]) -> Result<(usize, u64), std::io::Error> {
         if self.compress_old_files {
             for f in files {

crates/obs/src/cleaner/mod.rs

@@ -24,17 +24,19 @@
 //! | `types`     | Shared data types (`FileInfo`)                           |
 //! | `scanner`   | Filesystem traversal — discovers eligible files          |
 //! | `compress`  | Gzip compression helper                                  |
-//! | `cleaner`   | Core orchestration — selection, compression, deletion    |
+//! | `core`      | Core orchestration — selection, compression, deletion    |
 //!
 //! ## Usage
 //!
 //! ```no_run
 //! use std::path::PathBuf;
 //! use rustfs_obs::LogCleaner;
+//! use rustfs_obs::types::FileMatchMode;
 //!
 //! let cleaner = LogCleaner::new(
 //!     PathBuf::from("/var/log/rustfs"),
 //!     "rustfs.log.".to_string(),
+//!     FileMatchMode::Prefix,
 //!     10,              // keep_count
 //!     2 * 1024 * 1024 * 1024, // max_total_size_bytes (2 GiB)
 //!     0,               // max_single_file_size_bytes (unlimited)
@@ -51,17 +53,18 @@
 //! println!("Deleted {deleted} files, freed {freed_bytes} bytes");
 //! ```
 
-mod cleaner;
 mod compress;
+mod core;
 mod scanner;
-mod types;
+pub mod types;
 
-pub use cleaner::LogCleaner;
+pub use core::LogCleaner;
 
 #[cfg(test)]
 mod tests {
-    use super::cleaner::LogCleaner;
+    use super::core::LogCleaner;
     use super::scanner;
+    use super::types::FileMatchMode;
     use std::fs::File;
     use std::io::Write;
     use std::path::Path;
@@ -79,6 +82,7 @@ mod tests {
         LogCleaner::new(
             dir,
             "app.log.".to_string(),
+            FileMatchMode::Prefix,
             keep,
             max_bytes,
             0,          // max_single_file_size_bytes
@@ -153,7 +157,7 @@ mod tests {
         create_log_file(&dir, "app.log.2024-01-02", 2048)?;
         create_log_file(&dir, "other.log", 512)?;
 
-        let files = scanner::collect_log_files(&dir, "app.log.", &[], 0, true, false)?;
+        let files = scanner::collect_log_files(&dir, "app.log.", FileMatchMode::Prefix, &[], 0, true, false)?;
         assert_eq!(files.len(), 2, "scanner should find exactly 2 managed files");
         Ok(())
     }
@@ -167,7 +171,21 @@ mod tests {
         create_log_file(&dir, "app.log.2024-01-02", 1024)?;
         create_log_file(&dir, "app.log.2024-01-03", 1024)?;
 
-        let cleaner = LogCleaner::new(dir.clone(), "app.log.".to_string(), 1, 1024, 0, false, 6, 30, vec![], true, 0, true);
+        let cleaner = LogCleaner::new(
+            dir.clone(),
+            "app.log.".to_string(),
+            FileMatchMode::Prefix,
+            1,
+            1024,
+            0,
+            false,
+            6,
+            30,
+            vec![],
+            true,
+            0,
+            true,
+        );
         let (deleted, _freed) = cleaner.cleanup()?;
 
         // dry_run=true reports deletions but doesn't actually remove files.
@@ -175,4 +193,35 @@ mod tests {
         assert_eq!(std::fs::read_dir(&dir)?.count(), 3, "no files should actually be removed");
         Ok(())
     }
+
+    #[test]
+    fn test_cleanup_suffix_matching() -> std::io::Result<()> {
+        let tmp = TempDir::new()?;
+        let dir = tmp.path().to_path_buf();
+
+        create_log_file(&dir, "2026-03-01-06-21.rustfs.log", 1024)?;
+        create_log_file(&dir, "2026-03-01-06-22.rustfs.log", 1024)?;
+        create_log_file(&dir, "other.log", 1024)?; // not managed
+
+        let cleaner = LogCleaner::new(
+            dir.clone(),
+            "rustfs.log".to_string(),
+            FileMatchMode::Suffix,
+            1,
+            1024,
+            0,
+            false,
+            6,
+            30,
+            vec![],
+            true,
+            0,
+            false,
+        );
+        let (deleted, freed) = cleaner.cleanup()?;
+
+        assert_eq!(deleted, 1, "should delete exactly one file");
+        assert_eq!(freed, 1024);
+        Ok(())
+    }
 }

crates/obs/src/cleaner/scanner.rs

@@ -17,13 +17,13 @@
 //! This module is intentionally kept read-only: it does **not** delete or
 //! compress any files — it only reports what it found.
 
-use super::types::FileInfo;
+use super::types::{FileInfo, FileMatchMode};
 use std::path::Path;
 use std::time::{Duration, SystemTime};
 use tracing::debug;
 use walkdir::WalkDir;
 
-/// Collect all log files in `log_dir` whose name starts with `file_prefix`.
+/// Collect all log files in `log_dir` whose name matches `file_pattern` based on `match_mode`.
 ///
 /// Files that:
 /// - are already compressed (`.gz` extension),
@@ -36,14 +36,16 @@ use walkdir::WalkDir;
 ///
 /// # Arguments
 /// * `log_dir` - Root directory to scan (depth 1 only, no recursion).
-/// * `file_prefix` - Only filenames starting with this string are considered.
+/// * `file_pattern` - Pattern string to match filenames.
+/// * `match_mode` - Whether to match by prefix or suffix.
 /// * `exclude_patterns` - Compiled glob patterns; matching files are skipped.
 /// * `min_file_age_seconds` - Files younger than this threshold are skipped.
 /// * `delete_empty_files` - When `true`, zero-byte files trigger an immediate
 ///   delete by the caller before the rest of cleanup runs.
 pub(super) fn collect_log_files(
     log_dir: &Path,
-    file_prefix: &str,
+    file_pattern: &str,
+    match_mode: FileMatchMode,
     exclude_patterns: &[glob::Pattern],
     min_file_age_seconds: u64,
     delete_empty_files: bool,
@@ -68,8 +70,13 @@ pub(super) fn collect_log_files(
             None => continue,
         };
 
-        // Only manage files that carry our prefix.
+        // Match filename based on mode
-        if !filename.starts_with(file_prefix) {
+        let matches = match match_mode {
+            FileMatchMode::Prefix => filename.starts_with(file_pattern),
+            FileMatchMode::Suffix => filename.ends_with(file_pattern),
+        };
+
+        if !matches {
             continue;
         }
 
@@ -139,12 +146,14 @@ pub(super) fn collect_log_files(
 ///
 /// # Arguments
 /// * `log_dir` - Root directory to scan.
-/// * `file_prefix` - Only `.gz` files that also start with this prefix are considered.
+/// * `file_pattern` - Pattern string to match filenames.
+/// * `match_mode` - Whether to match by prefix or suffix.
 /// * `compressed_file_retention_days` - Files older than this are eligible for
 ///   deletion; `0` means never delete compressed files.
 pub(super) fn collect_expired_compressed_files(
     log_dir: &Path,
-    file_prefix: &str,
+    file_pattern: &str,
+    match_mode: FileMatchMode,
     compressed_file_retention_days: u64,
 ) -> Result<Vec<FileInfo>, std::io::Error> {
     if compressed_file_retention_days == 0 {
@@ -171,7 +180,18 @@ pub(super) fn collect_expired_compressed_files(
             None => continue,
         };
 
-        if !filename.starts_with(file_prefix) || !filename.ends_with(".gz") {
+        if !filename.ends_with(".gz") {
+            continue;
+        }
+
+        // Check if the base filename (without .gz) matches the pattern
+        let base_filename = &filename[..filename.len() - 3];
+        let matches = match match_mode {
+            FileMatchMode::Prefix => base_filename.starts_with(file_pattern),
+            FileMatchMode::Suffix => base_filename.ends_with(file_pattern),
+        };
+
+        if !matches {
             continue;
         }
 

crates/obs/src/cleaner/types.rs

@@ -17,6 +17,15 @@
 use std::path::PathBuf;
 use std::time::SystemTime;
 
+/// Strategy for matching log files against a pattern.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FileMatchMode {
+    /// The filename must start with the pattern (e.g. "app.log." matches "app.log.2024-01-01").
+    Prefix,
+    /// The filename must end with the pattern (e.g. ".log" matches "2024-01-01.log").
+    Suffix,
+}
+
 /// Metadata for a single log file discovered by the scanner.
 ///
 /// Carries enough information to make cleanup decisions (sort by age, compare

crates/obs/src/config.rs

@@ -25,16 +25,16 @@
 use rustfs_config::observability::{
     DEFAULT_OBS_ENVIRONMENT_PRODUCTION, DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS, DEFAULT_OBS_LOG_COMPRESS_OLD_FILES,
     DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, DEFAULT_OBS_LOG_DELETE_EMPTY_FILES, DEFAULT_OBS_LOG_DRY_RUN,
-    DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_KEEP_COUNT, DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES,
+    DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_KEEP_COUNT, DEFAULT_OBS_LOG_MATCH_MODE,
-    DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_ENDPOINT, ENV_OBS_ENVIRONMENT,
+    DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS,
-    ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, ENV_OBS_LOG_COMPRESS_OLD_FILES, ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS,
+    ENV_OBS_ENDPOINT, ENV_OBS_ENVIRONMENT, ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, ENV_OBS_LOG_COMPRESS_OLD_FILES,
-    ENV_OBS_LOG_DELETE_EMPTY_FILES, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN, ENV_OBS_LOG_ENDPOINT,
+    ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, ENV_OBS_LOG_DELETE_EMPTY_FILES, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN,
-    ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL, ENV_OBS_LOG_KEEP_COUNT,
+    ENV_OBS_LOG_ENDPOINT, ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL,
-    ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES,
+    ENV_OBS_LOG_KEEP_COUNT, ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES,
-    ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOG_STDOUT_ENABLED, ENV_OBS_LOGGER_LEVEL,
+    ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOG_STDOUT_ENABLED,
-    ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT, ENV_OBS_METRICS_EXPORT_ENABLED,
+    ENV_OBS_LOGGER_LEVEL, ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT,
-    ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED,
+    ENV_OBS_METRICS_EXPORT_ENABLED, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT,
-    ENV_OBS_USE_STDOUT,
+    ENV_OBS_TRACES_EXPORT_ENABLED, ENV_OBS_USE_STDOUT,
 };
 use rustfs_config::{
     APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_FILENAME,
@@ -157,6 +157,8 @@ pub struct OtelConfig {
     /// Log what *would* be deleted without actually removing anything
     /// (default: `false`).
     pub log_dry_run: Option<bool>,
+    /// File matching mode: "prefix" or "suffix" (default: "suffix").
+    pub log_match_mode: Option<String>,
 }
 
 impl OtelConfig {
@@ -259,6 +261,7 @@ impl OtelConfig {
                 DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS,
             )),
             log_dry_run: Some(get_env_bool(ENV_OBS_LOG_DRY_RUN, DEFAULT_OBS_LOG_DRY_RUN)),
+            log_match_mode: Some(get_env_str(ENV_OBS_LOG_MATCH_MODE, DEFAULT_OBS_LOG_MATCH_MODE)),
         }
     }
 

crates/obs/src/lib.rs

@@ -53,16 +53,16 @@
 //! #   // Guard will be dropped here, flushing telemetry data
 //! # }
 //! ```
+mod cleaner;
 mod config;
 mod error;
 mod global;
-mod log_cleanup;
 mod system;
 mod telemetry;
 
+pub use cleaner::*;
 pub use config::*;
 pub use error::*;
 pub use global::*;
-pub use log_cleanup::*;
 pub use system::SystemObserver;
 pub use telemetry::{OtelGuard, Recorder};

crates/obs/src/telemetry/local.rs

@@ -27,9 +27,10 @@
 //! cases; callers do **not** need to distinguish between stdout and file modes.
 
 use crate::TelemetryError;
+use crate::cleaner::LogCleaner;
+use crate::cleaner::types::FileMatchMode;
 use crate::config::OtelConfig;
 use crate::global::OBSERVABILITY_METRIC_ENABLED;
-use crate::log_cleanup::LogCleaner;
 use crate::telemetry::filter::build_env_filter;
 use metrics::counter;
 use rustfs_config::observability::{
@@ -38,6 +39,7 @@ use rustfs_config::observability::{
     DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS,
 };
 use rustfs_config::{APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_STDOUT_ENABLED};
+use std::sync::Arc;
 use std::{fs, io::IsTerminal, time::Duration};
 use tracing::info;
 use tracing_error::ErrorLayer;
@@ -176,6 +178,13 @@ fn init_file_logging_internal(
         .as_deref()
         .unwrap_or(DEFAULT_LOG_ROTATION_TIME)
         .to_lowercase();
+
+    // Determine match mode from config, defaulting to Suffix
+    let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() {
+        Some("prefix") => FileMatchMode::Prefix,
+        _ => FileMatchMode::Suffix,
+    };
+
     use tracing_appender::rolling::{RollingFileAppender, Rotation};
     let file_appender = {
         let rotation = match rotation.as_str() {
@@ -183,10 +192,17 @@ fn init_file_logging_internal(
             "hourly" => Rotation::HOURLY,
             _ => Rotation::DAILY,
         };
-        RollingFileAppender::builder()
+
+        let mut builder = RollingFileAppender::builder()
             .rotation(rotation)
-            .filename_suffix(log_filename)
+            .max_log_files(keep_files * 2); // Make sure there are some data files to archive to avoid premature deletion
-            .max_log_files(keep_files)
+
+        match match_mode {
+            FileMatchMode::Prefix => builder = builder.filename_prefix(log_filename),
+            FileMatchMode::Suffix => builder = builder.filename_suffix(log_filename),
+        }
+
+        builder
             .build(log_directory)
             .expect("failed to initialize rolling file appender")
     };
@@ -329,7 +345,16 @@ fn spawn_cleanup_task(
     keep_files: usize,
 ) -> tokio::task::JoinHandle<()> {
     let log_dir = std::path::PathBuf::from(log_directory);
-    let file_prefix = config.log_filename.as_deref().unwrap_or(log_filename).to_string();
+    // Use suffix matching for log files like "2026-03-01-06-21.rustfs.log"
+    // where "rustfs.log" is the suffix.
+    let file_pattern = config.log_filename.as_deref().unwrap_or(log_filename).to_string();
+
+    // Determine match mode from config, defaulting to Suffix
+    let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() {
+        Some("prefix") => FileMatchMode::Prefix,
+        _ => FileMatchMode::Suffix,
+    };
+
     let keep_count = config.log_keep_count.unwrap_or(keep_files);
     let max_total_size = config
         .log_max_total_size_bytes
@@ -358,9 +383,10 @@ fn spawn_cleanup_task(
         .log_cleanup_interval_seconds
         .unwrap_or(DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS);
 
-    let cleaner = LogCleaner::new(
+    let cleaner = Arc::new(LogCleaner::new(
         log_dir,
-        file_prefix,
+        file_pattern,
+        match_mode,
         keep_count,
         max_total_size,
         max_single_file_size,
@@ -371,14 +397,19 @@ fn spawn_cleanup_task(
         delete_empty,
         min_age,
         dry_run,
-    );
+    ));
 
     tokio::spawn(async move {
         let mut interval = tokio::time::interval(Duration::from_secs(cleanup_interval));
         loop {
             interval.tick().await;
-            if let Err(e) = cleaner.cleanup() {
+            let cleaner_clone = cleaner.clone();
-                tracing::warn!("Log cleanup failed: {}", e);
+            let result = tokio::task::spawn_blocking(move || cleaner_clone.cleanup()).await;
+
+            match result {
+                Ok(Ok(_)) => {} // Success
+                Ok(Err(e)) => tracing::warn!("Log cleanup failed: {}", e),
+                Err(e) => tracing::warn!("Log cleanup task panicked: {}", e),
             }
         }
     })

scripts/run.sh

@@ -79,6 +79,7 @@ export RUSTFS_OBS_LOG_FLUSH_MS=300 # Log flush interval in milliseconds
 export RUSTFS_RUNTIME_WORKER_THREADS=16
 export RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024
 export RUSTFS_RUNTIME_THREAD_PRINT_ENABLED=false
+export RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS=300
 # shellcheck disable=SC2125
 export RUSTFS_RUNTIME_THREAD_STACK_SIZE=1024*1024
 export RUSTFS_RUNTIME_THREAD_KEEP_ALIVE=60