From c452f244870e54596db47a894c9abcb59675d2df Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 1 Mar 2026 20:09:52 +0800 Subject: [PATCH] Optimize log cleanup and rotation, update dependencies (#2032) Co-authored-by: heihutu --- .gitignore | 1 + Cargo.lock | 13 +- Cargo.toml | 4 +- crates/config/src/observability/mod.rs | 4 + .../src/{log_cleanup => cleaner}/compress.rs | 0 .../cleaner.rs => cleaner/core.rs} | 193 +++++++++++++++--- .../obs/src/{log_cleanup => cleaner}/mod.rs | 63 +++++- .../src/{log_cleanup => cleaner}/scanner.rs | 38 +++- .../obs/src/{log_cleanup => cleaner}/types.rs | 9 + crates/obs/src/config.rs | 23 ++- crates/obs/src/lib.rs | 4 +- crates/obs/src/telemetry/local.rs | 51 ++++- scripts/run.sh | 1 + 13 files changed, 335 insertions(+), 69 deletions(-) rename crates/obs/src/{log_cleanup => cleaner}/compress.rs (100%) rename crates/obs/src/{log_cleanup/cleaner.rs => cleaner/core.rs} (56%) rename crates/obs/src/{log_cleanup => cleaner}/mod.rs (80%) rename crates/obs/src/{log_cleanup => cleaner}/scanner.rs (83%) rename crates/obs/src/{log_cleanup => cleaner}/types.rs (78%) diff --git a/.gitignore b/.gitignore index e71600a75..c80107ae6 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ docs # nix stuff result* +*.gz diff --git a/Cargo.lock b/Cargo.lock index 3a696c30c..4f20a17d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4529,9 +4529,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e3d65f018c6ae946ab16e80944b97096ed73c35b221d1c478a6c81d8f57940" +checksum = "819b44bc7c87d9117eb522f14d46e918add69ff12713c475946b0a29363ed1c2" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -4544,9 +4544,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17c2b211d863c7fde02cbea8a3c1a439b98e109286554f2860bdded7ff83818" +checksum = "470252db18ecc35fd766c0891b1e3ec6cbbcd62507e85276c01bf75d8e94d4a1" dependencies = [ "proc-macro2", "quote", @@ -8001,8 +8001,9 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" -version = "0.13.0-alpha.3" -source = "git+https://github.com/s3s-project/s3s.git?rev=3600aad5bcd691683e99d1e5fa17bae3bc21791d#3600aad5bcd691683e99d1e5fa17bae3bc21791d" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf5572ca9bae5fd92d4e5a3e934c73793ab743bcbc761cd5e86a622c2a5e98a" dependencies = [ "arc-swap", "arrayvec", diff --git a/Cargo.toml b/Cargo.toml index 606d862d2..2fc9740c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,7 +171,7 @@ zeroize = { version = "1.8.2", features = ["derive"] } # Time and Date chrono = { version = "0.4.44", features = ["serde"] } humantime = "2.3.0" -jiff = { version = "0.2.21", features = ["serde"] } +jiff = { version = "0.2.22", features = ["serde"] } time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] } # Utilities and Tools @@ -235,7 +235,7 @@ rumqttc = { version = "0.25.1" } rustix = { version = "1.1.4", features = ["fs"] } rust-embed = { version = "8.11.0" } rustc-hash = { version = "2.1.1" } -s3s = { version = "0.13.0-alpha.3", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", rev = "3600aad5bcd691683e99d1e5fa17bae3bc21791d" } +s3s = { version = "0.13.0", features = ["minio"] } serial_test = "3.4.0" shadow-rs = { version = "1.7.0", default-features = false } siphasher = "1.0.2" diff --git a/crates/config/src/observability/mod.rs b/crates/config/src/observability/mod.rs index b83c59000..1812284c4 100644 --- a/crates/config/src/observability/mod.rs +++ b/crates/config/src/observability/mod.rs @@ -53,6 +53,7 @@ pub const ENV_OBS_LOG_DELETE_EMPTY_FILES: &str = "RUSTFS_OBS_LOG_DELETE_EMPTY_FI pub const ENV_OBS_LOG_MIN_FILE_AGE_SECONDS: &str = "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS"; pub const ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS: &str = "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS"; pub const ENV_OBS_LOG_DRY_RUN: &str = "RUSTFS_OBS_LOG_DRY_RUN"; +pub const ENV_OBS_LOG_MATCH_MODE: &str = "RUSTFS_OBS_LOG_MATCH_MODE"; /// Default values for log cleanup pub const DEFAULT_OBS_LOG_KEEP_COUNT: usize = 10; @@ -65,6 +66,7 @@ pub const DEFAULT_OBS_LOG_DELETE_EMPTY_FILES: bool = true; pub const DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS: u64 = 3600; // 1 hour pub const DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS: u64 = 6 * 3600; // 6 hours pub const DEFAULT_OBS_LOG_DRY_RUN: bool = false; +pub const DEFAULT_OBS_LOG_MATCH_MODE: &str = "suffix"; /// Default values for observability configuration // ### Supported Environment Values @@ -118,6 +120,7 @@ mod tests { assert_eq!(ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS"); assert_eq!(ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS"); assert_eq!(ENV_OBS_LOG_DRY_RUN, "RUSTFS_OBS_LOG_DRY_RUN"); + assert_eq!(ENV_OBS_LOG_MATCH_MODE, "RUSTFS_OBS_LOG_MATCH_MODE"); } #[test] @@ -126,5 +129,6 @@ mod tests { assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development"); assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test"); assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging"); + assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE, "suffix"); } } diff --git a/crates/obs/src/log_cleanup/compress.rs b/crates/obs/src/cleaner/compress.rs similarity index 100% rename from crates/obs/src/log_cleanup/compress.rs rename to crates/obs/src/cleaner/compress.rs diff --git a/crates/obs/src/log_cleanup/cleaner.rs b/crates/obs/src/cleaner/core.rs similarity index 56% rename from crates/obs/src/log_cleanup/cleaner.rs rename to crates/obs/src/cleaner/core.rs index 0026e496b..955679536 100644 --- a/crates/obs/src/log_cleanup/cleaner.rs +++ b/crates/obs/src/cleaner/core.rs @@ -25,7 +25,7 @@ use super::compress::compress_file; use super::scanner::{collect_expired_compressed_files, collect_log_files}; -use super::types::FileInfo; +use super::types::{FileInfo, FileMatchMode}; use std::path::PathBuf; use tracing::{debug, error, info}; @@ -41,8 +41,10 @@ use tracing::{debug, error, info}; pub struct LogCleaner { /// Directory containing the managed log files. pub(super) log_dir: PathBuf, - /// Filename prefix that identifies managed files (e.g. `"rustfs.log."`). - pub(super) file_prefix: String, + /// Pattern string to match files (used as prefix or suffix). + pub(super) file_pattern: String, + /// Whether to match by prefix or suffix. + pub(super) match_mode: FileMatchMode, /// The cleaner will never delete files if doing so would leave fewer than /// this many files in the directory. pub(super) keep_count: usize, @@ -77,7 +79,8 @@ impl LogCleaner { #[allow(clippy::too_many_arguments)] pub fn new( log_dir: PathBuf, - file_prefix: String, + file_pattern: String, + match_mode: FileMatchMode, keep_count: usize, max_total_size_bytes: u64, max_single_file_size_bytes: u64, @@ -96,7 +99,8 @@ impl LogCleaner { Self { log_dir, - file_prefix, + file_pattern, + match_mode, keep_count, max_total_size_bytes, max_single_file_size_bytes, @@ -136,7 +140,8 @@ impl LogCleaner { // ── 1. Discover active log files ────────────────────────────────────── let mut files = collect_log_files( &self.log_dir, - &self.file_prefix, + &self.file_pattern, + self.match_mode, &self.exclude_patterns, self.min_file_age_seconds, self.delete_empty_files, @@ -156,7 +161,15 @@ impl LogCleaner { ); // ── 2. Select + compress + delete ───────────────────────────────── - let to_delete = self.select_files_to_delete(&files, total_size); + let (to_delete, to_rotate) = self.select_files_to_process(&files, total_size); + + // Handle rotation for active file if needed + if let Some(active_file) = to_rotate + && let Err(e) = self.rotate_active_file(&active_file) + { + error!("Failed to rotate active file {:?}: {}", active_file.path, e); + } + if !to_delete.is_empty() { let (d, f) = self.compress_and_delete(&to_delete)?; total_deleted += d; @@ -165,7 +178,12 @@ impl LogCleaner { } // ── 3. Remove expired compressed archives ───────────────────────────── - let expired_gz = collect_expired_compressed_files(&self.log_dir, &self.file_prefix, self.compressed_file_retention_days)?; + let expired_gz = collect_expired_compressed_files( + &self.log_dir, + &self.file_pattern, + self.match_mode, + self.compressed_file_retention_days, + )?; if !expired_gz.is_empty() { let (d, f) = self.delete_files(&expired_gz)?; total_deleted += d; @@ -186,51 +204,180 @@ impl LogCleaner { // ─── Selection ──────────────────────────────────────────────────────────── - /// Choose which files from `files` (sorted oldest-first) should be deleted. + /// Choose which files from `files` (sorted oldest-first) should be deleted or rotated. /// /// The algorithm respects three constraints in order: /// 1. Always keep at least `keep_count` files. /// 2. Delete old files while the total size exceeds `max_total_size_bytes`. /// 3. Delete any file whose individual size exceeds `max_single_file_size_bytes`. - pub(super) fn select_files_to_delete(&self, files: &[FileInfo], total_size: u64) -> Vec { + /// + /// **Note**: The most recent file (assumed to be the active log) is exempt + /// from size-based deletion. If it exceeds the size limit, it is returned + /// as `to_rotate`. + pub(super) fn select_files_to_process(&self, files: &[FileInfo], total_size: u64) -> (Vec, Option) { let mut to_delete = Vec::new(); + let mut to_rotate = None; - if files.len() <= self.keep_count { - return to_delete; + if files.is_empty() { + return (to_delete, to_rotate); } + // Identify the index of the most recent file (last in the sorted list). + // We will protect this file from size-based deletion. + let active_file_idx = files.len() - 1; + + // The number of files we are allowed to delete. + // Any file with index >= max_deletable_count is protected by keep_count. + let max_deletable_count = files.len().saturating_sub(self.keep_count); + let mut current_size = total_size; - let deletable = files.len() - self.keep_count; for (idx, file) in files.iter().enumerate() { - if idx >= deletable { - break; + // If we are in the protected range, we stop deleting. + if idx >= max_deletable_count { + // However, if the active file is too large, we might rotate it. + if idx == active_file_idx { + let over_single = self.max_single_file_size_bytes > 0 && file.size > self.max_single_file_size_bytes; + if over_single { + to_rotate = Some(file.clone()); + } + } + continue; } + // We are in the deletable range. Check if we *should* delete. + + // Condition 2: Enforce max_total_size_bytes. let over_total = self.max_total_size_bytes > 0 && current_size > self.max_total_size_bytes; + + // Condition 3: Enforce max_single_file_size_bytes. let over_single = self.max_single_file_size_bytes > 0 && file.size > self.max_single_file_size_bytes; - if over_total || over_single { - if over_single { + if over_total { + // If we are over total size, we delete unless it's the active file. + if idx == active_file_idx { + debug!( + "Active log file contributes to total size limit overflow, but skipping deletion to preserve current logs." + ); + } else { + current_size = current_size.saturating_sub(file.size); + to_delete.push(file.clone()); + } + } else if over_single { + // For single file limits, we MUST NOT delete the active file. + if idx == active_file_idx { + // Mark active file for rotation instead of deletion + to_rotate = Some(file.clone()); + } else { debug!( "File exceeds single-file size limit: {:?} ({} > {} bytes)", file.path, file.size, self.max_single_file_size_bytes ); + current_size = current_size.saturating_sub(file.size); + to_delete.push(file.clone()); } - current_size = current_size.saturating_sub(file.size); - to_delete.push(file.clone()); - } else { - // Neither limit is breached; stop scanning. - break; } } - to_delete + (to_delete, to_rotate) + } + + // ─── Rotation ───────────────────────────────────────────────────────────── + + /// Rotate the active file by renaming it with a timestamp suffix. + /// The original filename will be recreated by the logging appender on next write. + fn rotate_active_file(&self, file: &FileInfo) -> Result<(), std::io::Error> { + if self.dry_run { + info!("[DRY RUN] Would rotate active file: {:?} ({} bytes)", file.path, file.size); + return Ok(()); + } + + // Generate timestamp: unix timestamp in seconds + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(std::io::Error::other)? + .as_secs(); + + let file_name = file + .path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid filename"))?; + + // Construct the rotated filename. + // We must ensure the new filename still matches the file_pattern so it can be discovered + // by the scanner in future runs (and eventually deleted). + // + // Suffix mode: Insert timestamp BEFORE the suffix. + // Example: "2026-03-01.rustfs.log" (pattern="rustfs.log") + // -> "2026-03-01.1740810000.rustfs.log" + // + // Prefix mode: Append timestamp at the end. + // Example: "app.log" (pattern="app") + // -> "app.log.1740810000" + let rotated_name = match self.match_mode { + FileMatchMode::Suffix => { + if let Some(base) = file_name.strip_suffix(&self.file_pattern) { + let mut new_name = String::with_capacity(file_name.len() + 20); + new_name.push_str(base); + + // Ensure separator between base and timestamp + if !base.is_empty() && !base.ends_with('.') { + new_name.push('.'); + } + new_name.push_str(×tamp.to_string()); + + // Ensure separator between timestamp and suffix + if !self.file_pattern.starts_with('.') { + new_name.push('.'); + } + new_name.push_str(&self.file_pattern); + + new_name + } else { + // Should not happen if scanner works correctly, but fallback safely + format!("{}.{}", file_name, timestamp) + } + } + FileMatchMode::Prefix => { + // For prefix matching, appending to the end preserves the prefix. + format!("{}.{}", file_name, timestamp) + } + }; + + let rotated_path = file.path.with_file_name(&rotated_name); + + // Check if target already exists to avoid overwriting (unlikely with timestamp but possible) + if rotated_path.exists() { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!("Rotated file already exists: {:?}", rotated_path), + )); + } + + info!("Rotating active log file: {:?} -> {:?}", file.path, rotated_path); + + // Rename the current active file to the rotated name. + // The logging appender (tracing-appender) will automatically create a new file + // with the original name when it next attempts to write. + // Note: On Linux/Unix, this rename is atomic and safe even if the file is open. + if let Err(e) = std::fs::rename(&file.path, &rotated_path) { + // Add context to the error + return Err(std::io::Error::new( + e.kind(), + format!("Failed to rename {:?} to {:?}: {}", file.path, rotated_path, e), + )); + } + + Ok(()) } // ─── Compression + deletion ─────────────────────────────────────────────── /// Optionally compress and then delete the given files. + /// + /// This function is synchronous and blocking. It should be called within a + /// `spawn_blocking` task if running in an async context. fn compress_and_delete(&self, files: &[FileInfo]) -> Result<(usize, u64), std::io::Error> { if self.compress_old_files { for f in files { diff --git a/crates/obs/src/log_cleanup/mod.rs b/crates/obs/src/cleaner/mod.rs similarity index 80% rename from crates/obs/src/log_cleanup/mod.rs rename to crates/obs/src/cleaner/mod.rs index 3644bcb3f..8404ce85a 100644 --- a/crates/obs/src/log_cleanup/mod.rs +++ b/crates/obs/src/cleaner/mod.rs @@ -24,17 +24,19 @@ //! | `types` | Shared data types (`FileInfo`) | //! | `scanner` | Filesystem traversal — discovers eligible files | //! | `compress` | Gzip compression helper | -//! | `cleaner` | Core orchestration — selection, compression, deletion | +//! | `core` | Core orchestration — selection, compression, deletion | //! //! ## Usage //! //! ```no_run //! use std::path::PathBuf; //! use rustfs_obs::LogCleaner; +//! use rustfs_obs::types::FileMatchMode; //! //! let cleaner = LogCleaner::new( //! PathBuf::from("/var/log/rustfs"), //! "rustfs.log.".to_string(), +//! FileMatchMode::Prefix, //! 10, // keep_count //! 2 * 1024 * 1024 * 1024, // max_total_size_bytes (2 GiB) //! 0, // max_single_file_size_bytes (unlimited) @@ -51,17 +53,18 @@ //! println!("Deleted {deleted} files, freed {freed_bytes} bytes"); //! ``` -mod cleaner; mod compress; +mod core; mod scanner; -mod types; +pub mod types; -pub use cleaner::LogCleaner; +pub use core::LogCleaner; #[cfg(test)] mod tests { - use super::cleaner::LogCleaner; + use super::core::LogCleaner; use super::scanner; + use super::types::FileMatchMode; use std::fs::File; use std::io::Write; use std::path::Path; @@ -79,6 +82,7 @@ mod tests { LogCleaner::new( dir, "app.log.".to_string(), + FileMatchMode::Prefix, keep, max_bytes, 0, // max_single_file_size_bytes @@ -153,7 +157,7 @@ mod tests { create_log_file(&dir, "app.log.2024-01-02", 2048)?; create_log_file(&dir, "other.log", 512)?; - let files = scanner::collect_log_files(&dir, "app.log.", &[], 0, true, false)?; + let files = scanner::collect_log_files(&dir, "app.log.", FileMatchMode::Prefix, &[], 0, true, false)?; assert_eq!(files.len(), 2, "scanner should find exactly 2 managed files"); Ok(()) } @@ -167,7 +171,21 @@ mod tests { create_log_file(&dir, "app.log.2024-01-02", 1024)?; create_log_file(&dir, "app.log.2024-01-03", 1024)?; - let cleaner = LogCleaner::new(dir.clone(), "app.log.".to_string(), 1, 1024, 0, false, 6, 30, vec![], true, 0, true); + let cleaner = LogCleaner::new( + dir.clone(), + "app.log.".to_string(), + FileMatchMode::Prefix, + 1, + 1024, + 0, + false, + 6, + 30, + vec![], + true, + 0, + true, + ); let (deleted, _freed) = cleaner.cleanup()?; // dry_run=true reports deletions but doesn't actually remove files. @@ -175,4 +193,35 @@ mod tests { assert_eq!(std::fs::read_dir(&dir)?.count(), 3, "no files should actually be removed"); Ok(()) } + + #[test] + fn test_cleanup_suffix_matching() -> std::io::Result<()> { + let tmp = TempDir::new()?; + let dir = tmp.path().to_path_buf(); + + create_log_file(&dir, "2026-03-01-06-21.rustfs.log", 1024)?; + create_log_file(&dir, "2026-03-01-06-22.rustfs.log", 1024)?; + create_log_file(&dir, "other.log", 1024)?; // not managed + + let cleaner = LogCleaner::new( + dir.clone(), + "rustfs.log".to_string(), + FileMatchMode::Suffix, + 1, + 1024, + 0, + false, + 6, + 30, + vec![], + true, + 0, + false, + ); + let (deleted, freed) = cleaner.cleanup()?; + + assert_eq!(deleted, 1, "should delete exactly one file"); + assert_eq!(freed, 1024); + Ok(()) + } } diff --git a/crates/obs/src/log_cleanup/scanner.rs b/crates/obs/src/cleaner/scanner.rs similarity index 83% rename from crates/obs/src/log_cleanup/scanner.rs rename to crates/obs/src/cleaner/scanner.rs index 305cae76f..8f6393745 100644 --- a/crates/obs/src/log_cleanup/scanner.rs +++ b/crates/obs/src/cleaner/scanner.rs @@ -17,13 +17,13 @@ //! This module is intentionally kept read-only: it does **not** delete or //! compress any files — it only reports what it found. -use super::types::FileInfo; +use super::types::{FileInfo, FileMatchMode}; use std::path::Path; use std::time::{Duration, SystemTime}; use tracing::debug; use walkdir::WalkDir; -/// Collect all log files in `log_dir` whose name starts with `file_prefix`. +/// Collect all log files in `log_dir` whose name matches `file_pattern` based on `match_mode`. /// /// Files that: /// - are already compressed (`.gz` extension), @@ -36,14 +36,16 @@ use walkdir::WalkDir; /// /// # Arguments /// * `log_dir` - Root directory to scan (depth 1 only, no recursion). -/// * `file_prefix` - Only filenames starting with this string are considered. +/// * `file_pattern` - Pattern string to match filenames. +/// * `match_mode` - Whether to match by prefix or suffix. /// * `exclude_patterns` - Compiled glob patterns; matching files are skipped. /// * `min_file_age_seconds` - Files younger than this threshold are skipped. /// * `delete_empty_files` - When `true`, zero-byte files trigger an immediate /// delete by the caller before the rest of cleanup runs. pub(super) fn collect_log_files( log_dir: &Path, - file_prefix: &str, + file_pattern: &str, + match_mode: FileMatchMode, exclude_patterns: &[glob::Pattern], min_file_age_seconds: u64, delete_empty_files: bool, @@ -68,8 +70,13 @@ pub(super) fn collect_log_files( None => continue, }; - // Only manage files that carry our prefix. - if !filename.starts_with(file_prefix) { + // Match filename based on mode + let matches = match match_mode { + FileMatchMode::Prefix => filename.starts_with(file_pattern), + FileMatchMode::Suffix => filename.ends_with(file_pattern), + }; + + if !matches { continue; } @@ -139,12 +146,14 @@ pub(super) fn collect_log_files( /// /// # Arguments /// * `log_dir` - Root directory to scan. -/// * `file_prefix` - Only `.gz` files that also start with this prefix are considered. +/// * `file_pattern` - Pattern string to match filenames. +/// * `match_mode` - Whether to match by prefix or suffix. /// * `compressed_file_retention_days` - Files older than this are eligible for /// deletion; `0` means never delete compressed files. pub(super) fn collect_expired_compressed_files( log_dir: &Path, - file_prefix: &str, + file_pattern: &str, + match_mode: FileMatchMode, compressed_file_retention_days: u64, ) -> Result, std::io::Error> { if compressed_file_retention_days == 0 { @@ -171,7 +180,18 @@ pub(super) fn collect_expired_compressed_files( None => continue, }; - if !filename.starts_with(file_prefix) || !filename.ends_with(".gz") { + if !filename.ends_with(".gz") { + continue; + } + + // Check if the base filename (without .gz) matches the pattern + let base_filename = &filename[..filename.len() - 3]; + let matches = match match_mode { + FileMatchMode::Prefix => base_filename.starts_with(file_pattern), + FileMatchMode::Suffix => base_filename.ends_with(file_pattern), + }; + + if !matches { continue; } diff --git a/crates/obs/src/log_cleanup/types.rs b/crates/obs/src/cleaner/types.rs similarity index 78% rename from crates/obs/src/log_cleanup/types.rs rename to crates/obs/src/cleaner/types.rs index f9b999fc0..ff1b8b54e 100644 --- a/crates/obs/src/log_cleanup/types.rs +++ b/crates/obs/src/cleaner/types.rs @@ -17,6 +17,15 @@ use std::path::PathBuf; use std::time::SystemTime; +/// Strategy for matching log files against a pattern. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileMatchMode { + /// The filename must start with the pattern (e.g. "app.log." matches "app.log.2024-01-01"). + Prefix, + /// The filename must end with the pattern (e.g. ".log" matches "2024-01-01.log"). + Suffix, +} + /// Metadata for a single log file discovered by the scanner. /// /// Carries enough information to make cleanup decisions (sort by age, compare diff --git a/crates/obs/src/config.rs b/crates/obs/src/config.rs index 6c6165429..a4fc209f0 100644 --- a/crates/obs/src/config.rs +++ b/crates/obs/src/config.rs @@ -25,16 +25,16 @@ use rustfs_config::observability::{ DEFAULT_OBS_ENVIRONMENT_PRODUCTION, DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS, DEFAULT_OBS_LOG_COMPRESS_OLD_FILES, DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, DEFAULT_OBS_LOG_DELETE_EMPTY_FILES, DEFAULT_OBS_LOG_DRY_RUN, - DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_KEEP_COUNT, DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, - DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_ENDPOINT, ENV_OBS_ENVIRONMENT, - ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, ENV_OBS_LOG_COMPRESS_OLD_FILES, ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, - ENV_OBS_LOG_DELETE_EMPTY_FILES, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN, ENV_OBS_LOG_ENDPOINT, - ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL, ENV_OBS_LOG_KEEP_COUNT, - ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, - ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOG_STDOUT_ENABLED, ENV_OBS_LOGGER_LEVEL, - ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT, ENV_OBS_METRICS_EXPORT_ENABLED, - ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED, - ENV_OBS_USE_STDOUT, + DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_KEEP_COUNT, DEFAULT_OBS_LOG_MATCH_MODE, + DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS, + ENV_OBS_ENDPOINT, ENV_OBS_ENVIRONMENT, ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, ENV_OBS_LOG_COMPRESS_OLD_FILES, + ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, ENV_OBS_LOG_DELETE_EMPTY_FILES, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN, + ENV_OBS_LOG_ENDPOINT, ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL, + ENV_OBS_LOG_KEEP_COUNT, ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, + ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOG_STDOUT_ENABLED, + ENV_OBS_LOGGER_LEVEL, ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT, + ENV_OBS_METRICS_EXPORT_ENABLED, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT, + ENV_OBS_TRACES_EXPORT_ENABLED, ENV_OBS_USE_STDOUT, }; use rustfs_config::{ APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_FILENAME, @@ -157,6 +157,8 @@ pub struct OtelConfig { /// Log what *would* be deleted without actually removing anything /// (default: `false`). pub log_dry_run: Option, + /// File matching mode: "prefix" or "suffix" (default: "suffix"). + pub log_match_mode: Option, } impl OtelConfig { @@ -259,6 +261,7 @@ impl OtelConfig { DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS, )), log_dry_run: Some(get_env_bool(ENV_OBS_LOG_DRY_RUN, DEFAULT_OBS_LOG_DRY_RUN)), + log_match_mode: Some(get_env_str(ENV_OBS_LOG_MATCH_MODE, DEFAULT_OBS_LOG_MATCH_MODE)), } } diff --git a/crates/obs/src/lib.rs b/crates/obs/src/lib.rs index 0c3a8c847..08338d525 100644 --- a/crates/obs/src/lib.rs +++ b/crates/obs/src/lib.rs @@ -53,16 +53,16 @@ //! # // Guard will be dropped here, flushing telemetry data //! # } //! ``` +mod cleaner; mod config; mod error; mod global; -mod log_cleanup; mod system; mod telemetry; +pub use cleaner::*; pub use config::*; pub use error::*; pub use global::*; -pub use log_cleanup::*; pub use system::SystemObserver; pub use telemetry::{OtelGuard, Recorder}; diff --git a/crates/obs/src/telemetry/local.rs b/crates/obs/src/telemetry/local.rs index 3d76402dd..ed507e091 100644 --- a/crates/obs/src/telemetry/local.rs +++ b/crates/obs/src/telemetry/local.rs @@ -27,9 +27,10 @@ //! cases; callers do **not** need to distinguish between stdout and file modes. use crate::TelemetryError; +use crate::cleaner::LogCleaner; +use crate::cleaner::types::FileMatchMode; use crate::config::OtelConfig; use crate::global::OBSERVABILITY_METRIC_ENABLED; -use crate::log_cleanup::LogCleaner; use crate::telemetry::filter::build_env_filter; use metrics::counter; use rustfs_config::observability::{ @@ -38,6 +39,7 @@ use rustfs_config::observability::{ DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS, }; use rustfs_config::{APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_STDOUT_ENABLED}; +use std::sync::Arc; use std::{fs, io::IsTerminal, time::Duration}; use tracing::info; use tracing_error::ErrorLayer; @@ -176,6 +178,13 @@ fn init_file_logging_internal( .as_deref() .unwrap_or(DEFAULT_LOG_ROTATION_TIME) .to_lowercase(); + + // Determine match mode from config, defaulting to Suffix + let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() { + Some("prefix") => FileMatchMode::Prefix, + _ => FileMatchMode::Suffix, + }; + use tracing_appender::rolling::{RollingFileAppender, Rotation}; let file_appender = { let rotation = match rotation.as_str() { @@ -183,10 +192,17 @@ fn init_file_logging_internal( "hourly" => Rotation::HOURLY, _ => Rotation::DAILY, }; - RollingFileAppender::builder() + + let mut builder = RollingFileAppender::builder() .rotation(rotation) - .filename_suffix(log_filename) - .max_log_files(keep_files) + .max_log_files(keep_files * 2); // Make sure there are some data files to archive to avoid premature deletion + + match match_mode { + FileMatchMode::Prefix => builder = builder.filename_prefix(log_filename), + FileMatchMode::Suffix => builder = builder.filename_suffix(log_filename), + } + + builder .build(log_directory) .expect("failed to initialize rolling file appender") }; @@ -329,7 +345,16 @@ fn spawn_cleanup_task( keep_files: usize, ) -> tokio::task::JoinHandle<()> { let log_dir = std::path::PathBuf::from(log_directory); - let file_prefix = config.log_filename.as_deref().unwrap_or(log_filename).to_string(); + // Use suffix matching for log files like "2026-03-01-06-21.rustfs.log" + // where "rustfs.log" is the suffix. + let file_pattern = config.log_filename.as_deref().unwrap_or(log_filename).to_string(); + + // Determine match mode from config, defaulting to Suffix + let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() { + Some("prefix") => FileMatchMode::Prefix, + _ => FileMatchMode::Suffix, + }; + let keep_count = config.log_keep_count.unwrap_or(keep_files); let max_total_size = config .log_max_total_size_bytes @@ -358,9 +383,10 @@ fn spawn_cleanup_task( .log_cleanup_interval_seconds .unwrap_or(DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS); - let cleaner = LogCleaner::new( + let cleaner = Arc::new(LogCleaner::new( log_dir, - file_prefix, + file_pattern, + match_mode, keep_count, max_total_size, max_single_file_size, @@ -371,14 +397,19 @@ fn spawn_cleanup_task( delete_empty, min_age, dry_run, - ); + )); tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(cleanup_interval)); loop { interval.tick().await; - if let Err(e) = cleaner.cleanup() { - tracing::warn!("Log cleanup failed: {}", e); + let cleaner_clone = cleaner.clone(); + let result = tokio::task::spawn_blocking(move || cleaner_clone.cleanup()).await; + + match result { + Ok(Ok(_)) => {} // Success + Ok(Err(e)) => tracing::warn!("Log cleanup failed: {}", e), + Err(e) => tracing::warn!("Log cleanup task panicked: {}", e), } } }) diff --git a/scripts/run.sh b/scripts/run.sh index b5b898fff..01179f450 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -79,6 +79,7 @@ export RUSTFS_OBS_LOG_FLUSH_MS=300 # Log flush interval in milliseconds export RUSTFS_RUNTIME_WORKER_THREADS=16 export RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024 export RUSTFS_RUNTIME_THREAD_PRINT_ENABLED=false +export RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS=300 # shellcheck disable=SC2125 export RUSTFS_RUNTIME_THREAD_STACK_SIZE=1024*1024 export RUSTFS_RUNTIME_THREAD_KEEP_ALIVE=60