gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimize log cleanup and rotation, update dependencies (#2032)

Browse Source 
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-03-01 20:09:52 +08:00
committed by GitHub
parent 798e620088
commit c452f24487
13 changed files with 335 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -43,3 +43,4 @@ docs
# nix stuff
result*
*.gz
Cargo.lock
Generated
+7 -6
View File
@@ -4529,9 +4529,9 @@ dependencies = [
[[package]]
name = "jiff"
version = "0.2.21"
version = "0.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3e3d65f018c6ae946ab16e80944b97096ed73c35b221d1c478a6c81d8f57940"
checksum = "819b44bc7c87d9117eb522f14d46e918add69ff12713c475946b0a29363ed1c2"
dependencies = [
"jiff-static",
"jiff-tzdb-platform",
@@ -4544,9 +4544,9 @@ dependencies = [
[[package]]
name = "jiff-static"
version = "0.2.21"
version = "0.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a17c2b211d863c7fde02cbea8a3c1a439b98e109286554f2860bdded7ff83818"
checksum = "470252db18ecc35fd766c0891b1e3ec6cbbcd62507e85276c01bf75d8e94d4a1"
dependencies = [
"proc-macro2",
"quote",
@@ -8001,8 +8001,9 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "s3s"
version = "0.13.0-alpha.3"
source = "git+https://github.com/s3s-project/s3s.git?rev=3600aad5bcd691683e99d1e5fa17bae3bc21791d#3600aad5bcd691683e99d1e5fa17bae3bc21791d"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcf5572ca9bae5fd92d4e5a3e934c73793ab743bcbc761cd5e86a622c2a5e98a"
dependencies = [
"arc-swap",
"arrayvec",
Cargo.toml
+2 -2
View File
@@ -171,7 +171,7 @@ zeroize = { version = "1.8.2", features = ["derive"] }
# Time and Date
chrono = { version = "0.4.44", features = ["serde"] }
humantime = "2.3.0"
jiff = { version = "0.2.21", features = ["serde"] }
jiff = { version = "0.2.22", features = ["serde"] }
time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
# Utilities and Tools
@@ -235,7 +235,7 @@ rumqttc = { version = "0.25.1" }
rustix = { version = "1.1.4", features = ["fs"] }
rust-embed = { version = "8.11.0" }
rustc-hash = { version = "2.1.1" }
s3s = { version = "0.13.0-alpha.3", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", rev = "3600aad5bcd691683e99d1e5fa17bae3bc21791d" }
s3s = { version = "0.13.0", features = ["minio"] }
serial_test = "3.4.0"
shadow-rs = { version = "1.7.0", default-features = false }
siphasher = "1.0.2"
crates/config/src/observability/mod.rs
+4
View File
@@ -53,6 +53,7 @@ pub const ENV_OBS_LOG_DELETE_EMPTY_FILES: &str = "RUSTFS_OBS_LOG_DELETE_EMPTY_FI
pub const ENV_OBS_LOG_MIN_FILE_AGE_SECONDS: &str = "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS";
pub const ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS: &str = "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS";
pub const ENV_OBS_LOG_DRY_RUN: &str = "RUSTFS_OBS_LOG_DRY_RUN";
pub const ENV_OBS_LOG_MATCH_MODE: &str = "RUSTFS_OBS_LOG_MATCH_MODE";
/// Default values for log cleanup
pub const DEFAULT_OBS_LOG_KEEP_COUNT: usize = 10;
@@ -65,6 +66,7 @@ pub const DEFAULT_OBS_LOG_DELETE_EMPTY_FILES: bool = true;
pub const DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS: u64 = 3600; // 1 hour
pub const DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS: u64 = 6 * 3600; // 6 hours
pub const DEFAULT_OBS_LOG_DRY_RUN: bool = false;
pub const DEFAULT_OBS_LOG_MATCH_MODE: &str = "suffix";
/// Default values for observability configuration
// ### Supported Environment Values
@@ -118,6 +120,7 @@ mod tests {
assert_eq!(ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, "RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS");
assert_eq!(ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, "RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS");
assert_eq!(ENV_OBS_LOG_DRY_RUN, "RUSTFS_OBS_LOG_DRY_RUN");
assert_eq!(ENV_OBS_LOG_MATCH_MODE, "RUSTFS_OBS_LOG_MATCH_MODE");
}
#[test]
@@ -126,5 +129,6 @@ mod tests {
assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development");
assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test");
assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging");
assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE, "suffix");
}
}
crates/obs/src/log_cleanup/compress.rs → crates/obs/src/cleaner/compress.rs
View File
crates/obs/src/log_cleanup/cleaner.rs → crates/obs/src/cleaner/core.rs
+170 -23
View File
@@ -25,7 +25,7 @@
use super::compress::compress_file;
use super::scanner::{collect_expired_compressed_files, collect_log_files};
use super::types::FileInfo;
use super::types::{FileInfo, FileMatchMode};
use std::path::PathBuf;
use tracing::{debug, error, info};
@@ -41,8 +41,10 @@ use tracing::{debug, error, info};
pub struct LogCleaner {
/// Directory containing the managed log files.
pub(super) log_dir: PathBuf,
/// Filename prefix that identifies managed files (e.g. `"rustfs.log."`).
pub(super) file_prefix: String,
/// Pattern string to match files (used as prefix or suffix).
pub(super) file_pattern: String,
/// Whether to match by prefix or suffix.
pub(super) match_mode: FileMatchMode,
/// The cleaner will never delete files if doing so would leave fewer than
/// this many files in the directory.
pub(super) keep_count: usize,
@@ -77,7 +79,8 @@ impl LogCleaner {
#[allow(clippy::too_many_arguments)]
pub fn new(
log_dir: PathBuf,
file_prefix: String,
file_pattern: String,
match_mode: FileMatchMode,
keep_count: usize,
max_total_size_bytes: u64,
max_single_file_size_bytes: u64,
@@ -96,7 +99,8 @@ impl LogCleaner {
Self {
log_dir,
file_prefix,
file_pattern,
match_mode,
keep_count,
max_total_size_bytes,
max_single_file_size_bytes,
@@ -136,7 +140,8 @@ impl LogCleaner {
// ── 1. Discover active log files ──────────────────────────────────────
let mut files = collect_log_files(
&self.log_dir,
&self.file_prefix,
&self.file_pattern,
self.match_mode,
&self.exclude_patterns,
self.min_file_age_seconds,
self.delete_empty_files,
@@ -156,7 +161,15 @@ impl LogCleaner {
);
// ── 2. Select + compress + delete ─────────────────────────────────
let to_delete = self.select_files_to_delete(&files, total_size);
let (to_delete, to_rotate) = self.select_files_to_process(&files, total_size);
// Handle rotation for active file if needed
if let Some(active_file) = to_rotate
&& let Err(e) = self.rotate_active_file(&active_file)
{
error!("Failed to rotate active file {:?}: {}", active_file.path, e);
}
if !to_delete.is_empty() {
let (d, f) = self.compress_and_delete(&to_delete)?;
total_deleted += d;
@@ -165,7 +178,12 @@ impl LogCleaner {
}
// ── 3. Remove expired compressed archives ─────────────────────────────
let expired_gz = collect_expired_compressed_files(&self.log_dir, &self.file_prefix, self.compressed_file_retention_days)?;
let expired_gz = collect_expired_compressed_files(
&self.log_dir,
&self.file_pattern,
self.match_mode,
self.compressed_file_retention_days,
)?;
if !expired_gz.is_empty() {
let (d, f) = self.delete_files(&expired_gz)?;
total_deleted += d;
@@ -186,51 +204,180 @@ impl LogCleaner {
// ─── Selection ────────────────────────────────────────────────────────────
/// Choose which files from `files` (sorted oldest-first) should be deleted.
/// Choose which files from `files` (sorted oldest-first) should be deleted or rotated.
///
/// The algorithm respects three constraints in order:
/// 1. Always keep at least `keep_count` files.
/// 2. Delete old files while the total size exceeds `max_total_size_bytes`.
/// 3. Delete any file whose individual size exceeds `max_single_file_size_bytes`.
pub(super) fn select_files_to_delete(&self, files: &[FileInfo], total_size: u64) -> Vec<FileInfo> {
///
/// **Note**: The most recent file (assumed to be the active log) is exempt
/// from size-based deletion. If it exceeds the size limit, it is returned
/// as `to_rotate`.
pub(super) fn select_files_to_process(&self, files: &[FileInfo], total_size: u64) -> (Vec<FileInfo>, Option<FileInfo>) {
let mut to_delete = Vec::new();
let mut to_rotate = None;
if files.len() <= self.keep_count {
return to_delete;
if files.is_empty() {
return (to_delete, to_rotate);
}
// Identify the index of the most recent file (last in the sorted list).
// We will protect this file from size-based deletion.
let active_file_idx = files.len() - 1;
// The number of files we are allowed to delete.
// Any file with index >= max_deletable_count is protected by keep_count.
let max_deletable_count = files.len().saturating_sub(self.keep_count);
let mut current_size = total_size;
let deletable = files.len() - self.keep_count;
for (idx, file) in files.iter().enumerate() {
if idx >= deletable {
break;
// If we are in the protected range, we stop deleting.
if idx >= max_deletable_count {
// However, if the active file is too large, we might rotate it.
if idx == active_file_idx {
let over_single = self.max_single_file_size_bytes > 0 && file.size > self.max_single_file_size_bytes;
if over_single {
to_rotate = Some(file.clone());
}
}
continue;
}
// We are in the deletable range. Check if we *should* delete.
// Condition 2: Enforce max_total_size_bytes.
let over_total = self.max_total_size_bytes > 0 && current_size > self.max_total_size_bytes;
// Condition 3: Enforce max_single_file_size_bytes.
let over_single = self.max_single_file_size_bytes > 0 && file.size > self.max_single_file_size_bytes;
if over_total || over_single {
if over_single {
if over_total {
// If we are over total size, we delete unless it's the active file.
if idx == active_file_idx {
debug!(
"Active log file contributes to total size limit overflow, but skipping deletion to preserve current logs."
);
} else {
current_size = current_size.saturating_sub(file.size);
to_delete.push(file.clone());
}
} else if over_single {
// For single file limits, we MUST NOT delete the active file.
if idx == active_file_idx {
// Mark active file for rotation instead of deletion
to_rotate = Some(file.clone());
} else {
debug!(
"File exceeds single-file size limit: {:?} ({} > {} bytes)",
file.path, file.size, self.max_single_file_size_bytes
);
current_size = current_size.saturating_sub(file.size);
to_delete.push(file.clone());
}
current_size = current_size.saturating_sub(file.size);
to_delete.push(file.clone());
} else {
// Neither limit is breached; stop scanning.
break;
}
}
to_delete
(to_delete, to_rotate)
}
// ─── Rotation ─────────────────────────────────────────────────────────────
/// Rotate the active file by renaming it with a timestamp suffix.
/// The original filename will be recreated by the logging appender on next write.
fn rotate_active_file(&self, file: &FileInfo) -> Result<(), std::io::Error> {
if self.dry_run {
info!("[DRY RUN] Would rotate active file: {:?} ({} bytes)", file.path, file.size);
return Ok(());
}
// Generate timestamp: unix timestamp in seconds
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_err(std::io::Error::other)?
.as_secs();
let file_name = file
.path
.file_name()
.and_then(|n| n.to_str())
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid filename"))?;
// Construct the rotated filename.
// We must ensure the new filename still matches the file_pattern so it can be discovered
// by the scanner in future runs (and eventually deleted).
//
// Suffix mode: Insert timestamp BEFORE the suffix.
// Example: "2026-03-01.rustfs.log" (pattern="rustfs.log")
// -> "2026-03-01.1740810000.rustfs.log"
//
// Prefix mode: Append timestamp at the end.
// Example: "app.log" (pattern="app")
// -> "app.log.1740810000"
let rotated_name = match self.match_mode {
FileMatchMode::Suffix => {
if let Some(base) = file_name.strip_suffix(&self.file_pattern) {
let mut new_name = String::with_capacity(file_name.len() + 20);
new_name.push_str(base);
// Ensure separator between base and timestamp
if !base.is_empty() && !base.ends_with('.') {
new_name.push('.');
}
new_name.push_str(&timestamp.to_string());
// Ensure separator between timestamp and suffix
if !self.file_pattern.starts_with('.') {
new_name.push('.');
}
new_name.push_str(&self.file_pattern);
new_name
} else {
// Should not happen if scanner works correctly, but fallback safely
format!("{}.{}", file_name, timestamp)
}
}
FileMatchMode::Prefix => {
// For prefix matching, appending to the end preserves the prefix.
format!("{}.{}", file_name, timestamp)
}
};
let rotated_path = file.path.with_file_name(&rotated_name);
// Check if target already exists to avoid overwriting (unlikely with timestamp but possible)
if rotated_path.exists() {
return Err(std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
format!("Rotated file already exists: {:?}", rotated_path),
));
}
info!("Rotating active log file: {:?} -> {:?}", file.path, rotated_path);
// Rename the current active file to the rotated name.
// The logging appender (tracing-appender) will automatically create a new file
// with the original name when it next attempts to write.
// Note: On Linux/Unix, this rename is atomic and safe even if the file is open.
if let Err(e) = std::fs::rename(&file.path, &rotated_path) {
// Add context to the error
return Err(std::io::Error::new(
e.kind(),
format!("Failed to rename {:?} to {:?}: {}", file.path, rotated_path, e),
));
}
Ok(())
}
// ─── Compression + deletion ───────────────────────────────────────────────
/// Optionally compress and then delete the given files.
///
/// This function is synchronous and blocking. It should be called within a
/// `spawn_blocking` task if running in an async context.
fn compress_and_delete(&self, files: &[FileInfo]) -> Result<(usize, u64), std::io::Error> {
if self.compress_old_files {
for f in files {
crates/obs/src/log_cleanup/mod.rs → crates/obs/src/cleaner/mod.rs
+56 -7
View File
@@ -24,17 +24,19 @@
//! | `types` | Shared data types (`FileInfo`) |
//! | `scanner` | Filesystem traversal — discovers eligible files |
//! | `compress` | Gzip compression helper |
//! | `cleaner` | Core orchestration — selection, compression, deletion |
//! | `core` | Core orchestration — selection, compression, deletion |
//!
//! ## Usage
//!
//! ```no_run
//! use std::path::PathBuf;
//! use rustfs_obs::LogCleaner;
//! use rustfs_obs::types::FileMatchMode;
//!
//! let cleaner = LogCleaner::new(
//! PathBuf::from("/var/log/rustfs"),
//! "rustfs.log.".to_string(),
//! FileMatchMode::Prefix,
//! 10, // keep_count
//! 2 * 1024 * 1024 * 1024, // max_total_size_bytes (2 GiB)
//! 0, // max_single_file_size_bytes (unlimited)
@@ -51,17 +53,18 @@
//! println!("Deleted {deleted} files, freed {freed_bytes} bytes");
//! ```
mod cleaner;
mod compress;
mod core;
mod scanner;
mod types;
pub mod types;
pub use cleaner::LogCleaner;
pub use core::LogCleaner;
#[cfg(test)]
mod tests {
use super::cleaner::LogCleaner;
use super::core::LogCleaner;
use super::scanner;
use super::types::FileMatchMode;
use std::fs::File;
use std::io::Write;
use std::path::Path;
@@ -79,6 +82,7 @@ mod tests {
LogCleaner::new(
dir,
"app.log.".to_string(),
FileMatchMode::Prefix,
keep,
max_bytes,
0, // max_single_file_size_bytes
@@ -153,7 +157,7 @@ mod tests {
create_log_file(&dir, "app.log.2024-01-02", 2048)?;
create_log_file(&dir, "other.log", 512)?;
let files = scanner::collect_log_files(&dir, "app.log.", &[], 0, true, false)?;
let files = scanner::collect_log_files(&dir, "app.log.", FileMatchMode::Prefix, &[], 0, true, false)?;
assert_eq!(files.len(), 2, "scanner should find exactly 2 managed files");
Ok(())
}
@@ -167,7 +171,21 @@ mod tests {
create_log_file(&dir, "app.log.2024-01-02", 1024)?;
create_log_file(&dir, "app.log.2024-01-03", 1024)?;
let cleaner = LogCleaner::new(dir.clone(), "app.log.".to_string(), 1, 1024, 0, false, 6, 30, vec![], true, 0, true);
let cleaner = LogCleaner::new(
dir.clone(),
"app.log.".to_string(),
FileMatchMode::Prefix,
1,
1024,
0,
false,
6,
30,
vec![],
true,
0,
true,
);
let (deleted, _freed) = cleaner.cleanup()?;
// dry_run=true reports deletions but doesn't actually remove files.
@@ -175,4 +193,35 @@ mod tests {
assert_eq!(std::fs::read_dir(&dir)?.count(), 3, "no files should actually be removed");
Ok(())
}
#[test]
fn test_cleanup_suffix_matching() -> std::io::Result<()> {
let tmp = TempDir::new()?;
let dir = tmp.path().to_path_buf();
create_log_file(&dir, "2026-03-01-06-21.rustfs.log", 1024)?;
create_log_file(&dir, "2026-03-01-06-22.rustfs.log", 1024)?;
create_log_file(&dir, "other.log", 1024)?; // not managed
let cleaner = LogCleaner::new(
dir.clone(),
"rustfs.log".to_string(),
FileMatchMode::Suffix,
1,
1024,
0,
false,
6,
30,
vec![],
true,
0,
false,
);
let (deleted, freed) = cleaner.cleanup()?;
assert_eq!(deleted, 1, "should delete exactly one file");
assert_eq!(freed, 1024);
Ok(())
}
}
crates/obs/src/log_cleanup/scanner.rs → crates/obs/src/cleaner/scanner.rs
+29 -9
View File
@@ -17,13 +17,13 @@
//! This module is intentionally kept read-only: it does **not** delete or
//! compress any files — it only reports what it found.
use super::types::FileInfo;
use super::types::{FileInfo, FileMatchMode};
use std::path::Path;
use std::time::{Duration, SystemTime};
use tracing::debug;
use walkdir::WalkDir;
/// Collect all log files in `log_dir` whose name starts with `file_prefix`.
/// Collect all log files in `log_dir` whose name matches `file_pattern` based on `match_mode`.
///
/// Files that:
/// - are already compressed (`.gz` extension),
@@ -36,14 +36,16 @@ use walkdir::WalkDir;
///
/// # Arguments
/// * `log_dir` - Root directory to scan (depth 1 only, no recursion).
/// * `file_prefix` - Only filenames starting with this string are considered.
/// * `file_pattern` - Pattern string to match filenames.
/// * `match_mode` - Whether to match by prefix or suffix.
/// * `exclude_patterns` - Compiled glob patterns; matching files are skipped.
/// * `min_file_age_seconds` - Files younger than this threshold are skipped.
/// * `delete_empty_files` - When `true`, zero-byte files trigger an immediate
/// delete by the caller before the rest of cleanup runs.
pub(super) fn collect_log_files(
log_dir: &Path,
file_prefix: &str,
file_pattern: &str,
match_mode: FileMatchMode,
exclude_patterns: &[glob::Pattern],
min_file_age_seconds: u64,
delete_empty_files: bool,
@@ -68,8 +70,13 @@ pub(super) fn collect_log_files(
None => continue,
};
// Only manage files that carry our prefix.
if !filename.starts_with(file_prefix) {
// Match filename based on mode
let matches = match match_mode {
FileMatchMode::Prefix => filename.starts_with(file_pattern),
FileMatchMode::Suffix => filename.ends_with(file_pattern),
};
if !matches {
continue;
}
@@ -139,12 +146,14 @@ pub(super) fn collect_log_files(
///
/// # Arguments
/// * `log_dir` - Root directory to scan.
/// * `file_prefix` - Only `.gz` files that also start with this prefix are considered.
/// * `file_pattern` - Pattern string to match filenames.
/// * `match_mode` - Whether to match by prefix or suffix.
/// * `compressed_file_retention_days` - Files older than this are eligible for
/// deletion; `0` means never delete compressed files.
pub(super) fn collect_expired_compressed_files(
log_dir: &Path,
file_prefix: &str,
file_pattern: &str,
match_mode: FileMatchMode,
compressed_file_retention_days: u64,
) -> Result<Vec<FileInfo>, std::io::Error> {
if compressed_file_retention_days == 0 {
@@ -171,7 +180,18 @@ pub(super) fn collect_expired_compressed_files(
None => continue,
};
if !filename.starts_with(file_prefix) || !filename.ends_with(".gz") {
if !filename.ends_with(".gz") {
continue;
}
// Check if the base filename (without .gz) matches the pattern
let base_filename = &filename[..filename.len() - 3];
let matches = match match_mode {
FileMatchMode::Prefix => base_filename.starts_with(file_pattern),
FileMatchMode::Suffix => base_filename.ends_with(file_pattern),
};
if !matches {
continue;
}
crates/obs/src/log_cleanup/types.rs → crates/obs/src/cleaner/types.rs
+9
View File
@@ -17,6 +17,15 @@
use std::path::PathBuf;
use std::time::SystemTime;
/// Strategy for matching log files against a pattern.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FileMatchMode {
/// The filename must start with the pattern (e.g. "app.log." matches "app.log.2024-01-01").
Prefix,
/// The filename must end with the pattern (e.g. ".log" matches "2024-01-01.log").
Suffix,
}
/// Metadata for a single log file discovered by the scanner.
///
/// Carries enough information to make cleanup decisions (sort by age, compare
crates/obs/src/config.rs
+13 -10
View File
@@ -25,16 +25,16 @@
use rustfs_config::observability::{
DEFAULT_OBS_ENVIRONMENT_PRODUCTION, DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS, DEFAULT_OBS_LOG_COMPRESS_OLD_FILES,
DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, DEFAULT_OBS_LOG_DELETE_EMPTY_FILES, DEFAULT_OBS_LOG_DRY_RUN,
DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_KEEP_COUNT, DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES,
DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_ENDPOINT, ENV_OBS_ENVIRONMENT,
ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, ENV_OBS_LOG_COMPRESS_OLD_FILES, ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS,
ENV_OBS_LOG_DELETE_EMPTY_FILES, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN, ENV_OBS_LOG_ENDPOINT,
ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL, ENV_OBS_LOG_KEEP_COUNT,
ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES,
ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOG_STDOUT_ENABLED, ENV_OBS_LOGGER_LEVEL,
ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT, ENV_OBS_METRICS_EXPORT_ENABLED,
ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED,
ENV_OBS_USE_STDOUT,
DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_KEEP_COUNT, DEFAULT_OBS_LOG_MATCH_MODE,
DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS,
ENV_OBS_ENDPOINT, ENV_OBS_ENVIRONMENT, ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, ENV_OBS_LOG_COMPRESS_OLD_FILES,
ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, ENV_OBS_LOG_DELETE_EMPTY_FILES, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN,
ENV_OBS_LOG_ENDPOINT, ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL,
ENV_OBS_LOG_KEEP_COUNT, ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES,
ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOG_STDOUT_ENABLED,
ENV_OBS_LOGGER_LEVEL, ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT,
ENV_OBS_METRICS_EXPORT_ENABLED, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT,
ENV_OBS_TRACES_EXPORT_ENABLED, ENV_OBS_USE_STDOUT,
};
use rustfs_config::{
APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_FILENAME,
@@ -157,6 +157,8 @@ pub struct OtelConfig {
/// Log what *would* be deleted without actually removing anything
/// (default: `false`).
pub log_dry_run: Option<bool>,
/// File matching mode: "prefix" or "suffix" (default: "suffix").
pub log_match_mode: Option<String>,
}
impl OtelConfig {
@@ -259,6 +261,7 @@ impl OtelConfig {
DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS,
)),
log_dry_run: Some(get_env_bool(ENV_OBS_LOG_DRY_RUN, DEFAULT_OBS_LOG_DRY_RUN)),
log_match_mode: Some(get_env_str(ENV_OBS_LOG_MATCH_MODE, DEFAULT_OBS_LOG_MATCH_MODE)),
}
}
crates/obs/src/lib.rs
+2 -2
View File
@@ -53,16 +53,16 @@
//! # // Guard will be dropped here, flushing telemetry data
//! # }
//! ```
mod cleaner;
mod config;
mod error;
mod global;
mod log_cleanup;
mod system;
mod telemetry;
pub use cleaner::*;
pub use config::*;
pub use error::*;
pub use global::*;
pub use log_cleanup::*;
pub use system::SystemObserver;
pub use telemetry::{OtelGuard, Recorder};
crates/obs/src/telemetry/local.rs
+41 -10
View File
@@ -27,9 +27,10 @@
//! cases; callers do **not** need to distinguish between stdout and file modes.
use crate::TelemetryError;
use crate::cleaner::LogCleaner;
use crate::cleaner::types::FileMatchMode;
use crate::config::OtelConfig;
use crate::global::OBSERVABILITY_METRIC_ENABLED;
use crate::log_cleanup::LogCleaner;
use crate::telemetry::filter::build_env_filter;
use metrics::counter;
use rustfs_config::observability::{
@@ -38,6 +39,7 @@ use rustfs_config::observability::{
DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS,
};
use rustfs_config::{APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_STDOUT_ENABLED};
use std::sync::Arc;
use std::{fs, io::IsTerminal, time::Duration};
use tracing::info;
use tracing_error::ErrorLayer;
@@ -176,6 +178,13 @@ fn init_file_logging_internal(
.as_deref()
.unwrap_or(DEFAULT_LOG_ROTATION_TIME)
.to_lowercase();
// Determine match mode from config, defaulting to Suffix
let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() {
Some("prefix") => FileMatchMode::Prefix,
_ => FileMatchMode::Suffix,
};
use tracing_appender::rolling::{RollingFileAppender, Rotation};
let file_appender = {
let rotation = match rotation.as_str() {
@@ -183,10 +192,17 @@ fn init_file_logging_internal(
"hourly" => Rotation::HOURLY,
_ => Rotation::DAILY,
};
RollingFileAppender::builder()
let mut builder = RollingFileAppender::builder()
.rotation(rotation)
.filename_suffix(log_filename)
.max_log_files(keep_files)
.max_log_files(keep_files * 2); // Make sure there are some data files to archive to avoid premature deletion
match match_mode {
FileMatchMode::Prefix => builder = builder.filename_prefix(log_filename),
FileMatchMode::Suffix => builder = builder.filename_suffix(log_filename),
}
builder
.build(log_directory)
.expect("failed to initialize rolling file appender")
};
@@ -329,7 +345,16 @@ fn spawn_cleanup_task(
keep_files: usize,
) -> tokio::task::JoinHandle<()> {
let log_dir = std::path::PathBuf::from(log_directory);
let file_prefix = config.log_filename.as_deref().unwrap_or(log_filename).to_string();
// Use suffix matching for log files like "2026-03-01-06-21.rustfs.log"
// where "rustfs.log" is the suffix.
let file_pattern = config.log_filename.as_deref().unwrap_or(log_filename).to_string();
// Determine match mode from config, defaulting to Suffix
let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() {
Some("prefix") => FileMatchMode::Prefix,
_ => FileMatchMode::Suffix,
};
let keep_count = config.log_keep_count.unwrap_or(keep_files);
let max_total_size = config
.log_max_total_size_bytes
@@ -358,9 +383,10 @@ fn spawn_cleanup_task(
.log_cleanup_interval_seconds
.unwrap_or(DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS);
let cleaner = LogCleaner::new(
let cleaner = Arc::new(LogCleaner::new(
log_dir,
file_prefix,
file_pattern,
match_mode,
keep_count,
max_total_size,
max_single_file_size,
@@ -371,14 +397,19 @@ fn spawn_cleanup_task(
delete_empty,
min_age,
dry_run,
);
));
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(cleanup_interval));
loop {
interval.tick().await;
if let Err(e) = cleaner.cleanup() {
tracing::warn!("Log cleanup failed: {}", e);
let cleaner_clone = cleaner.clone();
let result = tokio::task::spawn_blocking(move || cleaner_clone.cleanup()).await;
match result {
Ok(Ok(_)) => {} // Success
Ok(Err(e)) => tracing::warn!("Log cleanup failed: {}", e),
Err(e) => tracing::warn!("Log cleanup task panicked: {}", e),
}
}
})
scripts/run.sh
+1
View File
@@ -79,6 +79,7 @@ export RUSTFS_OBS_LOG_FLUSH_MS=300 # Log flush interval in milliseconds
export RUSTFS_RUNTIME_WORKER_THREADS=16
export RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024
export RUSTFS_RUNTIME_THREAD_PRINT_ENABLED=false
export RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS=300
# shellcheck disable=SC2125
export RUSTFS_RUNTIME_THREAD_STACK_SIZE=1024*1024
export RUSTFS_RUNTIME_THREAD_KEEP_ALIVE=60