fix(obs): tighten low-risk telemetry correctness (#4483)

Refs rustfs/backlog#1007
Refs rustfs/backlog#986

- respect explicit log-target directives when injecting noisy-crate suppressions
- fix daily rotation day-boundary checks and add a short retry cooldown after failed rotations
- preserve recorder metadata across label variants and serialize gauge updates before export
- correct profiling target_env reporting, trace all=true parsing, cleaner accounting/docs, and legacy system interval rounding

Co-authored-by: heihutu <heihutu@gmail.com>

crates/obs/src/cleaner/README.md

@@ -73,7 +73,7 @@ These values can be wired into dashboards and alert rules for cleanup health.
 
 For regular logs, the cleaner evaluates candidates in this order:
 
-1. keep at least `keep_files` newest matching generations;
+1. keep at most `keep_files` newest matching generations;
 2. remove older files if total retained size still exceeds `max_total_size_bytes`;
 3. remove any file whose individual size exceeds `max_single_file_size_bytes`;
 4. if compression is enabled, archive before deletion;

crates/obs/src/cleaner/core.rs

@@ -69,7 +69,7 @@ pub struct LogCleaner {
     pub(super) active_filename: String,
     /// Whether `file_pattern` is interpreted as a prefix or suffix.
     pub(super) match_mode: FileMatchMode,
-    /// Minimum number of regular log files to keep regardless of size.
+    /// Maximum number of newest regular log files to keep.
     pub(super) keep_files: usize,
     /// Optional cap for the cumulative size of regular logs.
     pub(super) max_total_size_bytes: u64,
@@ -205,8 +205,8 @@ impl LogCleaner {
     /// Choose regular log files that should be compressed and/or deleted.
     ///
     /// The `files` slice must already be sorted from oldest to newest. The
-    /// method first preserves the newest `keep_files` generations, then applies
-    /// total-size and per-file-size limits to the remaining tail.
+    /// method first enforces the `keep_files` ceiling, then applies total-size
+    /// and per-file-size limits to the remaining tail.
     pub(super) fn select_files_to_process(&self, files: &[FileInfo], total_size: u64) -> Vec<FileInfo> {
         let mut to_delete = Vec::new();
         if files.is_empty() {
@@ -325,8 +325,6 @@ impl LogCleaner {
                         } else {
                             match injector.steal_batch_and_pop(&local_worker) {
                                 Steal::Success(file) => {
-                                    attempts.fetch_add(1, Ordering::Relaxed);
-                                    successes.fetch_add(1, Ordering::Relaxed);
                                     Some(file)
                                 }
                                 Steal::Retry => continue,
@@ -357,6 +355,7 @@ impl LogCleaner {
                             continue;
                         };
 
+                        let mut file = file;
                         let compressed = match compress_file(&file.path, &options) {
                             Ok(output) => {
                                 debug!(
@@ -371,6 +370,7 @@ impl LogCleaner {
                                     state = "parallel_compression_done",
                                     "log cleaner state changed"
                                 );
+                                file.projected_freed_bytes = output.input_bytes.saturating_sub(output.output_bytes);
                                 true
                             }
                             Err(err) => {
@@ -508,7 +508,9 @@ impl LogCleaner {
                             state = "serial_compression_done",
                             "log cleaner state changed"
                         );
-                        deletable.push(file.clone());
+                        let mut file = file.clone();
+                        file.projected_freed_bytes = output.input_bytes.saturating_sub(output.output_bytes);
+                        deletable.push(file);
                     }
                     Err(err) => {
                         warn!(event = EVENT_LOG_CLEANER_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_LOG_CLEANER, file = ?file.path, error = %err, result = "serial_compression_failed", "log cleaner state changed");
@@ -585,15 +587,15 @@ impl LogCleaner {
             if self.dry_run {
                 info!(event = EVENT_LOG_CLEANER_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_LOG_CLEANER, state = "dry_run_delete", file = ?f.path, bytes = f.size, "log cleaner state changed");
                 deleted += 1;
-                freed += f.size;
+                freed += f.projected_freed_bytes;
                 continue;
             }
 
             match self.secure_delete(&f.path) {
                 Ok(()) => {
                     deleted += 1;
-                    freed += f.size;
-                    debug!(event = EVENT_LOG_CLEANER_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_LOG_CLEANER, state = "deleted", file = ?f.path, bytes = f.size, "log cleaner state changed");
+                    freed += f.projected_freed_bytes;
+                    debug!(event = EVENT_LOG_CLEANER_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_LOG_CLEANER, state = "deleted", file = ?f.path, bytes = f.size, projected_freed_bytes = f.projected_freed_bytes, "log cleaner state changed");
                 }
                 Err(e) => {
                     error!(event = EVENT_LOG_CLEANER_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_LOG_CLEANER, result = "delete_failed", file = ?f.path, error = %e, "log cleaner state changed");
@@ -665,7 +667,7 @@ impl LogCleanerBuilder {
         self
     }
 
-    /// Preserve at least this many newest regular log files.
+    /// Keep at most this many newest regular log files.
     pub fn keep_files(mut self, keep_files: usize) -> Self {
         self.keep_files = keep_files;
         self

crates/obs/src/cleaner/mod.rs

@@ -127,8 +127,6 @@ mod tests {
     fn assert_parallel_cleanup_completes(file_count: usize, workers: usize) -> std::io::Result<()> {
         let tmp = TempDir::new()?;
         let dir = tmp.path().to_path_buf();
-        let expected_freed = (file_count * 256) as u64;
-
         for i in 0..file_count {
             create_log_file(&dir, &format!("app.log.2024-01-{i:03}"), 256)?;
         }
@@ -160,9 +158,22 @@ mod tests {
                 name.starts_with("app.log.") && !compressed_suffixes.iter().any(|suffix| name.ends_with(suffix))
             })
             .count();
+        let archive_bytes: u64 = std::fs::read_dir(&dir)?
+            .filter_map(Result::ok)
+            .filter(|entry| {
+                let name = entry.file_name();
+                let name = name.to_string_lossy();
+                compressed_suffixes.iter().any(|suffix| name.ends_with(suffix))
+            })
+            .map(|entry| entry.metadata().map(|metadata| metadata.len()).unwrap_or(0))
+            .sum();
 
         assert_eq!(result.0, file_count, "all rotated logs should be deleted after compression");
-        assert_eq!(result.1, expected_freed, "freed bytes should match the removed source files");
+        assert_eq!(
+            result.1,
+            (file_count * 256) as u64 - archive_bytes,
+            "freed bytes should exclude archive bytes that still remain on disk"
+        );
         assert_eq!(archive_count, file_count, "each rotated log should leave behind one archive");
         assert_eq!(original_count, 0, "compressed source logs should be removed");
         Ok(())

crates/obs/src/cleaner/scanner.rs

@@ -199,6 +199,7 @@ pub(super) fn scan_log_directory(
         let info = FileInfo {
             path,
             size: file_size,
+            projected_freed_bytes: file_size,
             modified,
         };
 

crates/obs/src/cleaner/types.rs

@@ -176,6 +176,11 @@ pub(super) struct FileInfo {
     ///
     /// This value is used for retention accounting and freed-byte metrics.
     pub size: u64,
+    /// Projected bytes reclaimed when this file is deleted.
+    ///
+    /// For source logs that were compressed first, this reflects the original
+    /// size minus the archive bytes that remain on disk.
+    pub projected_freed_bytes: u64,
     /// Last-modification timestamp from the filesystem.
     ///
     /// The selection phase sorts on this timestamp so the oldest files are