From 35a30cd6144f5f37f3b03e2964b73655ed408274 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 08:46:32 +0800 Subject: [PATCH] feat(scanner): emit excess alerts as S3 notification events (HS-04) (#6176) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(scanner): emit excess alerts as S3 notification events The excess-versions / excess-version-size / excess-folders alerts were metrics-and-logs only; consoles and external auditors had no way to hear them (rustfs/backlog#1868, HS-04). MinIO emits s3:ObjectManyVersions / s3:ObjectLargeVersions / s3:PrefixManyFolders for the same conditions — RustFS carries those as EventName::Scanner* with s3:Scanner:* wire names that already existed unpublished. The three alert sites now also dispatch through the standard event pipeline (send_event via the storage_api owner facade), carrying the actual values and thresholds in req_params and UserAgent "Scanner". Without a cooldown a single over-threshold object would re-emit on every ~60s scan cycle, so emissions are edge-held per (kind, bucket, object) for 24h (RUSTFS_SCANNER_ALERT_COOLDOWN_SECS, 0 = every cycle), backed by a process-global map with a 4096-key hard cap that clears rather than grows. Metrics and structured logs stay level-triggered every cycle; only the notification events are held back. A restart resets the cooldown deliberately: one re-emission per still-hot key buys back visibility after the restarts that accompany incident response. Tests pin the edge-hold semantics (first fires, immediate re-check held, independent keys, cooldown expiry re-fires, zero cooldown always emits, hard bound) in one sequential test for the process-global map, and pin the emitted wire names against EventName's canonical string forms so a subscribed bucket notification can never silently stop matching. docs/operations/scanner-excess-alerts.md documents the three events, the metric-vs-event cadence difference, and the HS-15 threshold deltas (alert_excess_folders 65538 vs MinIO 50000 is deliberate: Proxmox Backup Server chunk layout compatibility). Closes rustfs/backlog#1868. Co-Authored-By: heihutu * docs(operations): split scanner excess alerts into English and Chinese pages The page shipped Chinese-only; keep it as scanner-excess-alerts_zh.md and add a faithful English translation at the original path, cross-linked at the top of both. Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- Cargo.lock | 1 + crates/ecstore/src/api/mod.rs | 2 +- crates/scanner/Cargo.toml | 3 + crates/scanner/src/scanner_folder.rs | 229 +++++++++++++++++++- crates/scanner/src/storage_api.rs | 7 +- docs/operations/scanner-excess-alerts.md | 37 ++++ docs/operations/scanner-excess-alerts_zh.md | 37 ++++ 7 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 docs/operations/scanner-excess-alerts.md create mode 100644 docs/operations/scanner-excess-alerts_zh.md diff --git a/Cargo.lock b/Cargo.lock index 85c91d523..cda581ffb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10252,6 +10252,7 @@ dependencies = [ "rustfs-ecstore", "rustfs-filemeta", "rustfs-lock", + "rustfs-s3-types", "rustfs-storage-api", "rustfs-utils", "s3s", diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 4b8bc3249..3a031d60a 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -380,7 +380,7 @@ pub mod erasure { pub mod event { pub use crate::event::name::EventName; - pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook}; + pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook, send_event}; } pub mod global { diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 16ff34a8e..0e2d039c6 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -108,6 +108,9 @@ temp-env = { workspace = true } tempfile = { workspace = true } uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] } tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } +# Test-only: pins the emitted scanner alert wire names against the canonical +# EventName string forms subscribers configure (rustfs/backlog#1868). +rustfs-s3-types.workspace = true # Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via # the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6). rustfs-ecstore = { workspace = true, features = ["test-util"] } diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 820cbb12f..15a01507c 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fs::FileType; use std::io::ErrorKind; -use std::sync::{Arc, Once}; +use std::sync::{Arc, Mutex, Once}; use std::time::{Duration, Instant, SystemTime}; use crate::ReplTargetSizeSummary; @@ -32,6 +32,7 @@ use crate::scanner_io::{ SCANNER_SKIP_FILE_ERROR, ScannerIODisk as _, is_scanner_metadata_corrupt_error, is_scanner_metadata_transient_error, }; use crate::sleeper::DynamicSleeper; +use crate::storage_api::owner::{EcstoreEventArgs, ecstore_send_event}; use metrics::{counter, describe_counter}; use rustfs_common::heal_channel::{ HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest, @@ -98,6 +99,101 @@ const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders const METRIC_SCANNER_PENDING_HEAL_PRUNE_TOTAL: &str = "rustfs_scanner_pending_heal_prune_total"; const METRIC_SCANNER_PENDING_HEAL_MALFORMED_TOTAL: &str = "rustfs_scanner_pending_heal_malformed_total"; const MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET: usize = 128; + +// --- scanner excess alerts as S3 notification events (rustfs/backlog#1868) -- +// +// The excess-versions / excess-version-size / excess-folders alerts were +// metrics-and-logs only; subscribers (consoles, external auditors) had no way +// to hear them. MinIO emits s3:ObjectManyVersions / s3:ObjectLargeVersions / +// s3:PrefixManyFolders for the same conditions — RustFS carries those as +// EventName::Scanner* with the wire names below. Without a cooldown a single +// over-threshold object would re-emit on every scan cycle (~a minute), so +// emissions are edge-held per (kind, bucket, object) for 24h. + +/// `s3:Scanner:ManyVersions` (MinIO `s3:ObjectManyVersions`). +pub const EVENT_SCANNER_MANY_VERSIONS: &str = "s3:Scanner:ManyVersions"; +/// `s3:Scanner:LargeVersions` (MinIO `s3:ObjectLargeVersions`). +pub const EVENT_SCANNER_LARGE_VERSIONS: &str = "s3:Scanner:LargeVersions"; +/// `s3:Scanner:BigPrefix` (MinIO `s3:PrefixManyFolders`). +pub const EVENT_SCANNER_BIG_PREFIX: &str = "s3:Scanner:BigPrefix"; +const ENV_SCANNER_ALERT_COOLDOWN_SECS: &str = "RUSTFS_SCANNER_ALERT_COOLDOWN_SECS"; +const DEFAULT_SCANNER_ALERT_COOLDOWN_SECS: u64 = 86_400; +/// Hard cap on distinct cooldown keys; a pathological number of over-threshold +/// objects clears the map wholesale instead of growing without bound (the +/// worst case is one re-emission per still-hot key per scan cycle). +const MAX_SCANNER_ALERT_COOLDOWN_KEYS: usize = 4096; + +/// Distinct alert kinds sharing one cooldown map. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum ScannerAlertKind { + ManyVersions, + LargeVersions, + BigPrefix, +} + +type ScannerAlertCooldownKey = (ScannerAlertKind, String, String); +type ScannerAlertCooldownMap = HashMap; + +static SCANNER_ALERT_EMISSION_COOLDOWN: Mutex> = Mutex::new(None); + +fn scanner_alert_cooldown() -> Duration { + let raw = std::env::var(ENV_SCANNER_ALERT_COOLDOWN_SECS) + .ok() + .and_then(|v| v.parse::().ok()); + Duration::from_secs(raw.unwrap_or(DEFAULT_SCANNER_ALERT_COOLDOWN_SECS)) +} + +/// Edge-held emission gate: returns `true` (and records the cooldown) only +/// when this (kind, bucket, object) last fired longer than the cooldown ago — +/// or never. Metrics and logs stay level-triggered every cycle; only the +/// notification events are held back. +fn scanner_alert_emission_allows(kind: ScannerAlertKind, bucket: &str, object: &str, cooldown: Duration) -> bool { + let key = (kind, bucket.to_string(), object.to_string()); + let mut guard = SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + let guard = guard.get_or_insert_with(ScannerAlertCooldownMap::new); + let now = Instant::now(); + // Expired entries leave first; the cap is still exceeded only when live + // keys alone overflow it, in which case a wholesale clear trades one + // extra emission per hot key for a hard memory bound. + if guard.len() >= MAX_SCANNER_ALERT_COOLDOWN_KEYS { + guard.retain(|_, fired_at| now.duration_since(*fired_at) < cooldown); + if guard.len() >= MAX_SCANNER_ALERT_COOLDOWN_KEYS { + guard.clear(); + } + } + match guard.get(&key) { + Some(fired_at) if now.duration_since(*fired_at) < cooldown => false, + _ => { + guard.insert(key, now); + true + } + } +} + +/// Emit a scanner alert as an S3 notification event through the standard +/// dispatch pipeline. Fire-and-forget: the notify layer owns delivery, +/// retry, and target filtering; the scanner never waits on it. +fn emit_scanner_alert_event(event_name: &str, bucket: &str, object: &str, size: i64, details: &[(&str, String)]) { + let mut req_params = HashMap::with_capacity(details.len()); + for (key, value) in details { + req_params.insert((*key).to_string(), value.clone()); + } + ecstore_send_event(EcstoreEventArgs { + event_name: event_name.to_string(), + bucket_name: bucket.to_string(), + object: crate::ScannerObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size, + ..Default::default() + }, + req_params, + user_agent: "Scanner".to_string(), + ..Default::default() + }); +} const MAX_PENDING_SCANNER_HEALS_PER_BUCKET: usize = 10_000; static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new(); @@ -1350,6 +1446,7 @@ impl ScannerItem { fn alert_excessive_versions(&self, remaining_versions: usize, cumulative_size: i64) { ensure_scanner_alert_metrics_registered(); let (too_many_versions, too_large_versions) = should_alert_excessive_versions(remaining_versions, cumulative_size); + let object_path = self.object_path(); if too_many_versions { global_metrics().record_scanner_source_executed(ScannerWorkSource::Alerts, 1); counter!( @@ -1357,13 +1454,26 @@ impl ScannerItem { "bucket" => self.bucket.clone() ) .increment(1); + if scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, &self.bucket, &object_path, scanner_alert_cooldown()) + { + emit_scanner_alert_event( + EVENT_SCANNER_MANY_VERSIONS, + &self.bucket, + &object_path, + cumulative_size, + &[ + ("versions", remaining_versions.to_string()), + ("threshold", scanner_excess_versions_threshold().to_string()), + ], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_FOLDER, bucket = %self.bucket, - object = %self.object_path(), + object = %object_path, versions = remaining_versions, threshold = scanner_excess_versions_threshold(), state = "excess_versions", @@ -1377,13 +1487,31 @@ impl ScannerItem { "bucket" => self.bucket.clone() ) .increment(1); + if scanner_alert_emission_allows( + ScannerAlertKind::LargeVersions, + &self.bucket, + &object_path, + scanner_alert_cooldown(), + ) { + emit_scanner_alert_event( + EVENT_SCANNER_LARGE_VERSIONS, + &self.bucket, + &object_path, + cumulative_size, + &[ + ("versions", remaining_versions.to_string()), + ("cumulativeSize", cumulative_size.to_string()), + ("threshold", scanner_excess_version_size_threshold().to_string()), + ], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_FOLDER, bucket = %self.bucket, - object = %self.object_path(), + object = %object_path, versions = remaining_versions, cumulative_size, threshold = scanner_excess_version_size_threshold(), @@ -1764,6 +1892,15 @@ impl FolderScanner { "root" => self.root.clone() ) .increment(1); + if scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, &self.root, folder, scanner_alert_cooldown()) { + emit_scanner_alert_event( + EVENT_SCANNER_BIG_PREFIX, + &self.root, + folder, + 0, + &[("folders", total_folders.to_string()), ("threshold", threshold.to_string())], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, @@ -3232,6 +3369,90 @@ mod tests { #[cfg(unix)] use std::os::unix::fs::{PermissionsExt, symlink}; use std::sync::Mutex; + + /// Reset the process-global alert cooldown map; test-only. + fn reset_alert_cooldowns() { + *SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some(ScannerAlertCooldownMap::new()); + } + + /// The emitted event-name strings must be exactly what `EventName` + /// serializes, or a bucket notification subscribed to the documented name + /// would silently never match (rustfs/backlog#1868). + #[test] + fn scanner_alert_wire_names_match_canonical_event_names() { + use rustfs_s3_types::EventName; + assert_eq!(EVENT_SCANNER_MANY_VERSIONS, EventName::ScannerManyVersions.to_string()); + assert_eq!(EVENT_SCANNER_LARGE_VERSIONS, EventName::ScannerLargeVersions.to_string()); + assert_eq!(EVENT_SCANNER_BIG_PREFIX, EventName::ScannerBigPrefix.to_string()); + } + + fn cooldown_map_len() -> usize { + SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .as_ref() + .map(|map| map.len()) + .unwrap_or(0) + } + + /// Backdate every recorded cooldown so the next check fires again. + fn expire_all_alert_cooldowns(cooldown: Duration) { + let now = Instant::now(); + let mut guard = SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if let Some(map) = guard.as_mut() { + for fired_at in map.values_mut() { + if let Some(expired) = now.checked_sub(cooldown + Duration::from_secs(1)) { + *fired_at = expired; + } + } + } + } + + /// The emission gate is the only thing standing between an over-threshold + /// object and one S3 event per scan cycle, so its edge semantics get + /// pinned directly. All scenarios share one #[test] because the cooldown + /// map is process-global and parallel tests would read each other's + /// firings. + #[test] + fn scanner_alert_emission_is_edge_held_per_key_and_bounded() { + reset_alert_cooldowns(); + let cooldown = Duration::from_secs(3600); + + // First firing allows, an immediate re-check is held. + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + assert!(!scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + + // Different kind, object, and bucket are independent keys. + assert!(scanner_alert_emission_allows(ScannerAlertKind::LargeVersions, "bkt", "obj", cooldown)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "other", cooldown)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "other", "obj", cooldown)); + assert_eq!(cooldown_map_len(), 4); + + // After the cooldown elapses the same key fires again. + expire_all_alert_cooldowns(cooldown); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + + // A zero cooldown degenerates to always-emit (operators may want that). + assert!(scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", "dir", Duration::ZERO)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", "dir", Duration::ZERO)); + + // Hard bound: overflow the cap with zero-cooldown keys and confirm the + // map clears rather than growing past it. + reset_alert_cooldowns(); + for index in 0..=(MAX_SCANNER_ALERT_COOLDOWN_KEYS + 8) { + let _ = scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", &format!("dir-{index}"), Duration::ZERO); + } + assert!( + cooldown_map_len() <= MAX_SCANNER_ALERT_COOLDOWN_KEYS, + "cooldown map must stay bounded, got {}", + cooldown_map_len() + ); + } + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use temp_env::{with_var, with_var_unset}; use tracing_subscriber::fmt::MakeWriter; diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index e77033aa2..cbd7c6486 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -78,6 +78,7 @@ pub(crate) use rustfs_ecstore::api::disk::{ pub(crate) use rustfs_ecstore::api::error::{ Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError, }; +pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_event as ecstore_send_event}; #[cfg(test)] pub(crate) use rustfs_ecstore::api::layout::{ EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints, @@ -110,8 +111,8 @@ pub(crate) mod owner { ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS, ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions, - EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc, - EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, + EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreEventArgs, + EcstoreLcEventSrc, EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi, ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, @@ -121,7 +122,7 @@ pub(crate) mod owner { ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, - ecstore_save_config, scanner_replication_config_for_lifecycle_eval, + ecstore_save_config, ecstore_send_event, scanner_replication_config_for_lifecycle_eval, }; #[cfg(test)] diff --git a/docs/operations/scanner-excess-alerts.md b/docs/operations/scanner-excess-alerts.md new file mode 100644 index 000000000..8079daba6 --- /dev/null +++ b/docs/operations/scanner-excess-alerts.md @@ -0,0 +1,37 @@ +# Scanner Excess Alerts: Metrics, S3 Events, and Thresholds + +> 中文版：[scanner-excess-alerts_zh.md](scanner-excess-alerts_zh.md) + +Date: 2026-08-18 (rustfs/backlog#1868 / HS-04; includes the HS-15 threshold-delta notes) + +The background scanner detects three classes of "excess" conditions while it walks buckets and surfaces them as alerts. This page documents each alert's trigger condition, the subscribable S3 event, the cooldown semantics, and the threshold differences versus MinIO — for operators debugging alerts and for event consumers wiring up subscriptions. + +## The three alerts + +| Alert | Trigger (per scan cycle) | Metric | S3 event (RustFS wire name) | MinIO event name | +|---|---|---|---|---| +| Excess versions | Retained versions of one object ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` | +| Excess version size | Cumulative bytes of all versions of one object ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` | +| Excess folders | Direct subfolders of one directory > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` | + +Subscribe like any bucket notification: configure a notification on the target bucket with the RustFS wire name above (or the `s3:Scanner:*` wildcard). Events carry `UserAgent: Scanner` as their origin marker, and `req_params` holds the observed value and the threshold (`versions` / `cumulativeSize` / `folders` / `threshold`), so consumers can judge severity directly. + +## Metrics and events fire on different cadences + +- **Metrics and structured logs are level-triggered**: as long as the object stays over the threshold, every scan cycle counts and logs it (default cycle ≈ 60s; see `scanner:speed`). +- **S3 events are edge-triggered with a cooldown**: the same (alert kind, bucket, object) emits at most once per cooldown window — 24 hours by default (`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`; set it to 0 to emit every cycle). When the window lapses and the object is still over the threshold, the event fires again. The cooldown table lives in process memory with a 4096-entry hard cap; on overflow it is cleared and rebuilt (worst case: one extra emission per still-hot key). +- A process restart resets the cooldown (every still-over-threshold object emits once more after a restart) — deliberately: restarts usually accompany incident response, and the re-emission buys visibility. + +## Threshold defaults and the MinIO deltas (HS-15) + +| Config key | ENV | RustFS default | MinIO default | Notes | +|---|---|---|---|---| +| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | Identical | +| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | Same order of magnitude; different unit basis (TiB vs TB) | +| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **Deliberate divergence**: 65538 tolerates the Proxmox Backup Server chunk layout (65536 chunks per directory plus the directory's own entries); MinIO's 50000 would fire continuously for PBS users. Set it to 50000 explicitly to match MinIO behavior | + +All three keys accept both env and admin config (`PUT /rustfs/admin/v3/config`, `scanner` subsystem); hot updates take effect immediately. + +## Why the event names are mapped + +RustFS's event enum (`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`) keeps the repo's established `s3:Scanner:*` wire names (literally different from MinIO's `s3:ObjectManyVersions`; the enum comments preserve the mapping). Subscribers should use the RustFS wire names in this page. If you need MinIO-literal compatibility, map the names on the console/consumer side — do not change the published wire names. diff --git a/docs/operations/scanner-excess-alerts_zh.md b/docs/operations/scanner-excess-alerts_zh.md new file mode 100644 index 000000000..d4f4995ac --- /dev/null +++ b/docs/operations/scanner-excess-alerts_zh.md @@ -0,0 +1,37 @@ +# Scanner 超限告警：指标、S3 事件与阈值 + +> English version: [scanner-excess-alerts.md](scanner-excess-alerts.md) + +日期：2026-08-18（rustfs/backlog#1868 / HS-04，含 HS-15 阈值差异说明） + +后台 scanner 在扫描过程中检测三类"超限"状态并对外告警。本文说明每类告警的触发条件、可订阅的 S3 事件、冷却语义，以及与 MinIO 的阈值差异，供运维排障与事件消费方对接。 + +## 三类告警 + +| 告警 | 触发条件（任一扫描周期） | 指标 | S3 事件（RustFS wire 名） | MinIO 对应事件名 | +|---|---|---|---|---| +| 版本数超限 | 单对象保留版本数 ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` | +| 版本总大小超限 | 单对象全部版本累计字节 ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` | +| 子目录数超限 | 单目录直接子目录数 > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` | + +订阅方式与普通桶通知一致：对目标桶配置 notification，事件名填上表 RustFS wire 名（或通配 `s3:Scanner:*`）。事件以 `UserAgent: Scanner` 标记来源，`req_params` 携带实际值与阈值（`versions` / `cumulativeSize` / `folders` / `threshold`），便于消费方直接判断严重程度。 + +## 指标与事件的触发节奏不同 + +- **指标与结构化日志是电平触发**：只要对象仍在阈值之上，每个扫描周期都会计数/打日志（默认周期约 60s，见 `scanner:speed`）。 +- **S3 事件是边沿触发 + 冷却**：同一 (告警类型， 桶, 对象) 在冷却窗口内只发一次，默认 24 小时（`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`，设 0 表示每周期都发）。窗口过后对象仍超限会再次发出。冷却表在进程内有 4096 条硬顶，超限清空重建（最坏情况是每个仍超限的 key 多发一次）。 +- 进程重启会重置冷却（重启后每个仍超限的对象会再发一次）——这是有意为之：重启常伴随排障，重发提供可见性。 + +## 阈值默认值与 MinIO 差异（HS-15） + +| 配置键 | ENV | RustFS 默认 | MinIO 默认 | 差异说明 | +|---|---|---|---|---| +| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | 一致 | +| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | 语义同量级，单位口径不同（TiB vs TB） | +| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **有意差异**：65538 兼容 Proxmox Backup Server 的 chunk 布局（每目录 65536 个 chunk + 目录自身条目），按 MinIO 的 50000 会对 PBS 用户持续误报。如需与 MinIO 行为一致可显式配置为 50000 | + +三个键均支持 env 与 admin config（`PUT /rustfs/admin/v3/config` 的 `scanner` 子系统）双通道，热更新即时生效。 + +## 事件名映射的由来 + +RustFS 的事件枚举（`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`）沿用仓库既有 wire 名 `s3:Scanner:*`（与 MinIO 的 `s3:ObjectManyVersions` 字面不同，枚举注释中保留了映射关系）。订阅方应以本文的 RustFS wire 名为准；如需 MinIO 字面兼容，请在 console/消费侧做名称映射，不要修改已发布的 wire 名。