mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 02:33:15 +00:00
houseme
8c9645924a
The excess-versions / excess-version-size / excess-folders alerts were metrics-and-logs only; consoles and external auditors had no way to hear them (rustfs/backlog#1868, HS-04). MinIO emits s3:ObjectManyVersions / s3:ObjectLargeVersions / s3:PrefixManyFolders for the same conditions — RustFS carries those as EventName::Scanner* with s3:Scanner:* wire names that already existed unpublished. The three alert sites now also dispatch through the standard event pipeline (send_event via the storage_api owner facade), carrying the actual values and thresholds in req_params and UserAgent "Scanner". Without a cooldown a single over-threshold object would re-emit on every ~60s scan cycle, so emissions are edge-held per (kind, bucket, object) for 24h (RUSTFS_SCANNER_ALERT_COOLDOWN_SECS, 0 = every cycle), backed by a process-global map with a 4096-key hard cap that clears rather than grows. Metrics and structured logs stay level-triggered every cycle; only the notification events are held back. A restart resets the cooldown deliberately: one re-emission per still-hot key buys back visibility after the restarts that accompany incident response. Tests pin the edge-hold semantics (first fires, immediate re-check held, independent keys, cooldown expiry re-fires, zero cooldown always emits, hard bound) in one sequential test for the process-global map, and pin the emitted wire names against EventName's canonical string forms so a subscribed bucket notification can never silently stop matching. docs/operations/scanner-excess-alerts.md documents the three events, the metric-vs-event cadence difference, and the HS-15 threshold deltas (alert_excess_folders 65538 vs MinIO 50000 is deliberate: Proxmox Backup Server chunk layout compatibility). Closes rustfs/backlog#1868. Co-Authored-By: heihutu <heihutu@gmail.com>
120 lines
4.3 KiB
TOML
120 lines
4.3 KiB
TOML
# Copyright 2024 RustFS Team
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
[package]
|
|
name = "rustfs-scanner"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
authors = ["RustFS Team"]
|
|
license.workspace = true
|
|
repository.workspace = true
|
|
rust-version.workspace = true
|
|
homepage.workspace = true
|
|
description = "RustFS Scanner provides scanning capabilities for data integrity checks, health monitoring, and storage analysis."
|
|
keywords = ["RustFS", "scanner", "health-monitoring", "data-integrity", "storage-analysis"]
|
|
categories = ["web-programming", "development-tools", "filesystem"]
|
|
documentation = "https://docs.rs/rustfs-scanner/latest/rustfs_scanner/"
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
hotpath = [
|
|
"hotpath/hotpath",
|
|
"hotpath/tokio",
|
|
"hotpath/futures",
|
|
"rustfs-common/hotpath",
|
|
"rustfs-config/hotpath",
|
|
"rustfs-credentials/hotpath",
|
|
"rustfs-data-usage/hotpath",
|
|
"rustfs-ecstore/hotpath",
|
|
"rustfs-filemeta/hotpath",
|
|
"rustfs-storage-api/hotpath",
|
|
"rustfs-utils/hotpath",
|
|
]
|
|
hotpath-alloc = [
|
|
"hotpath",
|
|
"hotpath/hotpath-alloc",
|
|
"rustfs-common/hotpath-alloc",
|
|
"rustfs-config/hotpath-alloc",
|
|
"rustfs-credentials/hotpath-alloc",
|
|
"rustfs-data-usage/hotpath-alloc",
|
|
"rustfs-ecstore/hotpath-alloc",
|
|
"rustfs-filemeta/hotpath-alloc",
|
|
"rustfs-storage-api/hotpath-alloc",
|
|
"rustfs-utils/hotpath-alloc",
|
|
]
|
|
hotpath-cpu = [
|
|
"hotpath",
|
|
"hotpath/hotpath-cpu",
|
|
"rustfs-common/hotpath-cpu",
|
|
"rustfs-config/hotpath-cpu",
|
|
"rustfs-credentials/hotpath-cpu",
|
|
"rustfs-data-usage/hotpath-cpu",
|
|
"rustfs-ecstore/hotpath-cpu",
|
|
"rustfs-filemeta/hotpath-cpu",
|
|
"rustfs-storage-api/hotpath-cpu",
|
|
"rustfs-utils/hotpath-cpu",
|
|
]
|
|
|
|
[dependencies]
|
|
hotpath.workspace = true
|
|
rustfs-config = { workspace = true, features = ["server-config-model"] }
|
|
rustfs-common = { workspace = true }
|
|
rustfs-credentials = { workspace = true }
|
|
rustfs-utils = { workspace = true }
|
|
tokio = { workspace = true, features = ["fs", "sync", "time", "macros", "rt-multi-thread"] }
|
|
tracing = { workspace = true }
|
|
serde = { workspace = true, features = ["derive"] }
|
|
serde_json = { workspace = true, features = ["raw_value"] }
|
|
thiserror = { workspace = true }
|
|
async-trait = { workspace = true }
|
|
futures = { workspace = true }
|
|
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
|
chrono = { workspace = true, features = ["serde"] }
|
|
rmp-serde = { workspace = true }
|
|
hmac = { workspace = true }
|
|
sha2 = { workspace = true }
|
|
rustfs-filemeta = { workspace = true }
|
|
rustfs-lock.workspace = true
|
|
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
|
|
rustfs-ecstore = { workspace = true }
|
|
rustfs-storage-api = { workspace = true }
|
|
http = { workspace = true }
|
|
rand = { workspace = true, features = ["serde"] }
|
|
s3s = { workspace = true, features = ["minio"] }
|
|
metrics = { workspace = true }
|
|
rustfs-data-usage = { workspace = true }
|
|
uuid = { workspace = true, features = ["v4", "serde", "fast-rng"] }
|
|
bytes.workspace = true
|
|
hex-simd.workspace = true
|
|
|
|
[dev-dependencies]
|
|
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
|
serial_test = { workspace = true }
|
|
temp-env = { workspace = true }
|
|
tempfile = { workspace = true }
|
|
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
|
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
|
# Test-only: pins the emitted scanner alert wire names against the canonical
|
|
# EventName string forms subscribers configure (rustfs/backlog#1868).
|
|
rustfs-s3-types.workspace = true
|
|
# Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via
|
|
# the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6).
|
|
rustfs-ecstore = { workspace = true, features = ["test-util"] }
|
|
|
|
[lib]
|
|
doctest = false
|