mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 02:33:15 +00:00
Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5819ce0412 | |||
| 00de43528c | |||
| 35a30cd614 | |||
| 360bceafce | |||
| 7cb91a0190 | |||
| beb6e1383e | |||
| 59b7d13095 | |||
| e0b87b0e7e | |||
| 984c705713 | |||
| 23b17c2d5a | |||
| 89e2513205 | |||
| d091554ffe | |||
| c04ee41cf0 |
Generated
+54
-47
@@ -3764,7 +3764,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -9093,7 +9093,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9231,7 +9231,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"const-str",
|
||||
@@ -9254,7 +9254,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9270,7 +9270,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"hotpath",
|
||||
@@ -9280,6 +9280,7 @@ dependencies = [
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"tracing",
|
||||
@@ -9288,7 +9289,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
@@ -9301,7 +9302,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"hotpath",
|
||||
@@ -9311,7 +9312,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9325,7 +9326,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9346,7 +9347,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rmp-serde",
|
||||
@@ -9356,7 +9357,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -9495,7 +9496,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -9505,7 +9506,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"byteorder",
|
||||
@@ -9532,7 +9533,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
@@ -9563,7 +9564,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9604,7 +9605,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
@@ -9617,7 +9618,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"hotpath",
|
||||
@@ -9681,7 +9682,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -9708,7 +9709,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9757,7 +9758,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -9780,7 +9781,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
@@ -9803,7 +9804,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-log-analyzer"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"flate2",
|
||||
@@ -9822,22 +9823,27 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"humantime",
|
||||
"hyper",
|
||||
"jiff",
|
||||
"reqwest",
|
||||
"rmp-serde",
|
||||
"rustfs-signer",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sysinfo",
|
||||
"time",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9872,7 +9878,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -9892,7 +9898,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -9909,7 +9915,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -9964,7 +9970,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -9995,7 +10001,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -10057,7 +10063,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
@@ -10081,7 +10087,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -10099,7 +10105,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -10137,7 +10143,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -10160,7 +10166,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-s3-types",
|
||||
@@ -10168,7 +10174,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -10177,7 +10183,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10207,7 +10213,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
@@ -10226,7 +10232,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10246,6 +10252,7 @@ dependencies = [
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-lock",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
@@ -10266,7 +10273,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"thiserror 2.0.20",
|
||||
@@ -10274,7 +10281,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10292,7 +10299,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10307,7 +10314,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -10361,7 +10368,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-data-usage",
|
||||
@@ -10377,7 +10384,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"hotpath",
|
||||
@@ -10398,7 +10405,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -10435,7 +10442,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
@@ -10477,7 +10484,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
|
||||
+47
-47
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,52 +86,52 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.3" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.3" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.3" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.3" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.3" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.3" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.3" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.3" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.3" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.3" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.3" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.3" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.3" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.3" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.3" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.3" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.3" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.3" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.3" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.3" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.3" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.3" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.3", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.3" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.3" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.3" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.3" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.3" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.3" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.3" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.3" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.3" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.3" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.3" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.3" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.3" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.3" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.3" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.3" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.3" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.3" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.3" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.3" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.3" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.3" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.2" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.2" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.2" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.2" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.2" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.2" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.2" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.2" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.2" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.2" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.2" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.2" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.2" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.2" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.2" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.2" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.2" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.2" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.2" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.2" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.2" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.2" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.2", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.2" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.2" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.2" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.2" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.2" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.2" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.2" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.2" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.2" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.2" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.2" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.2" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.2" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.2" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.2" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.2" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.2" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.3
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.3
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -40,6 +40,7 @@ mak = "mak"
|
||||
gae = "gae"
|
||||
GAE = "GAE"
|
||||
thr = "thr"
|
||||
mis = "mis"
|
||||
# s3-tests original test names (cannot be changed)
|
||||
nonexisted = "nonexisted"
|
||||
consts = "consts"
|
||||
|
||||
@@ -42,6 +42,7 @@ chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
smallvec = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
@@ -19,6 +19,7 @@ pub mod last_minute;
|
||||
pub mod metrics;
|
||||
mod readiness;
|
||||
pub mod table_catalog;
|
||||
pub mod trace_bus;
|
||||
|
||||
pub use globals::*;
|
||||
pub use readiness::{GlobalReadiness, SystemStage};
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use smallvec::SmallVec;
|
||||
use std::{
|
||||
sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
const DEFAULT_TRACE_BUS_CAPACITY: usize = 1024;
|
||||
const TRACE_ATTR_INLINE_CAPACITY: usize = 8;
|
||||
|
||||
static GLOBAL_TRACE_BUS: OnceLock<TraceBus> = OnceLock::new();
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TraceKind {
|
||||
Heal,
|
||||
Scanner,
|
||||
}
|
||||
|
||||
impl TraceKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Heal => "heal",
|
||||
Self::Scanner => "scanner",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TraceFunc {
|
||||
HealTask,
|
||||
HealBucket,
|
||||
HealObject,
|
||||
HealCheckAbandonedParts,
|
||||
HealErasureSetPage,
|
||||
ScannerFolder,
|
||||
ScannerIlmAction,
|
||||
ScannerHealCandidate,
|
||||
Dropped,
|
||||
}
|
||||
|
||||
impl TraceFunc {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::HealTask => "heal.Task",
|
||||
Self::HealBucket => "heal.Bucket",
|
||||
Self::HealObject => "heal.Object",
|
||||
Self::HealCheckAbandonedParts => "heal.CheckAbandonedParts",
|
||||
Self::HealErasureSetPage => "heal.ErasureSetPage",
|
||||
Self::ScannerFolder => "scanner.Folder",
|
||||
Self::ScannerIlmAction => "scanner.IlmAction",
|
||||
Self::ScannerHealCandidate => "scanner.HealCandidate",
|
||||
Self::Dropped => "trace.Dropped",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TraceVal {
|
||||
Bool(bool),
|
||||
U64(u64),
|
||||
I64(i64),
|
||||
Str(Arc<str>),
|
||||
}
|
||||
|
||||
impl From<bool> for TraceVal {
|
||||
fn from(value: bool) -> Self {
|
||||
Self::Bool(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u64> for TraceVal {
|
||||
fn from(value: u64) -> Self {
|
||||
Self::U64(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<i64> for TraceVal {
|
||||
fn from(value: i64) -> Self {
|
||||
Self::I64(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for TraceVal {
|
||||
fn from(value: &str) -> Self {
|
||||
Self::Str(Arc::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for TraceVal {
|
||||
fn from(value: String) -> Self {
|
||||
Self::Str(Arc::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TraceAttr {
|
||||
pub key: &'static str,
|
||||
pub value: TraceVal,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TraceEvent {
|
||||
pub kind: TraceKind,
|
||||
pub func: TraceFunc,
|
||||
pub time: SystemTime,
|
||||
pub bucket: Option<Arc<str>>,
|
||||
pub object: Option<Arc<str>>,
|
||||
pub duration: Duration,
|
||||
pub bytes: u64,
|
||||
pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>,
|
||||
}
|
||||
|
||||
impl TraceEvent {
|
||||
pub fn new(kind: TraceKind, func: TraceFunc) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
func,
|
||||
time: SystemTime::now(),
|
||||
bucket: None,
|
||||
object: None,
|
||||
duration: Duration::ZERO,
|
||||
bytes: 0,
|
||||
attrs: SmallVec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_bucket(mut self, bucket: impl Into<Arc<str>>) -> Self {
|
||||
self.bucket = Some(bucket.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_object(mut self, object: impl Into<Arc<str>>) -> Self {
|
||||
self.object = Some(object.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_duration(mut self, duration: Duration) -> Self {
|
||||
self.duration = duration;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_bytes(mut self, bytes: u64) -> Self {
|
||||
self.bytes = bytes;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_attr(mut self, key: &'static str, value: impl Into<TraceVal>) -> Self {
|
||||
self.attrs.push(TraceAttr {
|
||||
key,
|
||||
value: value.into(),
|
||||
});
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TraceBus {
|
||||
sender: broadcast::Sender<Arc<TraceEvent>>,
|
||||
subscriber_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl TraceBus {
|
||||
pub fn new(capacity: usize) -> Self {
|
||||
let capacity = capacity.max(1);
|
||||
let (sender, _receiver) = broadcast::channel(capacity);
|
||||
Self {
|
||||
sender,
|
||||
subscriber_count: Arc::new(AtomicUsize::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn subscriber_count(&self) -> usize {
|
||||
self.subscriber_count.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> TraceSubscription {
|
||||
let receiver = self.sender.subscribe();
|
||||
self.subscriber_count.fetch_add(1, Ordering::AcqRel);
|
||||
TraceSubscription {
|
||||
receiver,
|
||||
subscriber_count: Arc::clone(&self.subscriber_count),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn emit(&self, build: impl FnOnce() -> TraceEvent) -> bool {
|
||||
if self.subscriber_count() == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.sender.send(Arc::new(build())).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TraceBus {
|
||||
fn default() -> Self {
|
||||
Self::new(DEFAULT_TRACE_BUS_CAPACITY)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TraceSubscription {
|
||||
receiver: broadcast::Receiver<Arc<TraceEvent>>,
|
||||
subscriber_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl TraceSubscription {
|
||||
pub async fn recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::RecvError> {
|
||||
self.receiver.recv().await
|
||||
}
|
||||
|
||||
pub fn try_recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::TryRecvError> {
|
||||
self.receiver.try_recv()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TraceSubscription {
|
||||
fn drop(&mut self) {
|
||||
self.subscriber_count.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn global_trace_bus() -> &'static TraceBus {
|
||||
GLOBAL_TRACE_BUS.get_or_init(TraceBus::default)
|
||||
}
|
||||
|
||||
pub fn subscribe_trace_events() -> TraceSubscription {
|
||||
global_trace_bus().subscribe()
|
||||
}
|
||||
|
||||
pub fn trace_emit(build: impl FnOnce() -> TraceEvent) -> bool {
|
||||
global_trace_bus().emit(build)
|
||||
}
|
||||
|
||||
pub fn trace_subscriber_count() -> usize {
|
||||
global_trace_bus().subscriber_count()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
|
||||
#[test]
|
||||
fn trace_emit_skips_builder_without_subscribers() {
|
||||
let bus = TraceBus::new(4);
|
||||
let built = AtomicUsize::new(0);
|
||||
|
||||
let sent = bus.emit(|| {
|
||||
built.fetch_add(1, Ordering::Relaxed);
|
||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
|
||||
});
|
||||
|
||||
assert!(!sent);
|
||||
assert_eq!(built.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn trace_subscriber_receives_event() {
|
||||
let bus = TraceBus::new(4);
|
||||
let mut subscription = bus.subscribe();
|
||||
|
||||
assert!(bus.emit(|| {
|
||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject)
|
||||
.with_bucket("bucket")
|
||||
.with_object("object")
|
||||
.with_duration(Duration::from_millis(7))
|
||||
.with_bytes(11)
|
||||
.with_attr("dry", true)
|
||||
}));
|
||||
|
||||
let event = subscription
|
||||
.recv()
|
||||
.await
|
||||
.expect("subscriber should receive emitted trace event");
|
||||
|
||||
assert_eq!(event.kind, TraceKind::Heal);
|
||||
assert_eq!(event.func, TraceFunc::HealObject);
|
||||
assert_eq!(event.bucket.as_deref(), Some("bucket"));
|
||||
assert_eq!(event.object.as_deref(), Some("object"));
|
||||
assert_eq!(event.duration, Duration::from_millis(7));
|
||||
assert_eq!(event.bytes, 11);
|
||||
assert_eq!(
|
||||
event.attrs.as_slice(),
|
||||
&[TraceAttr {
|
||||
key: "dry",
|
||||
value: TraceVal::Bool(true)
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_subscription_drop_decrements_count() {
|
||||
let bus = TraceBus::new(4);
|
||||
let subscription = bus.subscribe();
|
||||
|
||||
assert_eq!(bus.subscriber_count(), 1);
|
||||
drop(subscription);
|
||||
assert_eq!(bus.subscriber_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lagged_subscriber_drops_events_without_blocking_publishers() {
|
||||
let bus = TraceBus::new(2);
|
||||
let mut subscription = bus.subscribe();
|
||||
|
||||
for index in 0_u64..4 {
|
||||
assert!(bus.emit(|| { TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder).with_attr("index", index) }));
|
||||
}
|
||||
|
||||
let err = subscription
|
||||
.recv()
|
||||
.await
|
||||
.expect_err("receiver should observe lag instead of blocking publishers");
|
||||
assert!(matches!(err, broadcast::error::RecvError::Lagged(_)));
|
||||
}
|
||||
}
|
||||
@@ -870,6 +870,157 @@ pub struct DataUsageCacheInfo {
|
||||
pub snapshot_complete: bool,
|
||||
}
|
||||
|
||||
/// Prefix-level usage over a raw entry map — the shared core behind
|
||||
/// [`DataUsageCache::prefix_usage`], usable by any cache-shaped reader (the
|
||||
/// scanner's writer-side cache has the same map type).
|
||||
///
|
||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
||||
/// names come straight off the child keys — no reverse mapping exists or is
|
||||
/// needed. A compacted prefix carries its aggregate but no children, which
|
||||
/// the `compacted` flag reports so callers can say why the breakdown is
|
||||
/// empty. `truncated` is set when the breakdown exceeded `max_entries` and
|
||||
/// was cut (largest first).
|
||||
pub fn prefix_usage_in_cache(
|
||||
cache: &HashMap<String, DataUsageEntry>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
max_entries: usize,
|
||||
) -> Option<PrefixUsageQuery> {
|
||||
let prefix = prefix.trim_matches('/');
|
||||
let root = if prefix.is_empty() {
|
||||
bucket.to_string()
|
||||
} else {
|
||||
format!("{bucket}/{prefix}")
|
||||
};
|
||||
let entry = cache.get(&hash_path(&root).key())?.clone();
|
||||
|
||||
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
|
||||
|
||||
let child_prefix = format!("{root}/");
|
||||
let mut sub_prefixes: Vec<PrefixUsageEntry> = entry
|
||||
.children
|
||||
.iter()
|
||||
.filter_map(|child_key| {
|
||||
let child = cache.get(child_key)?;
|
||||
let child_flat = flatten_entry(cache, child, 1)?;
|
||||
// Child keys are literal `bucket/pre/name` paths; a trailing
|
||||
// slash marks a directory object and is display-only here.
|
||||
let name = child_key
|
||||
.strip_prefix(child_prefix.as_str())
|
||||
.unwrap_or(child_key.as_str())
|
||||
.trim_end_matches('/')
|
||||
.to_string();
|
||||
Some(PrefixUsageEntry {
|
||||
prefix: name,
|
||||
usage: PrefixUsageSummary::from_entry(&child_flat),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
sub_prefixes.sort_by(|left, right| {
|
||||
right
|
||||
.usage
|
||||
.size
|
||||
.cmp(&left.usage.size)
|
||||
.then_with(|| left.prefix.cmp(&right.prefix))
|
||||
});
|
||||
let truncated = sub_prefixes.len() > max_entries;
|
||||
sub_prefixes.truncate(max_entries);
|
||||
|
||||
Some(PrefixUsageQuery {
|
||||
usage,
|
||||
compacted: entry.compacted,
|
||||
truncated,
|
||||
sub_prefixes,
|
||||
})
|
||||
}
|
||||
|
||||
/// Maximum subtree depth [`flatten_entry`] will walk before declaring the
|
||||
/// cache corrupt — the same bound the scanner's checked flatten uses.
|
||||
const PREFIX_USAGE_MAX_DEPTH: usize = 1024;
|
||||
|
||||
/// Flatten one entry's subtree into an aggregate: the free-function twin of
|
||||
/// [`DataUsageCache::flatten`], carrying the scanner checked-flatten
|
||||
/// hardening so a corrupt cache (cycles, over-deep trees, overflowing
|
||||
/// counters) yields `None` instead of unbounded recursion or wrapped totals.
|
||||
fn flatten_entry(cache: &HashMap<String, DataUsageEntry>, root: &DataUsageEntry, depth: usize) -> Option<DataUsageEntry> {
|
||||
if depth > PREFIX_USAGE_MAX_DEPTH {
|
||||
return None;
|
||||
}
|
||||
let mut flattened = DataUsageEntry::default();
|
||||
if !flattened.checked_merge(root) {
|
||||
return None;
|
||||
}
|
||||
flattened.compacted = root.compacted;
|
||||
// The root itself is not pre-seeded: it is merged above, and a corrupt
|
||||
// child edge pointing back at the root's own key is still terminated by
|
||||
// the visited set on first encounter.
|
||||
let mut visited: HashSet<&str> = HashSet::new();
|
||||
let mut pending: Vec<(&String, usize)> = root.children.iter().map(|child| (child, depth + 1)).collect();
|
||||
while let Some((key, child_depth)) = pending.pop() {
|
||||
if child_depth > PREFIX_USAGE_MAX_DEPTH || !visited.insert(key.as_str()) {
|
||||
return None;
|
||||
}
|
||||
let entry = cache.get(key)?;
|
||||
if !flattened.checked_merge(entry) {
|
||||
return None;
|
||||
}
|
||||
pending.extend(entry.children.iter().map(|child| (child, child_depth + 1)));
|
||||
}
|
||||
flattened.children.clear();
|
||||
Some(flattened)
|
||||
}
|
||||
|
||||
/// Flattened counters of one prefix subtree, as returned by
|
||||
/// [`DataUsageCache::prefix_usage`].
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PrefixUsageSummary {
|
||||
pub size: u64,
|
||||
pub objects: u64,
|
||||
pub versions: u64,
|
||||
pub delete_markers: u64,
|
||||
}
|
||||
|
||||
impl PrefixUsageSummary {
|
||||
fn from_entry(entry: &DataUsageEntry) -> Self {
|
||||
Self {
|
||||
size: entry.size as u64,
|
||||
objects: entry.objects as u64,
|
||||
versions: entry.versions as u64,
|
||||
delete_markers: entry.delete_markers as u64,
|
||||
}
|
||||
}
|
||||
|
||||
/// Add another set's counters into this one (entries are partitioned by
|
||||
/// set, so per-set results sum).
|
||||
pub fn merge(&mut self, other: &Self) {
|
||||
self.size = self.size.saturating_add(other.size);
|
||||
self.objects = self.objects.saturating_add(other.objects);
|
||||
self.versions = self.versions.saturating_add(other.versions);
|
||||
self.delete_markers = self.delete_markers.saturating_add(other.delete_markers);
|
||||
}
|
||||
}
|
||||
|
||||
/// One first-level sub-prefix row of a [`PrefixUsageQuery`].
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct PrefixUsageEntry {
|
||||
pub prefix: String,
|
||||
pub usage: PrefixUsageSummary,
|
||||
}
|
||||
|
||||
/// Result of [`DataUsageCache::prefix_usage`].
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PrefixUsageQuery {
|
||||
pub usage: PrefixUsageSummary,
|
||||
/// The prefix entry was compacted by the scanner: its aggregate is valid
|
||||
/// but no sub-prefix breakdown exists on disk.
|
||||
pub compacted: bool,
|
||||
/// The breakdown had more entries than `max_entries`; the largest remain.
|
||||
pub truncated: bool,
|
||||
pub sub_prefixes: Vec<PrefixUsageEntry>,
|
||||
}
|
||||
|
||||
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
||||
///
|
||||
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
||||
@@ -997,6 +1148,21 @@ impl DataUsageCache {
|
||||
}
|
||||
}
|
||||
|
||||
/// Prefix-level usage for one bucket subtree, plus the one-level
|
||||
/// breakdown below it (rustfs/backlog#1872, MinIO
|
||||
/// `loadPrefixUsageFromBackend` parity and beyond: arbitrary prefixes and
|
||||
/// full counters instead of first-level sizes only).
|
||||
///
|
||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
||||
/// names come straight off the child keys — no reverse mapping exists or
|
||||
/// is needed. A compacted prefix carries its aggregate but no children,
|
||||
/// which the `compacted` flag reports so callers can say why the
|
||||
/// breakdown is empty. `truncated` is set when the breakdown exceeded
|
||||
/// `max_entries` and was cut (largest first).
|
||||
pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option<PrefixUsageQuery> {
|
||||
prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries)
|
||||
}
|
||||
|
||||
pub fn force_compact(&mut self, limit: usize) {
|
||||
if self.cache.len() < limit {
|
||||
return;
|
||||
@@ -1898,6 +2064,126 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Build a cache shaped like `bucket/{a,b/{c,d}},bucket/loose` with
|
||||
/// distinct counters so aggregation is observable.
|
||||
fn prefix_usage_fixture_cache() -> DataUsageCache {
|
||||
let mut cache = DataUsageCache::default();
|
||||
let mut insert = |path: &str, parent: &str, size: usize, objects: usize, versions: usize, delete_markers: usize| {
|
||||
cache.replace(
|
||||
path,
|
||||
parent,
|
||||
DataUsageEntry {
|
||||
size,
|
||||
objects,
|
||||
versions,
|
||||
delete_markers,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
};
|
||||
insert("bucket", "", 0, 0, 0, 0);
|
||||
insert("bucket/a", "bucket", 100, 1, 1, 0);
|
||||
insert("bucket/b", "bucket", 0, 0, 0, 0);
|
||||
insert("bucket/b/c", "bucket/b", 200, 2, 2, 1);
|
||||
insert("bucket/b/d", "bucket/b", 40, 1, 3, 0);
|
||||
insert("bucket/loose", "bucket", 10, 1, 1, 1);
|
||||
cache
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_aggregates_bucket_root_and_one_level_below() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
|
||||
let root = cache
|
||||
.prefix_usage("bucket", "", 100)
|
||||
.expect("root query must find the bucket entry");
|
||||
assert_eq!(root.usage.size, 350, "root aggregate flattens the whole subtree");
|
||||
assert_eq!(root.usage.objects, 5);
|
||||
assert_eq!(root.usage.versions, 7);
|
||||
assert_eq!(root.usage.delete_markers, 2);
|
||||
assert!(!root.compacted);
|
||||
assert!(!root.truncated);
|
||||
// Breakdown is one level: b (240) before a (100) before loose (10),
|
||||
// each flattened to its own subtree total.
|
||||
let names: Vec<(&str, u64)> = root
|
||||
.sub_prefixes
|
||||
.iter()
|
||||
.map(|entry| (entry.prefix.as_str(), entry.usage.size))
|
||||
.collect();
|
||||
assert_eq!(names, vec![("b", 240), ("a", 100), ("loose", 10)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_drills_into_arbitrary_prefixes() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
|
||||
let b = cache.prefix_usage("bucket", "b", 100).expect("nested prefix must resolve");
|
||||
assert_eq!(b.usage.size, 240);
|
||||
assert_eq!(b.usage.versions, 5);
|
||||
let names: Vec<&str> = b.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
||||
assert_eq!(names, vec!["c", "d"]);
|
||||
|
||||
// Prefix slashes are normalized away.
|
||||
let slashed = cache.prefix_usage("bucket", "/b/", 100).expect("slash-insensitive lookup");
|
||||
assert_eq!(slashed.usage.size, 240);
|
||||
|
||||
assert!(cache.prefix_usage("bucket", "absent", 100).is_none(), "unknown prefix must be a miss");
|
||||
assert!(cache.prefix_usage("other", "", 100).is_none(), "unknown bucket must be a miss");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_reports_and_respects_truncation() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
let capped = cache.prefix_usage("bucket", "", 2).expect("root query");
|
||||
assert!(capped.truncated, "three children capped to two must flag truncation");
|
||||
let names: Vec<&str> = capped.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
||||
assert_eq!(names, vec!["b", "a"], "largest prefixes survive the cut");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_marks_compacted_entries() {
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
size: 999,
|
||||
objects: 9,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let compacted = cache.prefix_usage("bucket", "", 100).expect("compacted root resolves");
|
||||
assert!(compacted.compacted, "compaction must be visible to callers");
|
||||
assert_eq!(compacted.usage.size, 999);
|
||||
assert!(compacted.sub_prefixes.is_empty(), "a compacted entry carries no children");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_rejects_cyclic_and_dangling_caches() {
|
||||
// A self-referencing child (corrupt cache) must yield a miss for the
|
||||
// whole query, not unbounded recursion.
|
||||
let mut cache = prefix_usage_fixture_cache();
|
||||
if let Some(entry) = cache.cache.get_mut("bucket/b") {
|
||||
entry.children.insert("bucket/b".to_string());
|
||||
}
|
||||
assert!(cache.prefix_usage("bucket", "b", 100).is_none(), "a cyclic subtree must be rejected");
|
||||
// The unaffected sibling still answers.
|
||||
assert!(cache.prefix_usage("bucket", "a", 100).is_some());
|
||||
|
||||
// A child key with no entry (dangling link) is rejected rather than
|
||||
// silently dropped: half a tree would under-report usage.
|
||||
let mut dangling = prefix_usage_fixture_cache();
|
||||
if let Some(entry) = dangling.cache.get_mut("bucket/b") {
|
||||
entry.children.insert("bucket/b/ghost".to_string());
|
||||
}
|
||||
assert!(
|
||||
dangling.prefix_usage("bucket", "b", 100).is_none(),
|
||||
"a dangling child link must be rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
|
||||
@@ -32,6 +32,7 @@ use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::ffi::OsStr;
|
||||
use std::fs as stdfs;
|
||||
use std::io::ErrorKind;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::Once;
|
||||
@@ -51,6 +52,11 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
|
||||
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||
const TEST_PORT_MIN: u16 = 20_000;
|
||||
const TEST_PORT_RANGE: u16 = 40_000;
|
||||
const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
|
||||
const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
|
||||
const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
|
||||
|
||||
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
||||
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
||||
@@ -67,6 +73,64 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
struct PortAllocatorGuard;
|
||||
|
||||
impl PortAllocatorGuard {
|
||||
async fn acquire() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
loop {
|
||||
match stdfs::create_dir(TEST_PORT_LOCK_DIR) {
|
||||
Ok(()) => return Ok(Self),
|
||||
Err(err) if err.kind() == ErrorKind::AlreadyExists => {
|
||||
remove_stale_port_allocator_lock();
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PortAllocatorGuard {
|
||||
fn drop(&mut self) {
|
||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_test_port(port: u16) -> u16 {
|
||||
let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE;
|
||||
TEST_PORT_MIN + offset
|
||||
}
|
||||
|
||||
fn seeded_test_port() -> u16 {
|
||||
let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16;
|
||||
TEST_PORT_MIN + offset
|
||||
}
|
||||
|
||||
fn read_next_test_port() -> u16 {
|
||||
stdfs::read_to_string(TEST_PORT_COUNTER_PATH)
|
||||
.ok()
|
||||
.and_then(|value| value.trim().parse::<u16>().ok())
|
||||
.filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port))
|
||||
.unwrap_or_else(seeded_test_port)
|
||||
}
|
||||
|
||||
fn remove_stale_port_allocator_lock() {
|
||||
let Ok(metadata) = stdfs::metadata(TEST_PORT_LOCK_DIR) else {
|
||||
return;
|
||||
};
|
||||
let Ok(modified) = metadata.modified() else {
|
||||
return;
|
||||
};
|
||||
if modified.elapsed().is_ok_and(|elapsed| elapsed > TEST_PORT_LOCK_STALE_AFTER) {
|
||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_next_test_port(port: u16) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
stdfs::write(TEST_PORT_COUNTER_PATH, port.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn capture_command_logs(
|
||||
command: &mut Command,
|
||||
log_path: Option<&str>,
|
||||
@@ -508,10 +572,21 @@ impl RustFSTestEnvironment {
|
||||
/// Find an available port for the test
|
||||
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
||||
use std::net::TcpListener;
|
||||
let listener = TcpListener::bind("127.0.0.1:0")?;
|
||||
let port = listener.local_addr()?.port();
|
||||
drop(listener);
|
||||
Ok(port)
|
||||
let _guard = PortAllocatorGuard::acquire().await?;
|
||||
let mut next_port = read_next_test_port();
|
||||
|
||||
for _ in 0..TEST_PORT_RANGE {
|
||||
let port = next_port;
|
||||
next_port = advance_test_port(next_port);
|
||||
write_next_test_port(next_port)?;
|
||||
|
||||
if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) {
|
||||
drop(listener);
|
||||
return Ok(port);
|
||||
}
|
||||
}
|
||||
|
||||
Err("no available E2E test port found".into())
|
||||
}
|
||||
|
||||
/// Kill any existing RustFS processes
|
||||
|
||||
@@ -373,14 +373,14 @@ pub mod error {
|
||||
|
||||
pub mod erasure {
|
||||
pub use crate::erasure::coding::{
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, ReedSolomonEncoder,
|
||||
calc_shard_size, calc_shard_size_legacy,
|
||||
BitrotReader, BitrotSelfTestError, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError,
|
||||
ReedSolomonEncoder, bitrot_self_test, calc_shard_size, calc_shard_size_legacy,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod event {
|
||||
pub use crate::event::name::EventName;
|
||||
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook};
|
||||
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook, send_event};
|
||||
}
|
||||
|
||||
pub mod global {
|
||||
@@ -483,6 +483,7 @@ pub mod store_list {
|
||||
}
|
||||
|
||||
pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||
|
||||
@@ -1549,8 +1549,8 @@ impl Default for PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjectOptions {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
@@ -1561,6 +1561,7 @@ impl PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag_except(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
@@ -1696,6 +1697,7 @@ impl PutObjectOptions {
|
||||
header
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn validate(&self, _c: Arc<TargetClient>) -> Result<(), std::io::Error> {
|
||||
//if self.checksum.is_set() {
|
||||
/*if !self.trailing_header_support {
|
||||
|
||||
@@ -456,16 +456,23 @@ impl<'a> LifecycleExpiryTrace<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ExpiryStats {
|
||||
pub fn missed_tasks(&self) -> i64 {
|
||||
self.missed_expiry_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
fn missed_free_vers_tasks(&self) -> i64 {
|
||||
self.missed_freevers_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
fn missed_tier_journal_tasks(&self) -> i64 {
|
||||
self.missed_tier_journal_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ pub mod core;
|
||||
pub mod evaluator;
|
||||
pub mod manual_transition_job;
|
||||
mod metadata_boundary;
|
||||
pub(crate) use metadata_boundary::get_expiry_configs;
|
||||
pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs};
|
||||
mod object_lock_boundary;
|
||||
pub use self::core as lifecycle;
|
||||
mod replication_sink;
|
||||
|
||||
@@ -80,7 +80,10 @@ impl LastDayTierStats {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
|
||||
let mut cl = self.clone();
|
||||
let mut cm = m;
|
||||
|
||||
@@ -177,9 +177,10 @@ fn should_record_remote_delete_failure(err: &std::io::Error) -> bool {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
struct ObjSweeper {
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
object: String,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
bucket: String,
|
||||
version_id: Option<Uuid>,
|
||||
versioned: bool,
|
||||
@@ -191,9 +192,9 @@ struct ObjSweeper {
|
||||
remote_object: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ObjSweeper {
|
||||
#[allow(clippy::new_ret_no_self)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
|
||||
Ok(Self {
|
||||
object: object.into(),
|
||||
@@ -202,17 +203,20 @@ impl ObjSweeper {
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
|
||||
self.version_id = vid.clone();
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
|
||||
self.versioned = versioned;
|
||||
self.suspended = suspended;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
|
||||
let mut opts = ObjectOpts {
|
||||
version_id: self.version_id.clone(),
|
||||
@@ -226,6 +230,7 @@ impl ObjSweeper {
|
||||
opts
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn set_transition_state(&mut self, info: TransitionedObject) {
|
||||
self.transition_tier = info.tier;
|
||||
self.transition_status = info.status;
|
||||
@@ -266,6 +271,7 @@ impl ObjSweeper {
|
||||
None
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub async fn sweep(&self, api: Arc<ECStore>) {
|
||||
let Some(je) = self.should_remove_remote_object() else {
|
||||
return;
|
||||
|
||||
@@ -312,9 +312,7 @@ mod tests {
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct LegacyBucketQuota {
|
||||
#[allow(dead_code)]
|
||||
quota: Option<u64>,
|
||||
#[allow(dead_code)]
|
||||
quota_type: LegacyQuotaType,
|
||||
}
|
||||
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
|
||||
|
||||
@@ -95,7 +95,6 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct GetRequest {
|
||||
pub buffer: Vec<u8>,
|
||||
pub offset: i64,
|
||||
@@ -107,11 +106,12 @@ pub struct GetRequest {
|
||||
pub setting_object_info: bool,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub struct GetResponse {
|
||||
pub size: i64,
|
||||
//pub error: error,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
pub did_read: bool,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
pub object_info: ObjectInfo,
|
||||
}
|
||||
|
||||
|
||||
@@ -27,7 +27,6 @@ use tracing::warn;
|
||||
use crate::client::api_error_response::err_invalid_argument;
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AdvancedGetOptions {
|
||||
pub replication_delete_marker: bool,
|
||||
pub is_replication_ready_for_delete_marker: bool,
|
||||
|
||||
@@ -360,7 +360,6 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct ListObjectsOptions {
|
||||
reverse_versions: bool,
|
||||
with_versions: bool,
|
||||
|
||||
@@ -137,8 +137,8 @@ impl Default for PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjectOptions {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
|
||||
@@ -149,6 +149,7 @@ impl PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag_except(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
|
||||
@@ -259,6 +260,7 @@ impl PutObjectOptions {
|
||||
header
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
|
||||
//if self.checksum.is_set() {
|
||||
/*if !self.trailing_header_support {
|
||||
|
||||
@@ -55,7 +55,6 @@ pub struct RemoveBucketOptions {
|
||||
const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024;
|
||||
|
||||
#[derive(Debug)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AdvancedRemoveOptions {
|
||||
pub replication_delete_marker: bool,
|
||||
pub replication_status: ReplicationStatus,
|
||||
@@ -465,10 +464,10 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct RemoveObjectError {
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
object_name: String,
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
version_id: String,
|
||||
err: Option<std::io::Error>,
|
||||
}
|
||||
|
||||
@@ -372,8 +372,8 @@ pub struct Checksum {
|
||||
computed: bool,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl Checksum {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
|
||||
if t.is_set() && b.len() == t.raw_byte_len() {
|
||||
return Checksum {
|
||||
@@ -385,7 +385,7 @@ impl Checksum {
|
||||
Checksum::default()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
|
||||
let b = match base64_decode(s.as_bytes()) {
|
||||
Ok(b) => b,
|
||||
@@ -412,7 +412,7 @@ impl Checksum {
|
||||
base64_encode(&self.r)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn raw(&self) -> Option<Vec<u8>> {
|
||||
if !self.is_set() {
|
||||
return None;
|
||||
|
||||
@@ -37,16 +37,17 @@ pub struct PutObjReader {
|
||||
//pub sealMD5Fn: SealMD5CurrFn,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjReader {
|
||||
pub fn new(reader: HashReader) -> Self {
|
||||
Self { reader }
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn md5_current_hex_string(&self) -> String {
|
||||
self.reader.checksum().map(|v| v.encoded).unwrap_or_default()
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
|
||||
self.reader = enc_reader;
|
||||
|
||||
|
||||
@@ -214,6 +214,19 @@ fn pool_write_quorum(participant_count: usize) -> usize {
|
||||
(participant_count / 2) + 1
|
||||
}
|
||||
|
||||
/// Error for a peer that reported `success = false` without an error payload.
|
||||
///
|
||||
/// The message must stay identical across the peers of one operation: `reduce_errs`
|
||||
/// buckets `Error::Io` by kind plus rendered message, so any per-peer detail (address,
|
||||
/// timing) would split one shared failure into single-count buckets and downgrade a real
|
||||
/// dominant error into `ErasureWriteQuorum`.
|
||||
fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error {
|
||||
match bucket {
|
||||
Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")),
|
||||
None => Error::other(format!("{op}: peer returned failure without error details")),
|
||||
}
|
||||
}
|
||||
|
||||
fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Error> {
|
||||
if per_pool_errs.is_empty() {
|
||||
return Some(Error::ErasureWriteQuorum);
|
||||
@@ -1078,7 +1091,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(Error::other(""))
|
||||
Err(peer_failure_without_details("heal_bucket", Some(bucket)))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1105,7 +1118,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(Error::other(""))
|
||||
Err(peer_failure_without_details("list_bucket", None))
|
||||
};
|
||||
}
|
||||
let bucket_infos = response
|
||||
@@ -1136,9 +1149,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(Error::other(format!(
|
||||
"make_bucket({bucket}): peer returned failure without error details"
|
||||
)))
|
||||
Err(peer_failure_without_details("make_bucket", Some(bucket)))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1162,7 +1173,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(Error::other(""))
|
||||
Err(peer_failure_without_details("get_bucket_info", Some(bucket)))
|
||||
};
|
||||
}
|
||||
let bucket_info = serde_json::from_str::<BucketInfo>(&response.bucket_info)?;
|
||||
@@ -1190,7 +1201,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(Error::other(""))
|
||||
Err(peer_failure_without_details("delete_bucket", Some(bucket)))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2314,4 +2325,37 @@ mod tests {
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(calls, vec![1, 1, 0, 0, 0, 0, 0, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_names_operation_and_bucket() {
|
||||
for op in ["heal_bucket", "make_bucket", "get_bucket_info", "delete_bucket"] {
|
||||
let message = peer_failure_without_details(op, Some("ops-bucket")).to_string();
|
||||
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
||||
assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}");
|
||||
}
|
||||
|
||||
let message = peer_failure_without_details("list_bucket", None).to_string();
|
||||
assert!(message.contains("list_bucket"), "cluster-wide message must name the operation");
|
||||
assert!(!message.trim().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() {
|
||||
// reduce_errs groups Io errors by kind plus rendered message: peers failing the
|
||||
// same operation on the same bucket must still reach quorum as one dominant error.
|
||||
let per_pool_errs = vec![
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||
];
|
||||
assert_eq!(
|
||||
reduce_pool_write_quorum_errs(&per_pool_errs),
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared")))
|
||||
);
|
||||
|
||||
assert_ne!(
|
||||
peer_failure_without_details("delete_bucket", Some("shared")),
|
||||
peer_failure_without_details("get_bucket_info", Some("shared"))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,7 +39,6 @@ use rustfs_config::{
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
/// Default KVS for audit webhook settings.
|
||||
pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
@@ -117,7 +116,6 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
])
|
||||
});
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
/// Default KVS for audit MQTT settings.
|
||||
pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
@@ -375,7 +373,6 @@ pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
])
|
||||
});
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
|
||||
@@ -12,12 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use rustfs_config::server_config::{KV, KVS};
|
||||
use rustfs_config::{DEFAULT_HEAL_BITROT_CYCLE_SECS, HEAL_BITROT_CYCLE};
|
||||
use rustfs_utils::string::parse_bool;
|
||||
use std::sync::LazyLock;
|
||||
use std::time::Duration;
|
||||
|
||||
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![KV {
|
||||
@@ -26,59 +23,3 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
hidden_if_empty: false,
|
||||
}])
|
||||
});
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Config {
|
||||
pub bitrot: String,
|
||||
pub sleep: Duration,
|
||||
pub io_count: usize,
|
||||
pub drive_workers: usize,
|
||||
pub cache: Duration,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn bitrot_scan_cycle(&self) -> Duration {
|
||||
self.cache
|
||||
}
|
||||
|
||||
pub fn get_workers(&self) -> usize {
|
||||
self.drive_workers
|
||||
}
|
||||
|
||||
pub fn update(&mut self, nopts: &Config) {
|
||||
self.bitrot = nopts.bitrot.clone();
|
||||
self.io_count = nopts.io_count;
|
||||
self.sleep = nopts.sleep;
|
||||
self.drive_workers = nopts.drive_workers;
|
||||
}
|
||||
}
|
||||
|
||||
const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1;
|
||||
|
||||
fn parse_bitrot_config(s: &str) -> Result<Duration> {
|
||||
match parse_bool(s) {
|
||||
Ok(enabled) => {
|
||||
if enabled {
|
||||
Ok(Duration::from_secs_f64(0.0))
|
||||
} else {
|
||||
Ok(Duration::from_secs_f64(-1.0))
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
if !s.ends_with("m") {
|
||||
return Err(Error::other("unknown format"));
|
||||
}
|
||||
|
||||
match s.trim_end_matches('m').parse::<u64>() {
|
||||
Ok(months) => {
|
||||
if months < RUSTFS_BITROT_CYCLE_IN_MONTHS {
|
||||
return Err(Error::other(format!("minimum bitrot cycle is {RUSTFS_BITROT_CYCLE_IN_MONTHS} month(s)")));
|
||||
}
|
||||
|
||||
Ok(Duration::from_secs(months * 30 * 24 * 60))
|
||||
}
|
||||
Err(err) => Err(Error::other(err)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
|
||||
mod audit;
|
||||
pub mod com;
|
||||
#[allow(dead_code)]
|
||||
pub mod heal;
|
||||
mod notify;
|
||||
mod oidc;
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::bucket::replication::replication_state_from_filemeta;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::bucket::{
|
||||
lifecycle::{
|
||||
LifecycleExpiryConfigs,
|
||||
bucket_lifecycle_audit::LcEventSrc,
|
||||
bucket_lifecycle_ops::{
|
||||
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle,
|
||||
@@ -1996,11 +1997,11 @@ impl PoolMeta {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
|
||||
struct PoolInfo {
|
||||
position: usize,
|
||||
completed: bool,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
decom_started: bool,
|
||||
}
|
||||
|
||||
@@ -2335,6 +2336,10 @@ fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool {
|
||||
action.delete()
|
||||
}
|
||||
|
||||
fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
|
||||
if !apply_actions || applied {
|
||||
return Ok(true);
|
||||
@@ -2385,7 +2390,80 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HealLifecycleExpiryContext {
|
||||
configs: LifecycleExpiryConfigs,
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
if bucket == RUSTFS_META_BUCKET {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let configs = get_expiry_configs(self, bucket).await?;
|
||||
if configs.lifecycle.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
Ok(Some(HealLifecycleExpiryContext { configs }))
|
||||
}
|
||||
|
||||
pub async fn enqueue_heal_lifecycle_expiry(
|
||||
self: &Arc<Self>,
|
||||
context: &HealLifecycleExpiryContext,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
object_info: Option<&crate::object_api::ObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
let object_info = if let Some(object_info) = object_info {
|
||||
if object_info.bucket != bucket || object_info.name != object {
|
||||
return Ok(false);
|
||||
}
|
||||
let snapshot_version_id = object_info
|
||||
.version_id
|
||||
.filter(|version_id| !version_id.is_nil())
|
||||
.map(|version_id| version_id.to_string());
|
||||
if snapshot_version_id.as_deref() != version_id {
|
||||
return Ok(false);
|
||||
}
|
||||
object_info.clone()
|
||||
} else {
|
||||
match self
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_id: version_id.map(str::to_string),
|
||||
versioned: version_id.is_some(),
|
||||
expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(object_info) => object_info,
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false),
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
};
|
||||
|
||||
let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await;
|
||||
if !lifecycle_action_skips_heal_version(event.action) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await)
|
||||
}
|
||||
|
||||
async fn save_current_pool_meta(&self) -> Result<()> {
|
||||
let _save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let snapshot = {
|
||||
@@ -4287,6 +4365,19 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_action_skips_heal_version_for_every_delete_action() {
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction));
|
||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction));
|
||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction));
|
||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() {
|
||||
let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false)
|
||||
@@ -4958,13 +5049,19 @@ fn is_disk_online_state(state: &str) -> bool {
|
||||
}
|
||||
|
||||
#[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")]
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
|
||||
)]
|
||||
fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
||||
fallback_total_capacity_dedup(disks)
|
||||
}
|
||||
|
||||
#[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")]
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
|
||||
)]
|
||||
fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
||||
fallback_free_capacity_dedup(disks)
|
||||
}
|
||||
|
||||
@@ -1140,11 +1140,11 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
|
||||
Err(Error::DiskNotFound)
|
||||
}
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
|
||||
// Multipart orphan reconciliation is intentionally retained above the pool/set layers
|
||||
// until there is a concrete caller and a stable lower-level contract to implement.
|
||||
Err(StorageError::NotImplemented)
|
||||
#[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))]
|
||||
async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
||||
self.get_disks_for_heal_object(object, opts)?
|
||||
.check_abandoned_parts(bucket, object, opts)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1996,7 +1996,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sets_check_abandoned_parts_returns_typed_not_implemented_error() {
|
||||
async fn sets_check_abandoned_parts_rejects_invalid_set_scope() {
|
||||
let format = FormatV3::new(1, 1);
|
||||
let sets = Sets {
|
||||
id: format.id,
|
||||
@@ -2021,10 +2021,21 @@ mod tests {
|
||||
};
|
||||
|
||||
let err = sets
|
||||
.check_abandoned_parts("bucket", "object", &HealOpts::default())
|
||||
.check_abandoned_parts(
|
||||
"bucket",
|
||||
"object",
|
||||
&HealOpts {
|
||||
set: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("abandoned-parts ownership should stay above the pool/set storage layers");
|
||||
assert!(matches!(err, StorageError::NotImplemented));
|
||||
.expect_err("out-of-range abandoned-parts set scope must fail closed");
|
||||
assert!(
|
||||
matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
|
||||
if field == "set" && reason.contains("invalid heal set index 1")),
|
||||
"unexpected invalid set error: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// Builds a single-set `Sets` over `SET_DRIVE_COUNT` local temp-dir disks,
|
||||
|
||||
@@ -6562,7 +6562,7 @@ impl LocalDisk {
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn get_metrics(&self) -> DiskMetrics {
|
||||
DiskMetrics::default()
|
||||
}
|
||||
|
||||
@@ -820,10 +820,263 @@ impl BitrotWriterWrapper {
|
||||
}
|
||||
}
|
||||
|
||||
// --- startup bitrot self-test (rustfs/backlog#1873, MinIO bitrotSelfTest parity) ---
|
||||
//
|
||||
// A broken hash implementation (bad SIMD feature combination, platform drift, a
|
||||
// key-handling regression) fails silently: every shard reads back "corrupt",
|
||||
// heal rewrites data that was fine, and cross-platform clusters disagree about
|
||||
// which copy is healthy. The self-test below pins the algorithms the moment a
|
||||
// process starts, so a drifted build announces itself instead of quietly
|
||||
// rewriting objects. See docs/rustfs-heal-scanner-vs-minio-comprehensive-
|
||||
// analysis-2026-08-16.md §6 HS-11.
|
||||
|
||||
/// Length of the deterministic self-test payload.
|
||||
pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096;
|
||||
|
||||
/// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S`
|
||||
/// (the production default). Pinned so any platform or build where the
|
||||
/// implementation drifts fails startup instead of miss-hashing shards.
|
||||
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [
|
||||
0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea,
|
||||
0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06,
|
||||
];
|
||||
|
||||
/// Known-answer digest of the same payload under `HighwayHash256SLegacy`.
|
||||
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY: [u8; 32] = [
|
||||
0x98, 0x24, 0x71, 0x4f, 0x16, 0xbb, 0x48, 0x39, 0xed, 0x68, 0xfa, 0x63, 0x5e, 0xd9, 0x07, 0x61, 0xdf, 0x0a, 0xff, 0xcf, 0x7d,
|
||||
0x8c, 0xa8, 0xc7, 0xc0, 0xb6, 0x6f, 0x05, 0xdb, 0xda, 0x5a, 0x22,
|
||||
];
|
||||
|
||||
/// FIPS 180-2 test vector: SHA-256 of the ASCII string "abc". Unlike the
|
||||
/// Highway digests above this one is externally verifiable, so it guards the
|
||||
/// whole `HashAlgorithm` plumbing even for readers who distrust pinned
|
||||
/// self-computed constants.
|
||||
const BITROT_SELF_TEST_KAT_SHA256_ABC: [u8; 32] = [
|
||||
0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96,
|
||||
0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad,
|
||||
];
|
||||
|
||||
/// Deterministic self-test payload: xorshift64* from a fixed seed, so every
|
||||
/// platform and every run hashes the same 4096 bytes.
|
||||
fn bitrot_self_test_payload() -> [u8; BITROT_SELF_TEST_PAYLOAD_LEN] {
|
||||
let mut state = 0x9E37_79B9_7F4A_7C15u64;
|
||||
let mut payload = [0u8; BITROT_SELF_TEST_PAYLOAD_LEN];
|
||||
for byte in payload.iter_mut() {
|
||||
state ^= state >> 12;
|
||||
state ^= state << 25;
|
||||
state ^= state >> 27;
|
||||
*byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D) as u8;
|
||||
}
|
||||
payload
|
||||
}
|
||||
|
||||
/// Why a bitrot self-test failed.
|
||||
#[derive(Debug)]
|
||||
pub enum BitrotSelfTestError {
|
||||
/// A known-answer digest mismatched the pinned constant.
|
||||
KnownAnswerMismatch {
|
||||
algorithm: &'static str,
|
||||
got: String,
|
||||
want: String,
|
||||
},
|
||||
/// A freshly encoded shard failed `bitrot_verify`.
|
||||
RoundtripVerify { algorithm: &'static str, detail: String },
|
||||
/// A verified roundtrip read back different bytes than were written.
|
||||
RoundtripReadback { algorithm: &'static str },
|
||||
/// A deliberately tampered shard was not rejected by `bitrot_verify`.
|
||||
TamperNotRejected {
|
||||
algorithm: &'static str,
|
||||
tampered: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Display for BitrotSelfTestError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::KnownAnswerMismatch { algorithm, got, want } => {
|
||||
write!(f, "known-answer mismatch for {algorithm}: got {got}, want {want}")
|
||||
}
|
||||
Self::RoundtripVerify { algorithm, detail } => write!(f, "{algorithm} roundtrip shard failed verification: {detail}"),
|
||||
Self::RoundtripReadback { algorithm } => write!(f, "{algorithm} roundtrip read back different bytes"),
|
||||
Self::TamperNotRejected { algorithm, tampered } => {
|
||||
write!(f, "{algorithm} tampered shard ({tampered}) was not rejected")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for BitrotSelfTestError {}
|
||||
|
||||
fn self_test_hex(bytes: &[u8]) -> String {
|
||||
rustfs_utils::hex(bytes)
|
||||
}
|
||||
|
||||
// (kept as a named one-liner so every KAT failure site reads the same; the
|
||||
// underlying formatter is the shared `rustfs_utils::hex`)
|
||||
|
||||
/// Compare a digest against its pinned constant. Split out so a test can drive
|
||||
/// it with a wrong constant and prove the mismatch path fires.
|
||||
fn bitrot_kat_check(
|
||||
algorithm: &'static str,
|
||||
algo: &HashAlgorithm,
|
||||
payload: &[u8],
|
||||
expected: &[u8; 32],
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let digest = algo.hash_encode(payload);
|
||||
let digest = digest.as_ref();
|
||||
if digest.len() != expected.len() || digest != expected.as_slice() {
|
||||
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
||||
algorithm,
|
||||
got: self_test_hex(digest),
|
||||
want: self_test_hex(expected),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encode `payload` with `shard_size` blocks, verify it end to end, and read
|
||||
/// every block back through `BitrotReader` comparing bytes.
|
||||
async fn bitrot_roundtrip_check(
|
||||
algorithm: &'static str,
|
||||
algo: HashAlgorithm,
|
||||
payload: &[u8],
|
||||
shard_size: usize,
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer
|
||||
.write(chunk)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("encode failed: {err}"),
|
||||
})?;
|
||||
}
|
||||
let encoded = writer.into_inner().into_inner();
|
||||
|
||||
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
||||
if encoded.len() != on_disk {
|
||||
return Err(BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("encoded {} bytes, size formula says {on_disk}", encoded.len()),
|
||||
});
|
||||
}
|
||||
bitrot_verify(std::io::Cursor::new(encoded.clone()), on_disk, payload.len(), algo.clone(), shard_size)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: err.to_string(),
|
||||
})?;
|
||||
|
||||
let mut reader = BitrotReader::new(std::io::Cursor::new(encoded), shard_size, algo, false);
|
||||
let mut offset = 0usize;
|
||||
while offset < payload.len() {
|
||||
let want = shard_size.min(payload.len() - offset);
|
||||
let mut buf = vec![0u8; want];
|
||||
let read = reader
|
||||
.read(&mut buf)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("read back failed at offset {offset}: {err}"),
|
||||
})?;
|
||||
if read != want || buf[..read] != payload[offset..offset + read] {
|
||||
return Err(BitrotSelfTestError::RoundtripReadback { algorithm });
|
||||
}
|
||||
offset += read;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flip one byte and require `bitrot_verify` to reject the result.
|
||||
async fn bitrot_tamper_check(
|
||||
algorithm: &'static str,
|
||||
algo: HashAlgorithm,
|
||||
payload: &[u8],
|
||||
shard_size: usize,
|
||||
tampered: &'static str,
|
||||
flip_at: usize,
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer.write(chunk).await.expect("self-test encode should not fail");
|
||||
}
|
||||
let mut corrupt = writer.into_inner().into_inner();
|
||||
let flip_index = flip_at % corrupt.len();
|
||||
corrupt[flip_index] ^= 0x80;
|
||||
|
||||
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
||||
match bitrot_verify(std::io::Cursor::new(corrupt), on_disk, payload.len(), algo, shard_size).await {
|
||||
// The flipped byte must be rejected as a hash mismatch specifically, not
|
||||
// by any incidental read error: an in-memory cursor cannot fail reads,
|
||||
// so accepting any other failure here would mask a verify path that
|
||||
// errors out before it ever compares hashes.
|
||||
Err(err) if err.to_string().contains("hash mismatch") => Ok(()),
|
||||
Ok(()) => Err(BitrotSelfTestError::TamperNotRejected { algorithm, tampered }),
|
||||
Err(err) => Err(BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("tampered shard rejected with an unexpected error: {err}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Verify every bitrot algorithm this crate can write or verify in production:
|
||||
/// both streaming Highway variants roundtrip end to end (encode → size formula
|
||||
/// → `bitrot_verify` → read back) and reject a flipped byte in both the data
|
||||
/// and the leading hash, while all three hashed algorithms reproduce their
|
||||
/// pinned known-answer digests.
|
||||
///
|
||||
/// Runs in well under a millisecond on 4 KiB of data; callers may run it inline
|
||||
/// at startup. Pure CPU, no allocation beyond a few KiB of scratch.
|
||||
pub async fn bitrot_self_test() -> Result<(), BitrotSelfTestError> {
|
||||
let payload = bitrot_self_test_payload();
|
||||
|
||||
// Externally verifiable vector first: it guards the HashAlgorithm plumbing
|
||||
// itself, before any self-pinned constants are consulted.
|
||||
let abc = HashAlgorithm::SHA256.hash_encode(b"abc");
|
||||
if abc.as_ref() != BITROT_SELF_TEST_KAT_SHA256_ABC.as_slice() {
|
||||
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
||||
algorithm: "SHA256",
|
||||
got: self_test_hex(abc.as_ref()),
|
||||
want: self_test_hex(&BITROT_SELF_TEST_KAT_SHA256_ABC),
|
||||
});
|
||||
}
|
||||
|
||||
bitrot_kat_check(
|
||||
"HighwayHash256S",
|
||||
&HashAlgorithm::HighwayHash256S,
|
||||
&payload,
|
||||
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S,
|
||||
)?;
|
||||
bitrot_kat_check(
|
||||
"HighwayHash256SLegacy",
|
||||
&HashAlgorithm::HighwayHash256SLegacy,
|
||||
&payload,
|
||||
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY,
|
||||
)?;
|
||||
|
||||
for (algorithm, algo) in [
|
||||
("HighwayHash256S", HashAlgorithm::HighwayHash256S),
|
||||
("HighwayHash256SLegacy", HashAlgorithm::HighwayHash256SLegacy),
|
||||
] {
|
||||
// Full blocks plus a partial tail, exactly like a real part stripe.
|
||||
let tail_len = 2 * 1024 + 333;
|
||||
bitrot_roundtrip_check(algorithm, algo.clone(), &payload, 1024).await?;
|
||||
bitrot_roundtrip_check(algorithm, algo.clone(), &payload[..tail_len], 1024).await?;
|
||||
// One flipped byte in the final data block, one in the first leading
|
||||
// hash: both must fail verification.
|
||||
bitrot_tamper_check(algorithm, algo.clone(), &payload, 1024, "final data byte", payload.len() - 1).await?;
|
||||
bitrot_tamper_check(algorithm, algo, &payload, 1024, "leading hash byte", 0).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_kat_check, bitrot_self_test,
|
||||
bitrot_self_test_payload, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
||||
};
|
||||
use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource};
|
||||
use bytes::Bytes;
|
||||
@@ -1090,6 +1343,32 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_self_test_payload_is_deterministic() {
|
||||
// Two independent builds of the payload must agree byte for byte, or
|
||||
// the pinned known-answer digests below would be meaningless.
|
||||
assert_eq!(bitrot_self_test_payload(), bitrot_self_test_payload());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_self_test_rejects_a_wrong_known_answer_digest() {
|
||||
let payload = bitrot_self_test_payload();
|
||||
let wrong = [0u8; 32];
|
||||
let err = bitrot_kat_check("HighwayHash256S", &HashAlgorithm::HighwayHash256S, &payload, &wrong)
|
||||
.expect_err("a zeroed digest must never match");
|
||||
match err {
|
||||
super::BitrotSelfTestError::KnownAnswerMismatch { algorithm, .. } => assert_eq!(algorithm, "HighwayHash256S"),
|
||||
other => panic!("expected KnownAnswerMismatch, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_self_test_passes() {
|
||||
bitrot_self_test()
|
||||
.await
|
||||
.expect("the pinned digests and roundtrip checks must all pass on this platform");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() {
|
||||
let mut counting = VectoredCountingWriter::default();
|
||||
@@ -1189,7 +1468,7 @@ mod tests {
|
||||
let last = corrupt.len() - 1;
|
||||
corrupt[last] ^= 0x80;
|
||||
let err = bitrot_verify(
|
||||
Cursor::new(corrupt),
|
||||
std::io::Cursor::new(corrupt),
|
||||
super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()),
|
||||
data.len(),
|
||||
algo,
|
||||
@@ -1282,7 +1561,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() {
|
||||
let mut reader = BitrotReader::new(Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
||||
let mut reader = BitrotReader::new(std::io::Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
||||
let mut out = [0u8; 5];
|
||||
let err = reader
|
||||
.read(&mut out)
|
||||
@@ -1407,7 +1686,7 @@ mod tests {
|
||||
(HashAlgorithm::HighwayHash256, true),
|
||||
] {
|
||||
let label = format!("{algo:?}");
|
||||
let writer = Cursor::new(Vec::<u8>::new());
|
||||
let writer = std::io::Cursor::new(Vec::<u8>::new());
|
||||
let mut w = BitrotWriter::new(writer, shard_size, algo.clone());
|
||||
w.write(&[7u8; 16]).await.unwrap();
|
||||
let written = w.into_inner().into_inner();
|
||||
@@ -1492,7 +1771,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec<u8> {
|
||||
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
||||
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
||||
w.write(payload).await.unwrap();
|
||||
w.into_inner().into_inner()
|
||||
}
|
||||
@@ -1600,7 +1879,7 @@ mod tests {
|
||||
for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] {
|
||||
for &size in &[1usize, 16, 17, 32, 40, 48] {
|
||||
let payload: Vec<u8> = (0..size).map(|i| i as u8).collect();
|
||||
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
w.write(chunk).await.unwrap();
|
||||
}
|
||||
@@ -1674,14 +1953,14 @@ mod tests {
|
||||
w.write(&data).await.expect("write shard");
|
||||
|
||||
let mut via_read = vec![0u8; SHARD];
|
||||
let n1 = BitrotReader::new(Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
||||
let n1 = BitrotReader::new(std::io::Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
||||
.read(&mut via_read)
|
||||
.await
|
||||
.expect("read");
|
||||
|
||||
// A buffer with only capacity — no initialized bytes at all.
|
||||
let mut via_append: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let n2 = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
let n2 = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
.read_appending(&mut via_append, SHARD)
|
||||
.await
|
||||
.expect("read_appending");
|
||||
@@ -1706,7 +1985,7 @@ mod tests {
|
||||
encoded.truncate(encoded.len() - 1);
|
||||
|
||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
.read_appending(&mut out, SHARD)
|
||||
.await
|
||||
.expect_err("a truncated shard must not succeed");
|
||||
@@ -1732,7 +2011,7 @@ mod tests {
|
||||
encoded[last] ^= 0xff;
|
||||
|
||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
||||
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
||||
.read_appending(&mut out, SHARD)
|
||||
.await
|
||||
.expect_err("a corrupt shard must not verify");
|
||||
@@ -1844,7 +2123,7 @@ mod tests {
|
||||
"Cursor<Bytes> must be able to hand out a block, otherwise the fast path is dead code"
|
||||
);
|
||||
assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length");
|
||||
let mut streamed = Cursor::new(encoded.clone());
|
||||
let mut streamed = std::io::Cursor::new(encoded.clone());
|
||||
assert!(
|
||||
ShardSource::try_take_block(&mut streamed, 8).is_none(),
|
||||
"a non-Bytes source must stay on the streaming path"
|
||||
@@ -1872,7 +2151,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let mut via_stream: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
||||
BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
||||
.read_appending(&mut via_stream, SHARD)
|
||||
.await
|
||||
.expect("streaming read");
|
||||
|
||||
@@ -132,7 +132,6 @@ impl RebalanceStopPropagationRecord {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DiskStat {
|
||||
pub total_space: u64,
|
||||
|
||||
@@ -16,8 +16,16 @@ use serde::{Deserialize, Serialize};
|
||||
use std::{fmt::Display, io};
|
||||
use tracing::info;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "tier config wire version stamped by the parity constructors below (backlog#1823)"
|
||||
)]
|
||||
const C_TIER_CONFIG_VER: &str = "v1";
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "tier-name validation message reached only from the parity constructors below (backlog#1823)"
|
||||
)]
|
||||
const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
|
||||
const WASABI_US_EAST_ENDPOINT: &str = "https://s3.wasabisys.com";
|
||||
const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
|
||||
@@ -264,7 +272,6 @@ impl Clone for TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TierConfig {
|
||||
pub(crate) fn clone_with_credentials(&self) -> Self {
|
||||
Self {
|
||||
@@ -284,6 +291,7 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn endpoint(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(),
|
||||
@@ -303,6 +311,7 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn bucket(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(),
|
||||
@@ -322,6 +331,7 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn prefix(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(),
|
||||
@@ -341,6 +351,7 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn region(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(),
|
||||
@@ -457,7 +468,7 @@ impl TierWasabi {
|
||||
}
|
||||
|
||||
impl TierS3 {
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn create<F>(
|
||||
name: &str,
|
||||
access_key: &str,
|
||||
@@ -528,7 +539,7 @@ pub struct TierMinIO {
|
||||
}
|
||||
|
||||
impl TierMinIO {
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn create<F>(
|
||||
name: &str,
|
||||
endpoint: &str,
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TierConfigMgr {
|
||||
pub fn msg_size(&self) -> usize {
|
||||
100
|
||||
|
||||
@@ -4860,6 +4860,14 @@ impl SetDisks {
|
||||
/// is best-effort maintenance: individual delete failures are logged and
|
||||
/// skipped rather than propagated.
|
||||
pub(crate) async fn reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
||||
self.reclaim_orphan_data_dirs_inner(bucket, object, false).await
|
||||
}
|
||||
|
||||
pub(crate) async fn dry_run_reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
||||
self.reclaim_orphan_data_dirs_inner(bucket, object, true).await
|
||||
}
|
||||
|
||||
async fn reclaim_orphan_data_dirs_inner(&self, bucket: &str, object: &str, dry_run: bool) -> disk::error::Result<usize> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
// Phase 1 (read-only): build the referenced-data-dir union and record the
|
||||
@@ -4967,6 +4975,20 @@ impl SetDisks {
|
||||
continue;
|
||||
}
|
||||
let stray = format!("{object}/{dir}");
|
||||
if dry_run {
|
||||
removed += 1;
|
||||
debug!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
event = "heal_abandoned_parts",
|
||||
component = "ecstore",
|
||||
subsystem = "heal",
|
||||
state = "dry_run_matched",
|
||||
result = "matched",
|
||||
bucket, object, data_dir = %dir,
|
||||
"Heal abandoned parts dry-run matched orphaned data directory"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
match disk
|
||||
.delete(
|
||||
bucket,
|
||||
|
||||
@@ -6998,6 +6998,100 @@ mod tests {
|
||||
assert!(object_dir.join(STORAGE_FORMAT_FILE).exists(), "metadata must be preserved");
|
||||
}
|
||||
|
||||
async fn recv_abandoned_parts_trace(
|
||||
trace: &mut rustfs_common::trace_bus::TraceSubscription,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
state: &str,
|
||||
) -> rustfs_common::trace_bus::TraceEvent {
|
||||
for _ in 0..32 {
|
||||
let event = tokio::time::timeout(std::time::Duration::from_secs(1), trace.recv())
|
||||
.await
|
||||
.expect("abandoned-parts trace event should arrive")
|
||||
.expect("trace bus should stay open");
|
||||
if event.kind == rustfs_common::trace_bus::TraceKind::Heal
|
||||
&& event.func == rustfs_common::trace_bus::TraceFunc::HealCheckAbandonedParts
|
||||
&& event.bucket.as_deref() == Some(bucket)
|
||||
&& event.object.as_deref() == Some(object)
|
||||
&& trace_attr_string(&event, "state").as_deref() == Some(state)
|
||||
{
|
||||
return (*event).clone();
|
||||
}
|
||||
}
|
||||
|
||||
panic!("expected abandoned-parts trace state {state} for {bucket}/{object}");
|
||||
}
|
||||
|
||||
fn trace_attr_string(event: &rustfs_common::trace_bus::TraceEvent, key: &str) -> Option<String> {
|
||||
event.attrs.iter().find_map(|attr| {
|
||||
if attr.key != key {
|
||||
return None;
|
||||
}
|
||||
Some(match &attr.value {
|
||||
rustfs_common::trace_bus::TraceVal::Bool(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::U64(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::I64(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::Str(value) => value.to_string(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_abandoned_parts_dry_run_counts_without_deleting() {
|
||||
let mut trace = rustfs_common::trace_bus::subscribe_trace_events();
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let live = Uuid::new_v4();
|
||||
let orphan = Uuid::new_v4();
|
||||
|
||||
let object_dir = dir.path().join("bucket").join("obj");
|
||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
|
||||
fs::create_dir_all(object_dir.join(live.to_string()))
|
||||
.await
|
||||
.expect("live data dir should be created");
|
||||
fs::create_dir_all(object_dir.join(orphan.to_string()))
|
||||
.await
|
||||
.expect("orphan data dir should be created");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
set.check_abandoned_parts(
|
||||
"bucket",
|
||||
"obj",
|
||||
&HealOpts {
|
||||
dry_run: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("dry-run abandoned-parts check should succeed");
|
||||
let dry_run_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "dry_run_matched").await;
|
||||
assert_eq!(trace_attr_string(&dry_run_trace, "dry_run").as_deref(), Some("true"));
|
||||
assert_eq!(trace_attr_string(&dry_run_trace, "data_dirs").as_deref(), Some("1"));
|
||||
|
||||
assert!(object_dir.join(live.to_string()).exists(), "referenced data dir must be preserved");
|
||||
assert!(object_dir.join(orphan.to_string()).exists(), "dry-run must not remove orphaned data dir");
|
||||
|
||||
set.check_abandoned_parts(
|
||||
"bucket",
|
||||
"obj",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("abandoned-parts check should reclaim stale data dir");
|
||||
let reclaim_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "reclaimed").await;
|
||||
assert_eq!(trace_attr_string(&reclaim_trace, "dry_run").as_deref(), Some("false"));
|
||||
assert_eq!(trace_attr_string(&reclaim_trace, "data_dirs").as_deref(), Some("1"));
|
||||
|
||||
assert!(
|
||||
object_dir.join(live.to_string()).exists(),
|
||||
"referenced data dir must remain after reclaim"
|
||||
);
|
||||
assert!(!object_dir.join(orphan.to_string()).exists(), "orphaned data dir must be removed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reclaim_orphan_data_dirs_recovers_deferred_cleanup_after_restart() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
@@ -12233,11 +12327,18 @@ mod tests {
|
||||
.expect_err("unsupported copy_object_part should return a typed error");
|
||||
assert!(matches!(copy_part_err, StorageError::NotImplemented));
|
||||
|
||||
let abandoned_err = set_disks
|
||||
.check_abandoned_parts("bucket", "object", &HealOpts::default())
|
||||
set_disks
|
||||
.check_abandoned_parts(
|
||||
"bucket",
|
||||
"object",
|
||||
&HealOpts {
|
||||
dry_run: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("abandoned-parts check should stay in the upper reconciliation layer");
|
||||
assert!(matches!(abandoned_err, StorageError::NotImplemented));
|
||||
.expect("abandoned-parts check should be callable on empty disk sets");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -16,6 +16,7 @@ use super::super::*;
|
||||
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
||||
use crate::io_support::bitrot::object_mmap_read_enabled;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit};
|
||||
use tracing::trace;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
@@ -2057,11 +2058,61 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
Err(Error::DiskNotFound)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
|
||||
// Multipart orphan reconciliation is intentionally retained above the set layer
|
||||
// until there is a concrete caller and a stable lower-level contract to implement.
|
||||
Err(StorageError::NotImplemented)
|
||||
#[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))]
|
||||
async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
||||
let started_at = std::time::Instant::now();
|
||||
let _write_lock_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(
|
||||
ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let removed = if opts.dry_run {
|
||||
self.dry_run_reclaim_orphan_data_dirs(bucket, object).await?
|
||||
} else {
|
||||
self.reclaim_orphan_data_dirs(bucket, object).await?
|
||||
};
|
||||
let state = if opts.dry_run && removed > 0 {
|
||||
"dry_run_matched"
|
||||
} else if removed > 0 {
|
||||
"reclaimed"
|
||||
} else {
|
||||
"checked"
|
||||
};
|
||||
let data_dirs = u64::try_from(removed).unwrap_or(u64::MAX);
|
||||
|
||||
trace_emit(|| {
|
||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealCheckAbandonedParts)
|
||||
.with_bucket(bucket)
|
||||
.with_object(object)
|
||||
.with_duration(started_at.elapsed())
|
||||
.with_attr("state", state)
|
||||
.with_attr("dry_run", opts.dry_run)
|
||||
.with_attr("data_dirs", data_dirs)
|
||||
});
|
||||
|
||||
if removed > 0 {
|
||||
trace!(
|
||||
event = "heal_abandoned_parts",
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
state = if opts.dry_run { "dry_run_matched" } else { "reclaimed" },
|
||||
result = "ok",
|
||||
bucket,
|
||||
object,
|
||||
dry_run = opts.dry_run,
|
||||
data_dirs = removed,
|
||||
"Heal abandoned parts checked object data directories"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
//! per-version `SetDisks::heal_object`.
|
||||
|
||||
use super::super::*;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
@@ -39,12 +40,16 @@ const BACKGROUND_WALKDIR_STALL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
/// it must not gate healing logic — the delete-marker vs data path is chosen
|
||||
/// inside `ops/heal.rs` from the resolved latest metadata. `version_id` is
|
||||
/// normalized (nil/absent UUID => `None`).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealWalkVersion {
|
||||
/// object key
|
||||
pub name: String,
|
||||
/// normalized version id (`None` when the version is nil/absent)
|
||||
pub version_id: Option<String>,
|
||||
/// version modification time as Unix nanoseconds
|
||||
pub mod_time_unix_nanos: Option<i128>,
|
||||
/// object snapshot for lifecycle evaluation
|
||||
pub lifecycle_object_info: Option<ObjectInfo>,
|
||||
/// whether this version is a delete marker (observability only)
|
||||
pub is_delete_marker: bool,
|
||||
}
|
||||
@@ -63,6 +68,7 @@ struct HealWalkCollector {
|
||||
bucket: String,
|
||||
batch_objects: usize,
|
||||
version_budget: usize,
|
||||
include_lifecycle_object_info: bool,
|
||||
objects: Mutex<Vec<HealWalkObject>>,
|
||||
decode_error: Mutex<Option<DiskError>>,
|
||||
version_total: AtomicUsize,
|
||||
@@ -116,10 +122,25 @@ impl HealWalkCollector {
|
||||
|
||||
let mut versions = Vec::with_capacity(fiv.versions.len() + fiv.free_versions.len());
|
||||
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
||||
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
|
||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
||||
let mut lifecycle_fi = fi.clone();
|
||||
lifecycle_fi.version_id = version_uuid;
|
||||
Some(ObjectInfo::from_file_info(
|
||||
&lifecycle_fi,
|
||||
&self.bucket,
|
||||
&entry.name,
|
||||
version_uuid.is_some(),
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
versions.push(HealWalkVersion {
|
||||
name: entry.name.clone(),
|
||||
// Normalize: nil/absent version id => None.
|
||||
version_id: fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()),
|
||||
version_id: version_uuid.map(|u| u.to_string()),
|
||||
mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()),
|
||||
lifecycle_object_info,
|
||||
is_delete_marker: fi.deleted,
|
||||
});
|
||||
}
|
||||
@@ -173,11 +194,26 @@ impl HealWalkCollector {
|
||||
}
|
||||
};
|
||||
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
||||
let vid = fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string());
|
||||
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
|
||||
let vid = version_uuid.map(|u| u.to_string());
|
||||
if seen.insert(vid.clone()) {
|
||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
||||
let mut lifecycle_fi = fi.clone();
|
||||
lifecycle_fi.version_id = version_uuid;
|
||||
Some(ObjectInfo::from_file_info(
|
||||
&lifecycle_fi,
|
||||
&self.bucket,
|
||||
&entry.name,
|
||||
version_uuid.is_some(),
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
versions.push(HealWalkVersion {
|
||||
name: entry.name.clone(),
|
||||
version_id: vid,
|
||||
mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()),
|
||||
lifecycle_object_info,
|
||||
is_delete_marker: fi.deleted,
|
||||
});
|
||||
}
|
||||
@@ -255,6 +291,7 @@ impl SetDisks {
|
||||
forward_to: Option<&str>,
|
||||
batch_objects: usize,
|
||||
version_budget: usize,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> disk::error::Result<(Vec<HealWalkVersion>, Option<String>, bool)> {
|
||||
assert!(batch_objects >= 2, "heal_walk_versions_page requires batch_objects >= 2");
|
||||
|
||||
@@ -264,6 +301,7 @@ impl SetDisks {
|
||||
bucket: bucket.to_string(),
|
||||
batch_objects,
|
||||
version_budget: version_budget.max(1),
|
||||
include_lifecycle_object_info,
|
||||
objects: Mutex::new(Vec::new()),
|
||||
decode_error: Mutex::new(None),
|
||||
version_total: AtomicUsize::new(0),
|
||||
@@ -347,6 +385,7 @@ mod tests {
|
||||
bucket: "bucket".to_string(),
|
||||
batch_objects: 2,
|
||||
version_budget: 2,
|
||||
include_lifecycle_object_info: false,
|
||||
objects: Mutex::new(Vec::new()),
|
||||
decode_error: Mutex::new(None),
|
||||
version_total: AtomicUsize::new(0),
|
||||
@@ -388,6 +427,8 @@ mod tests {
|
||||
HealWalkVersion {
|
||||
name: name.to_string(),
|
||||
version_id: Some(id.to_string()),
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: dm,
|
||||
}
|
||||
}
|
||||
@@ -491,6 +532,7 @@ mod tests {
|
||||
bucket: "bucket".to_string(),
|
||||
batch_objects: 1000,
|
||||
version_budget: 10_000,
|
||||
include_lifecycle_object_info: false,
|
||||
objects: Mutex::new(Vec::new()),
|
||||
version_total: AtomicUsize::new(0),
|
||||
decode_error: Mutex::new(None),
|
||||
@@ -567,7 +609,7 @@ mod tests {
|
||||
.expect("corrupt test metadata should be written");
|
||||
|
||||
let error = set_disks
|
||||
.heal_walk_versions_page(bucket, "", None, 2, 2)
|
||||
.heal_walk_versions_page(bucket, "", None, 2, 2, false)
|
||||
.await
|
||||
.expect_err("semantic metadata corruption must fail the heal disk walk");
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ use tracing::trace;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
||||
const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts";
|
||||
const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
|
||||
const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
|
||||
|
||||
@@ -256,13 +257,40 @@ impl ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
||||
let _ = (bucket, object, opts);
|
||||
// Stale multipart reconciliation is already owned by the lifecycle-driven
|
||||
// background cleanup path in `bucket_lifecycle_ops.rs`. There is currently
|
||||
// no stable object-heal contract that should fan this request out through
|
||||
// pool/set storage layers, so keep the placeholder explicit at the ECStore
|
||||
// boundary instead of dispatching into lower layers.
|
||||
Err(StorageError::NotImplemented)
|
||||
let object = encode_dir_object(object);
|
||||
let pools = self.get_pools_for_heal_object(opts)?;
|
||||
|
||||
let mut futures = Vec::with_capacity(pools.len());
|
||||
for pool in pools.iter() {
|
||||
futures.push(pool.check_abandoned_parts(bucket, &object, opts));
|
||||
}
|
||||
|
||||
let mut first_error = None;
|
||||
for result in join_all(futures).await {
|
||||
if let Err(err) = result
|
||||
&& first_error.is_none()
|
||||
{
|
||||
first_error = Some(err);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = first_error {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
trace!(
|
||||
event = EVENT_HEAL_ABANDONED_PARTS,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
state = "completed",
|
||||
result = "ok",
|
||||
bucket,
|
||||
object,
|
||||
dry_run = opts.dry_run,
|
||||
"Heal abandoned parts completed"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ impl ECStore {
|
||||
forward_to: Option<&str>,
|
||||
batch_objects: usize,
|
||||
version_budget: usize,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealWalkVersion>, Option<String>, bool)> {
|
||||
if pool_idx >= self.pools.len() || set_idx >= self.pools[pool_idx].disk_set.len() {
|
||||
return Err(Error::other(format!(
|
||||
@@ -43,7 +44,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
self.pools[pool_idx].disk_set[set_idx]
|
||||
.heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget)
|
||||
.heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget, include_lifecycle_object_info)
|
||||
.await
|
||||
.map_err(Error::from)
|
||||
}
|
||||
|
||||
@@ -216,6 +216,16 @@ impl std::fmt::Debug for ECStore {
|
||||
/// These delegate to the process-global statics. No local state — the globals
|
||||
/// remain the single source of truth until the migration is complete.
|
||||
impl ECStore {
|
||||
/// Every erasure set across all pools, pool-major order.
|
||||
///
|
||||
/// Read-only queries that must consult each set's own copy of a
|
||||
/// per-bucket object (e.g. the scanner's `.usage-cache.bin`) iterate
|
||||
/// this instead of the hash-routed store path, which would always land
|
||||
/// on one set (rustfs/backlog#1872).
|
||||
pub fn all_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
|
||||
self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect()
|
||||
}
|
||||
|
||||
/// Get server configuration (delegates to global)
|
||||
pub fn get_server_config(&self) -> Option<Config> {
|
||||
runtime_sources::server_config()
|
||||
|
||||
@@ -767,6 +767,7 @@ mod tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> crate::Result<(Vec<crate::heal::storage::HealListItem>, Option<String>, bool)> {
|
||||
Ok((vec![], None, false))
|
||||
}
|
||||
|
||||
@@ -23,13 +23,14 @@ use crate::heal::{
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use futures::{StreamExt, stream::FuturesUnordered};
|
||||
use metrics::gauge;
|
||||
use metrics::{counter, gauge};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, UNIX_EPOCH};
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
@@ -47,6 +48,21 @@ enum HealObjectOutcome {
|
||||
Failed,
|
||||
}
|
||||
|
||||
fn result_object_size_u64(result: &HealResultItem) -> u64 {
|
||||
u64::try_from(result.object_size).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
const NEW_VERSION_SKIP_GRACE_SECS: u64 = 60;
|
||||
const NANOS_PER_SECOND: i128 = 1_000_000_000;
|
||||
|
||||
fn should_skip_new_version(mod_time_unix_nanos: Option<i128>, started_at_secs: u64) -> bool {
|
||||
let Some(mod_time_unix_nanos) = mod_time_unix_nanos else {
|
||||
return false;
|
||||
};
|
||||
let cutoff_secs = started_at_secs.saturating_add(NEW_VERSION_SKIP_GRACE_SECS);
|
||||
mod_time_unix_nanos > i128::from(cutoff_secs).saturating_mul(NANOS_PER_SECOND)
|
||||
}
|
||||
|
||||
struct PageConcurrencyGuard {
|
||||
in_flight: Arc<AtomicUsize>,
|
||||
set_label: String,
|
||||
@@ -492,6 +508,7 @@ impl ErasureSetHealer {
|
||||
&mut skipped_objects,
|
||||
resume_manager,
|
||||
checkpoint_manager,
|
||||
state.start_time,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -658,6 +675,7 @@ impl ErasureSetHealer {
|
||||
skipped_objects: &mut u64,
|
||||
resume_manager: &ResumeManager,
|
||||
checkpoint_manager: &CheckpointManager,
|
||||
started_at_secs: u64,
|
||||
) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
@@ -710,6 +728,7 @@ impl ErasureSetHealer {
|
||||
// The end-of-pass summary reports the full failed/skipped counts.
|
||||
let mut transient_skip_samples_logged = 0_u64;
|
||||
let mut failure_samples_logged = 0_u64;
|
||||
let mut bytes_processed = self.progress.read().await.bytes_processed;
|
||||
|
||||
// backlog#920: select the per-erasure-set DISK-WALK union enumerator when
|
||||
// the scan is Deep OR the request came from AutoHeal — these are the paths
|
||||
@@ -718,17 +737,25 @@ impl ErasureSetHealer {
|
||||
// which stays the default.
|
||||
let use_disk_walk =
|
||||
matches!(self.heal_opts.scan_mode, HealScanMode::Deep) || matches!(self.source, HealRequestSource::AutoHeal);
|
||||
let lifecycle_expiry_context = self.storage.load_heal_lifecycle_expiry_context(bucket).await?;
|
||||
let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
|
||||
|
||||
loop {
|
||||
self.verify_replacement_identity_fence("page scan").await?;
|
||||
// Get one page of object versions
|
||||
let (objects, next_token, is_truncated) = if use_disk_walk {
|
||||
self.storage
|
||||
.list_versions_for_heal_page_disk_walk(set_disk_id, bucket, "", continuation_token.as_deref())
|
||||
.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
"",
|
||||
continuation_token.as_deref(),
|
||||
include_lifecycle_object_info,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
self.storage
|
||||
.list_objects_for_heal_page(bucket, "", continuation_token.as_deref())
|
||||
.list_objects_for_heal_page(bucket, "", continuation_token.as_deref(), include_lifecycle_object_info)
|
||||
.await?
|
||||
};
|
||||
let page_is_empty = objects.is_empty();
|
||||
@@ -736,6 +763,7 @@ impl ErasureSetHealer {
|
||||
let page_resume_index = *current_object_index;
|
||||
let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
|
||||
let mut page_tasks = FuturesUnordered::new();
|
||||
let mut completed_in_page = 0usize;
|
||||
|
||||
// Capture the last version identity of this page for the anti-loop guard.
|
||||
let page_last = objects.last().map(|item| (item.name.clone(), item.version_id.clone()));
|
||||
@@ -751,6 +779,75 @@ impl ErasureSetHealer {
|
||||
continue;
|
||||
}
|
||||
|
||||
if should_skip_new_version(item.mod_time_unix_nanos, started_at_secs) {
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
*processed_objects = processed_objects.saturating_add(1);
|
||||
completed_in_page = completed_in_page.saturating_add(1);
|
||||
counter!("rustfs_heal_skipped_new_versions_total").increment(1);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.record_skipped_new_version();
|
||||
progress.set_current_object(Some(format!("skipped_new: {bucket}/{}", item.name)));
|
||||
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket,
|
||||
object = %item.name,
|
||||
version_id = ?item.version_id,
|
||||
state = "skipped_new_version",
|
||||
"Erasure set object version skipped because it was written after heal started"
|
||||
);
|
||||
if completed_in_page.is_multiple_of(100) {
|
||||
checkpoint_manager.update_position(bucket_index, page_resume_index).await?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(context) = lifecycle_expiry_context.as_ref()
|
||||
&& self
|
||||
.storage
|
||||
.enqueue_heal_lifecycle_expiry(
|
||||
context,
|
||||
bucket,
|
||||
&item.name,
|
||||
item.version_id.as_deref(),
|
||||
item.lifecycle_object_info.as_ref(),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
*processed_objects = processed_objects.saturating_add(1);
|
||||
completed_in_page = completed_in_page.saturating_add(1);
|
||||
counter!("rustfs_heal_skipped_ilm_expired_total").increment(1);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.record_skipped_ilm_expired();
|
||||
progress.set_current_object(Some(format!("skipped_ilm: {bucket}/{}", item.name)));
|
||||
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket,
|
||||
object = %item.name,
|
||||
version_id = ?item.version_id,
|
||||
state = "skipped_ilm_expired",
|
||||
"Erasure set object version skipped because lifecycle expiry was queued"
|
||||
);
|
||||
if completed_in_page.is_multiple_of(100) {
|
||||
checkpoint_manager.update_position(bucket_index, page_resume_index).await?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
resume_manager
|
||||
.set_current_item(Some(bucket.to_string()), Some(item.name.clone()))
|
||||
.await?;
|
||||
@@ -777,7 +874,7 @@ impl ErasureSetHealer {
|
||||
|
||||
let _permit = match permit {
|
||||
Ok(permit) => permit,
|
||||
Err(err) => return (dedup_key, object_name, version_id, Err(err)),
|
||||
Err(err) => return (dedup_key, object_name, version_id, (0, Err(err))),
|
||||
};
|
||||
|
||||
let _in_flight_guard = PageConcurrencyGuard::new(in_flight, set_label);
|
||||
@@ -788,7 +885,7 @@ impl ErasureSetHealer {
|
||||
// recorded as skipped-ok rather than failed. The delete-marker
|
||||
// vs data path is chosen internally in ops/heal.rs.
|
||||
let result = if cancel_token.is_cancelled() {
|
||||
Err(Error::TaskCancelled)
|
||||
(0, Err(Error::TaskCancelled))
|
||||
} else {
|
||||
match storage
|
||||
.heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts)
|
||||
@@ -797,8 +894,9 @@ impl ErasureSetHealer {
|
||||
Ok((result, None))
|
||||
if target_outcomes_complete(&result, &target_endpoints) =>
|
||||
{
|
||||
let object_size = result_object_size_u64(&result);
|
||||
if !replacement_commit_evidence_required {
|
||||
Ok(true)
|
||||
(object_size, Ok(true))
|
||||
} else {
|
||||
match storage
|
||||
.replacement_targets_have_version(
|
||||
@@ -810,27 +908,42 @@ impl ErasureSetHealer {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(true) => Ok(true),
|
||||
Ok(false) => Err(Error::transient_skip(format!(
|
||||
Ok(true) => (object_size, Ok(true)),
|
||||
Ok(false) => (object_size, Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because replacement target readback did not confirm the committed version"
|
||||
))),
|
||||
Err(err) => Err(Error::transient_skip(format!(
|
||||
)))),
|
||||
Err(err) => (object_size, Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because replacement target readback failed: {err}"
|
||||
))),
|
||||
)))),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((_result, None)) if !target_endpoints.is_empty() => Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed"
|
||||
))),
|
||||
Ok((_result, None)) => Ok(true),
|
||||
Ok((_, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => Ok(false),
|
||||
Ok((_, Some(err))) | Err(err) => match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent => Ok(false),
|
||||
HealObjectOutcome::Transient => Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} due to transient error: {err}"
|
||||
},
|
||||
Ok((result, None)) if !target_endpoints.is_empty() => (
|
||||
result_object_size_u64(&result),
|
||||
Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed"
|
||||
))),
|
||||
HealObjectOutcome::Failed => Err(err),
|
||||
),
|
||||
Ok((result, None)) => (result_object_size_u64(&result), Ok(true)),
|
||||
Ok((result, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => {
|
||||
(result_object_size_u64(&result), Ok(false))
|
||||
}
|
||||
Ok((result, Some(err))) => {
|
||||
let object_size = result_object_size_u64(&result);
|
||||
match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent => (object_size, Ok(false)),
|
||||
HealObjectOutcome::Transient => (object_size, Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} due to transient error: {err}"
|
||||
)))),
|
||||
HealObjectOutcome::Failed => (object_size, Err(err)),
|
||||
}
|
||||
}
|
||||
Err(err) => match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent => (0, Ok(false)),
|
||||
HealObjectOutcome::Transient => (0, Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} due to transient error: {err}"
|
||||
)))),
|
||||
HealObjectOutcome::Failed => (0, Err(err)),
|
||||
},
|
||||
}
|
||||
};
|
||||
@@ -839,11 +952,12 @@ impl ErasureSetHealer {
|
||||
});
|
||||
}
|
||||
|
||||
let mut completed_in_page = 0usize;
|
||||
while let Some((key, object, version_id, result)) = page_tasks.next().await {
|
||||
let (object_size, result) = result;
|
||||
match result {
|
||||
Ok(true) => {
|
||||
*successful_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
@@ -861,6 +975,7 @@ impl ErasureSetHealer {
|
||||
Ok(false) => {
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
*successful_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
@@ -877,6 +992,7 @@ impl ErasureSetHealer {
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(Error::TransientSkip { message }) => {
|
||||
*skipped_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
checkpoint_manager.add_skipped_object(key).await?;
|
||||
demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
@@ -893,6 +1009,7 @@ impl ErasureSetHealer {
|
||||
}
|
||||
Err(err) => {
|
||||
*failed_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
checkpoint_manager.add_failed_object(key).await?;
|
||||
demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
@@ -911,6 +1028,11 @@ impl ErasureSetHealer {
|
||||
|
||||
*processed_objects += 1;
|
||||
completed_in_page += 1;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
|
||||
}
|
||||
|
||||
if completed_in_page.is_multiple_of(100) {
|
||||
checkpoint_manager.update_position(bucket_index, page_resume_index).await?;
|
||||
@@ -964,7 +1086,9 @@ impl ErasureSetHealer {
|
||||
progress.objects_scanned = state.total_objects;
|
||||
progress.objects_healed = state.successful_objects;
|
||||
progress.objects_failed = state.failed_objects;
|
||||
progress.bytes_processed = 0; // set to 0 for now, can be extended later
|
||||
progress.bytes_processed = 0; // Resume state tracks object counts, not byte counters.
|
||||
progress.start_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.start_time));
|
||||
progress.last_update_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.last_update));
|
||||
progress.set_current_object(state.current_object.clone());
|
||||
}
|
||||
}
|
||||
@@ -1135,13 +1259,15 @@ mod resume_loop_tests {
|
||||
//! that emits programmable multi-version pages. These exercise the real loop
|
||||
//! logic (cursor seeding, per-version dedup, anti-loop guard, absence
|
||||
//! handling) — not merely a mock's own output.
|
||||
use super::{ErasureSetHealer, target_outcomes_complete};
|
||||
use super::{
|
||||
ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete,
|
||||
};
|
||||
use crate::heal::progress::HealProgress;
|
||||
use crate::heal::resume::{
|
||||
CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils,
|
||||
compose_key,
|
||||
};
|
||||
use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage::{DiskStatus, HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage_api::status::BucketInfo;
|
||||
use crate::heal::{
|
||||
BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk,
|
||||
@@ -1149,7 +1275,7 @@ mod resume_loop_tests {
|
||||
use crate::{Error, Result};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource};
|
||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tempfile::TempDir;
|
||||
@@ -1160,10 +1286,37 @@ mod resume_loop_tests {
|
||||
HealListItem {
|
||||
name: name.to_string(),
|
||||
version_id: version.map(str::to_string),
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: delete_marker,
|
||||
}
|
||||
}
|
||||
|
||||
fn item_with_mod_time(name: &str, version: Option<&str>, mod_time_secs: u64) -> HealListItem {
|
||||
HealListItem {
|
||||
name: name.to_string(),
|
||||
version_id: version.map(str::to_string),
|
||||
mod_time_unix_nanos: Some(i128::from(mod_time_secs).saturating_mul(NANOS_PER_SECOND)),
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_version_filter_respects_grace_boundary() {
|
||||
let started_at = 1_700_000_000;
|
||||
|
||||
assert!(!should_skip_new_version(None, started_at));
|
||||
assert!(!should_skip_new_version(
|
||||
Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS).saturating_mul(NANOS_PER_SECOND)),
|
||||
started_at,
|
||||
));
|
||||
assert!(should_skip_new_version(
|
||||
Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS + 1).saturating_mul(NANOS_PER_SECOND)),
|
||||
started_at,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn target_outcomes_require_each_requested_endpoint_once_and_ok() {
|
||||
let result = HealResultItem {
|
||||
@@ -1246,8 +1399,10 @@ mod resume_loop_tests {
|
||||
/// Target-specific physical readback evidence per `compose_key`; the
|
||||
/// fake models a healthy backend unless a test explicitly revokes it.
|
||||
replacement_commit_evidence: Mutex<HashMap<String, ReplacementCommitEvidence>>,
|
||||
lifecycle_expired: Mutex<HashSet<String>>,
|
||||
/// every heal_object call recorded as (name, version_id)
|
||||
heal_calls: Mutex<Vec<(String, Option<String>)>>,
|
||||
list_include_lifecycle_object_info: Mutex<Vec<bool>>,
|
||||
replacement_target_identity_sequences: Mutex<VecDeque<Vec<ReplacementTargetIdentity>>>,
|
||||
fail_listing: AtomicBool,
|
||||
}
|
||||
@@ -1274,9 +1429,15 @@ mod resume_loop_tests {
|
||||
.unwrap()
|
||||
.insert(compose_key(name, version), ReplacementCommitEvidence::Error(message.to_string()));
|
||||
}
|
||||
fn set_lifecycle_expired(&self, name: &str, version: Option<&str>) {
|
||||
self.lifecycle_expired.lock().unwrap().insert(compose_key(name, version));
|
||||
}
|
||||
fn calls(&self) -> Vec<(String, Option<String>)> {
|
||||
self.heal_calls.lock().unwrap().clone()
|
||||
}
|
||||
fn list_include_lifecycle_object_info_calls(&self) -> Vec<bool> {
|
||||
self.list_include_lifecycle_object_info.lock().unwrap().clone()
|
||||
}
|
||||
fn fail_listing(&self) {
|
||||
self.fail_listing.store(true, Ordering::SeqCst);
|
||||
}
|
||||
@@ -1330,6 +1491,23 @@ mod resume_loop_tests {
|
||||
async fn get_object_checksum(&self, _b: &str, _o: &str) -> Result<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
Ok((!self.lifecycle_expired.lock().unwrap().is_empty()).then(HealLifecycleExpiryContext::test))
|
||||
}
|
||||
async fn enqueue_heal_lifecycle_expiry(
|
||||
&self,
|
||||
_context: &HealLifecycleExpiryContext,
|
||||
_bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
_object_info: Option<&HealObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
Ok(self
|
||||
.lifecycle_expired
|
||||
.lock()
|
||||
.unwrap()
|
||||
.contains(&compose_key(object, version_id)))
|
||||
}
|
||||
async fn heal_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -1386,7 +1564,12 @@ mod resume_loop_tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.list_include_lifecycle_object_info
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(include_lifecycle_object_info);
|
||||
if self.fail_listing.load(Ordering::SeqCst) {
|
||||
return Err(Error::other("injected listing failure"));
|
||||
}
|
||||
@@ -1476,6 +1659,7 @@ mod resume_loop_tests {
|
||||
|
||||
/// Drive one bucket heal pass; returns (processed, successful, failed, skipped, result).
|
||||
async fn run(env: &Env) -> (u64, u64, u64, u64, Result<()>) {
|
||||
let state = env.resume.get_state().await;
|
||||
let mut current_object_index = 0usize;
|
||||
let mut processed = 0u64;
|
||||
let mut successful = 0u64;
|
||||
@@ -1494,6 +1678,7 @@ mod resume_loop_tests {
|
||||
&mut skipped,
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
state.start_time,
|
||||
)
|
||||
.await;
|
||||
(processed, successful, failed, skipped, result)
|
||||
@@ -1559,6 +1744,7 @@ mod resume_loop_tests {
|
||||
let mut successful = 0;
|
||||
let mut failed = 0;
|
||||
let mut skipped = 0;
|
||||
let started_at = env.resume.get_state().await.start_time;
|
||||
|
||||
let error = healer
|
||||
.heal_bucket_with_resume(
|
||||
@@ -1572,6 +1758,7 @@ mod resume_loop_tests {
|
||||
&mut skipped,
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
started_at,
|
||||
)
|
||||
.await
|
||||
.expect_err("a remounted target must not begin a new page scan");
|
||||
@@ -1641,6 +1828,109 @@ mod resume_loop_tests {
|
||||
assert_eq!(skipped, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_progress_accumulates_healed_object_bytes() {
|
||||
let env = make_env().await;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![item("first", Some("v1"), false), item("second", Some("v2"), false)],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
env.storage.set_result(
|
||||
"first",
|
||||
Some("v1"),
|
||||
HealResultItem {
|
||||
object_size: 1024,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
env.storage.set_result(
|
||||
"second",
|
||||
Some("v2"),
|
||||
HealResultItem {
|
||||
object_size: 2048,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let (processed, successful, failed, skipped, result) = run(&env).await;
|
||||
|
||||
result.expect("page heal should succeed");
|
||||
assert_eq!(processed, 2);
|
||||
assert_eq!(successful, 2);
|
||||
assert_eq!(failed, 0);
|
||||
assert_eq!(skipped, 0);
|
||||
let progress = env.healer.progress.read().await;
|
||||
assert_eq!(progress.objects_scanned, 2);
|
||||
assert_eq!(progress.objects_healed, 2);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
assert_eq!(progress.bytes_processed, 3072);
|
||||
assert!(matches!(progress.current_object.as_deref(), Some("b/first" | "b/second")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_skips_versions_written_after_heal_started() {
|
||||
let env = make_env().await;
|
||||
let started_at = env.resume.get_state().await.start_time;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![
|
||||
item_with_mod_time("old", Some("v1"), started_at + NEW_VERSION_SKIP_GRACE_SECS),
|
||||
item_with_mod_time("new", Some("v2"), started_at + NEW_VERSION_SKIP_GRACE_SECS + 1),
|
||||
],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
|
||||
let (processed, successful, failed, skipped, result) = run(&env).await;
|
||||
|
||||
result.expect("page heal should succeed");
|
||||
assert_eq!(processed, 2);
|
||||
assert_eq!(successful, 1);
|
||||
assert_eq!(failed, 0);
|
||||
assert_eq!(skipped, 0);
|
||||
assert_eq!(env.storage.calls(), vec![("old".to_string(), Some("v1".to_string()))]);
|
||||
let progress = env.healer.progress.read().await;
|
||||
assert_eq!(progress.skipped_new_versions, 1);
|
||||
assert_eq!(progress.objects_scanned, 2);
|
||||
assert_eq!(progress.objects_healed, 1);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_skips_versions_queued_for_lifecycle_expiry() {
|
||||
let env = make_env().await;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![item("expired", Some("v1"), false), item("kept", Some("v2"), false)],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
env.storage.set_lifecycle_expired("expired", Some("v1"));
|
||||
|
||||
let (processed, successful, failed, skipped, result) = run(&env).await;
|
||||
|
||||
result.expect("page heal should succeed");
|
||||
assert_eq!(processed, 2);
|
||||
assert_eq!(successful, 1);
|
||||
assert_eq!(failed, 0);
|
||||
assert_eq!(skipped, 0);
|
||||
assert_eq!(env.storage.calls(), vec![("kept".to_string(), Some("v2".to_string()))]);
|
||||
assert_eq!(env.storage.list_include_lifecycle_object_info_calls(), vec![true]);
|
||||
let progress = env.healer.progress.read().await;
|
||||
assert_eq!(progress.skipped_ilm_expired, 1);
|
||||
assert_eq!(progress.objects_scanned, 2);
|
||||
assert_eq!(progress.objects_healed, 1);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_listing_failure_does_not_mark_set_completed() {
|
||||
let env = make_env().await;
|
||||
|
||||
@@ -2385,8 +2385,27 @@ impl HealManager {
|
||||
snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned);
|
||||
snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed);
|
||||
snapshot.objects_failed = snapshot.objects_failed.saturating_add(progress.objects_failed);
|
||||
snapshot.skipped_new_versions = snapshot.skipped_new_versions.saturating_add(progress.skipped_new_versions);
|
||||
snapshot.skipped_ilm_expired = snapshot.skipped_ilm_expired.saturating_add(progress.skipped_ilm_expired);
|
||||
snapshot.objects_total_count = snapshot.objects_total_count.saturating_add(progress.objects_total_count);
|
||||
snapshot.objects_total_size = snapshot.objects_total_size.saturating_add(progress.objects_total_size);
|
||||
snapshot.bytes_processed = snapshot.bytes_processed.saturating_add(progress.bytes_processed);
|
||||
snapshot.start_time = match (snapshot.start_time, progress.start_time) {
|
||||
(Some(current), Some(next)) => Some(current.min(next)),
|
||||
(None, next) => next,
|
||||
(current, None) => current,
|
||||
};
|
||||
snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) {
|
||||
(Some(current), Some(next)) => Some(current.max(next)),
|
||||
(None, next) => next,
|
||||
(current, None) => current,
|
||||
};
|
||||
if progress.current_object.is_some() {
|
||||
snapshot.current_object = progress.current_object;
|
||||
}
|
||||
}
|
||||
snapshot.refresh_progress_percentage();
|
||||
snapshot.refresh_estimated_completion_time();
|
||||
Some(snapshot)
|
||||
}
|
||||
|
||||
@@ -3208,6 +3227,7 @@ impl HealManager {
|
||||
} else {
|
||||
completed_task.get_status().await
|
||||
};
|
||||
let completed_progress = completed_task.get_progress().await;
|
||||
let completed_status_entry = CompletedHealStatus {
|
||||
heal_type: completed_task.heal_type.clone(),
|
||||
status: completed_status.clone(),
|
||||
@@ -3223,6 +3243,7 @@ impl HealManager {
|
||||
match completed_status {
|
||||
HealTaskStatus::Completed => {
|
||||
stats.update_task_completion(true);
|
||||
stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed);
|
||||
}
|
||||
HealTaskStatus::Retrying { .. } => {}
|
||||
_ => {
|
||||
@@ -3749,6 +3770,7 @@ mod tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<crate::heal::storage::HealListItem>, Option<String>, bool)> {
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
@@ -5396,6 +5418,8 @@ mod tests {
|
||||
));
|
||||
{
|
||||
let mut progress = first.progress.write().await;
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(20));
|
||||
progress.set_total_baseline(12, 8192);
|
||||
progress.update_progress(7, 3, 1, 4096);
|
||||
}
|
||||
|
||||
@@ -5405,6 +5429,8 @@ mod tests {
|
||||
));
|
||||
{
|
||||
let mut progress = second.progress.write().await;
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
progress.set_total_baseline(8, 4096);
|
||||
progress.update_progress(11, 5, 2, 2048);
|
||||
}
|
||||
|
||||
@@ -5419,7 +5445,11 @@ mod tests {
|
||||
assert_eq!(progress.objects_scanned, 18);
|
||||
assert_eq!(progress.objects_healed, 8);
|
||||
assert_eq!(progress.objects_failed, 3);
|
||||
assert_eq!(progress.objects_total_count, 20);
|
||||
assert_eq!(progress.objects_total_size, 12288);
|
||||
assert_eq!(progress.bytes_processed, 6144);
|
||||
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
|
||||
assert!(progress.estimated_completion_time.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::SystemTime;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
@@ -24,6 +24,14 @@ pub struct HealProgress {
|
||||
pub objects_healed: u64,
|
||||
/// Objects failed
|
||||
pub objects_failed: u64,
|
||||
/// Versions skipped because they were written after this heal started
|
||||
pub skipped_new_versions: u64,
|
||||
/// Versions skipped because lifecycle already selected them for expiry
|
||||
pub skipped_ilm_expired: u64,
|
||||
/// Baseline object count from the latest complete usage snapshot
|
||||
pub objects_total_count: u64,
|
||||
/// Baseline object bytes from the latest complete usage snapshot
|
||||
pub objects_total_size: u64,
|
||||
/// Bytes processed
|
||||
pub bytes_processed: u64,
|
||||
/// Current object
|
||||
@@ -54,10 +62,56 @@ impl HealProgress {
|
||||
self.bytes_processed = bytes;
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
|
||||
// calculate progress percentage
|
||||
let total = scanned + healed + failed;
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
pub fn set_total_baseline(&mut self, objects_total_count: u64, objects_total_size: u64) {
|
||||
self.objects_total_count = objects_total_count;
|
||||
self.objects_total_size = objects_total_size;
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
pub fn record_skipped_new_version(&mut self) {
|
||||
self.skipped_new_versions = self.skipped_new_versions.saturating_add(1);
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
pub fn record_skipped_ilm_expired(&mut self) {
|
||||
self.skipped_ilm_expired = self.skipped_ilm_expired.saturating_add(1);
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
fn completed_for_baseline(&self) -> u64 {
|
||||
self.objects_healed
|
||||
.saturating_add(self.objects_failed)
|
||||
.saturating_add(self.skipped_new_versions)
|
||||
.saturating_add(self.skipped_ilm_expired)
|
||||
}
|
||||
|
||||
pub(crate) fn refresh_progress_percentage(&mut self) {
|
||||
if self.objects_total_size > 0 {
|
||||
self.progress_percentage = ((self.bytes_processed as f64 / self.objects_total_size as f64) * 100.0).min(100.0);
|
||||
return;
|
||||
}
|
||||
if self.objects_total_count > 0 {
|
||||
let completed = self.completed_for_baseline();
|
||||
self.progress_percentage = ((completed as f64 / self.objects_total_count as f64) * 100.0).min(100.0);
|
||||
return;
|
||||
}
|
||||
|
||||
let total = self
|
||||
.objects_scanned
|
||||
.saturating_add(self.objects_healed)
|
||||
.saturating_add(self.objects_failed);
|
||||
if total > 0 {
|
||||
self.progress_percentage = (healed as f64 / total as f64) * 100.0;
|
||||
self.progress_percentage = (self.objects_healed as f64 / total as f64) * 100.0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,9 +120,36 @@ impl HealProgress {
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
}
|
||||
|
||||
pub fn refresh_estimated_completion_time(&mut self) {
|
||||
let Some(start_time) = self.start_time else {
|
||||
self.estimated_completion_time = None;
|
||||
return;
|
||||
};
|
||||
if self.is_completed() || !(0.0..100.0).contains(&self.progress_percentage) || self.bytes_processed == 0 {
|
||||
self.estimated_completion_time = None;
|
||||
return;
|
||||
}
|
||||
|
||||
let elapsed = match SystemTime::now().duration_since(start_time) {
|
||||
Ok(elapsed) if !elapsed.is_zero() => elapsed,
|
||||
_ => {
|
||||
self.estimated_completion_time = None;
|
||||
return;
|
||||
}
|
||||
};
|
||||
let estimated_total_secs = elapsed.as_secs_f64() * 100.0 / self.progress_percentage;
|
||||
self.estimated_completion_time = start_time.checked_add(Duration::from_secs_f64(estimated_total_secs));
|
||||
}
|
||||
|
||||
pub fn is_completed(&self) -> bool {
|
||||
self.progress_percentage >= 100.0
|
||||
|| self.objects_scanned > 0 && self.objects_healed + self.objects_failed >= self.objects_scanned
|
||||
if self.progress_percentage >= 100.0 {
|
||||
return true;
|
||||
}
|
||||
if self.objects_total_count > 0 || self.objects_total_size > 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.objects_scanned > 0 && self.objects_healed.saturating_add(self.objects_failed) >= self.objects_scanned
|
||||
}
|
||||
|
||||
pub fn get_success_rate(&self) -> f64 {
|
||||
@@ -158,6 +239,10 @@ mod tests {
|
||||
assert_eq!(progress.objects_scanned, 0);
|
||||
assert_eq!(progress.objects_healed, 0);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
assert_eq!(progress.skipped_new_versions, 0);
|
||||
assert_eq!(progress.skipped_ilm_expired, 0);
|
||||
assert_eq!(progress.objects_total_count, 0);
|
||||
assert_eq!(progress.objects_total_size, 0);
|
||||
assert_eq!(progress.bytes_processed, 0);
|
||||
assert_eq!(progress.progress_percentage, 0.0);
|
||||
assert!(progress.start_time.is_some());
|
||||
@@ -181,6 +266,73 @@ mod tests {
|
||||
assert!(progress.last_update_time.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_estimates_completion_time_from_progress() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
|
||||
progress.update_progress(100, 25, 0, 4096);
|
||||
|
||||
let eta = progress
|
||||
.estimated_completion_time
|
||||
.expect("partial byte progress should estimate completion");
|
||||
assert!(eta > SystemTime::now());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_uses_byte_baseline_for_percentage() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.set_total_baseline(10, 8192);
|
||||
|
||||
progress.update_progress(100, 25, 0, 4096);
|
||||
|
||||
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_uses_object_baseline_when_bytes_unknown() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.set_total_baseline(10, 0);
|
||||
|
||||
progress.update_progress(100, 3, 2, 0);
|
||||
|
||||
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_counts_skipped_versions_for_object_baseline() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.set_total_baseline(10, 0);
|
||||
|
||||
progress.update_progress(100, 3, 2, 0);
|
||||
progress.record_skipped_new_version();
|
||||
|
||||
assert_eq!(progress.skipped_new_versions, 1);
|
||||
assert!((progress.progress_percentage - 60.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_does_not_estimate_completion_without_bytes() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
|
||||
progress.update_progress(100, 25, 0, 0);
|
||||
|
||||
assert!(progress.estimated_completion_time.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_with_baseline_is_not_completed_by_processed_count() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
progress.set_total_baseline(10, 8192);
|
||||
|
||||
progress.update_progress(1, 1, 0, 1024);
|
||||
|
||||
assert!(!progress.is_completed());
|
||||
assert!(progress.estimated_completion_time.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_update_progress_zero_total() {
|
||||
let mut progress = HealProgress::new();
|
||||
@@ -251,6 +403,8 @@ mod tests {
|
||||
assert_eq!(json["objectsScanned"], 10);
|
||||
assert_eq!(json["objectsHealed"], 8);
|
||||
assert_eq!(json["objectsFailed"], 2);
|
||||
assert_eq!(json["skippedNewVersions"], 0);
|
||||
assert_eq!(json["skippedIlmExpired"], 0);
|
||||
assert_eq!(json["bytesProcessed"], 1024);
|
||||
assert_eq!(json["currentObject"], "test-bucket/test-object");
|
||||
assert!(json["progressPercentage"].is_number());
|
||||
|
||||
@@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::storage_api::owner::{EcstoreHealLifecycleExpiryContext, ecstore_load_admin_data_usage_from_backend_cached};
|
||||
use super::storage_api::storage::{
|
||||
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
|
||||
ObjectOperations as _, StorageAdminApi,
|
||||
@@ -29,6 +30,37 @@ use super::storage_api::storage::{
|
||||
use super::{DiskStore, ECStore, Endpoint, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity};
|
||||
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct HealBucketUsageBaseline {
|
||||
pub objects_count: u64,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
pub struct HealLifecycleExpiryContext {
|
||||
inner: HealLifecycleExpiryContextInner,
|
||||
}
|
||||
|
||||
enum HealLifecycleExpiryContextInner {
|
||||
Ecstore(EcstoreHealLifecycleExpiryContext),
|
||||
#[allow(dead_code)]
|
||||
Test,
|
||||
}
|
||||
|
||||
impl HealLifecycleExpiryContext {
|
||||
fn ecstore(inner: EcstoreHealLifecycleExpiryContext) -> Self {
|
||||
Self {
|
||||
inner: HealLifecycleExpiryContextInner::Ecstore(inner),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test() -> Self {
|
||||
Self {
|
||||
inner: HealLifecycleExpiryContextInner::Test,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_STORAGE: &str = "storage";
|
||||
const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io";
|
||||
@@ -272,6 +304,10 @@ pub struct HealListItem {
|
||||
pub name: String,
|
||||
/// normalized version id (`None` when the version is nil/absent)
|
||||
pub version_id: Option<String>,
|
||||
/// version modification time as Unix nanoseconds
|
||||
pub mod_time_unix_nanos: Option<i128>,
|
||||
/// object snapshot for lifecycle evaluation
|
||||
pub lifecycle_object_info: Option<HealObjectInfo>,
|
||||
/// whether this version is a delete marker (observability only)
|
||||
pub is_delete_marker: bool,
|
||||
}
|
||||
@@ -329,6 +365,28 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
/// Get bucket info
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>>;
|
||||
|
||||
/// Aggregate usage-cache baselines for the requested buckets.
|
||||
async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Load per-bucket lifecycle expiry context for heal skips.
|
||||
async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Queue lifecycle expiry for a version that heal can skip.
|
||||
async fn enqueue_heal_lifecycle_expiry(
|
||||
&self,
|
||||
_context: &HealLifecycleExpiryContext,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_version_id: Option<&str>,
|
||||
_object_info: Option<&HealObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Fix bucket metadata
|
||||
async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>;
|
||||
|
||||
@@ -409,6 +467,7 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)>;
|
||||
|
||||
/// List versions for healing via a per-erasure-set DISK-WALK union enumerator
|
||||
@@ -427,8 +486,10 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.list_objects_for_heal_page(bucket, prefix, continuation_token).await
|
||||
self.list_objects_for_heal_page(bucket, prefix, continuation_token, include_lifecycle_object_info)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Get disk for resume functionality.
|
||||
@@ -1021,6 +1082,85 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
if buckets.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let info = match ecstore_load_admin_data_usage_from_backend_cached(self.ecstore.clone()).await {
|
||||
Ok(info) if info.is_complete_bucket_usage_snapshot() => info,
|
||||
Ok(_) | Err(_) => return Ok(None),
|
||||
};
|
||||
|
||||
let mut baseline = HealBucketUsageBaseline::default();
|
||||
for bucket in buckets {
|
||||
if let Some(usage) = info.buckets_usage.get(bucket) {
|
||||
baseline.objects_count = baseline.objects_count.saturating_add(usage.objects_count);
|
||||
baseline.bytes = baseline.bytes.saturating_add(usage.size);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Some(baseline))
|
||||
}
|
||||
|
||||
async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
match self.ecstore.load_heal_lifecycle_expiry_context(bucket).await {
|
||||
Ok(Some(context)) => Ok(Some(HealLifecycleExpiryContext::ecstore(context))),
|
||||
Ok(None) => Ok(None),
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "load_heal_lifecycle_expiry_context",
|
||||
bucket,
|
||||
result = "failed",
|
||||
error = %err,
|
||||
"Heal storage lifecycle expiry context load failed"
|
||||
);
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn enqueue_heal_lifecycle_expiry(
|
||||
&self,
|
||||
context: &HealLifecycleExpiryContext,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
object_info: Option<&HealObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
let context = match &context.inner {
|
||||
HealLifecycleExpiryContextInner::Ecstore(context) => context,
|
||||
HealLifecycleExpiryContextInner::Test => return Ok(false),
|
||||
};
|
||||
match self
|
||||
.ecstore
|
||||
.enqueue_heal_lifecycle_expiry(context, bucket, object, version_id, object_info)
|
||||
.await
|
||||
{
|
||||
Ok(queued) => Ok(queued),
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "enqueue_heal_lifecycle_expiry",
|
||||
bucket,
|
||||
object,
|
||||
version_id = ?version_id,
|
||||
result = "failed",
|
||||
error = %err,
|
||||
"Heal storage lifecycle expiry check failed"
|
||||
);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -1436,7 +1576,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
loop {
|
||||
let (page_objects, next_token, is_truncated) = self
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref())
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false)
|
||||
.await?;
|
||||
|
||||
all_objects.extend(page_objects);
|
||||
@@ -1471,6 +1611,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -1522,10 +1663,19 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
let page_objects: Vec<HealListItem> = list_info
|
||||
.objects
|
||||
.into_iter()
|
||||
.map(|obj| HealListItem {
|
||||
name: obj.name,
|
||||
version_id: obj.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()),
|
||||
is_delete_marker: obj.delete_marker,
|
||||
.map(|mut obj| {
|
||||
obj.version_id = obj.version_id.filter(|u| !u.is_nil());
|
||||
let version_id = obj.version_id.map(|u| u.to_string());
|
||||
let mod_time_unix_nanos = obj.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos());
|
||||
let is_delete_marker = obj.delete_marker;
|
||||
let lifecycle_object_info = include_lifecycle_object_info.then(|| obj.clone());
|
||||
HealListItem {
|
||||
name: obj.name,
|
||||
version_id,
|
||||
mod_time_unix_nanos,
|
||||
lifecycle_object_info,
|
||||
is_delete_marker,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let page_count = page_objects.len();
|
||||
@@ -1562,6 +1712,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
// Per-page bounds for the disk-walk union enumerator. Objects are atomic
|
||||
// (never split across pages), so version_budget only bounds how many
|
||||
@@ -1590,7 +1741,16 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
let (versions, next_forward, is_truncated) = self
|
||||
.ecstore
|
||||
.heal_walk_versions_page(pool_idx, set_idx, bucket, prefix, forward_to.as_deref(), BATCH_OBJECTS, VERSION_BUDGET)
|
||||
.heal_walk_versions_page(
|
||||
pool_idx,
|
||||
set_idx,
|
||||
bucket,
|
||||
prefix,
|
||||
forward_to.as_deref(),
|
||||
BATCH_OBJECTS,
|
||||
VERSION_BUDGET,
|
||||
include_lifecycle_object_info,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
@@ -1614,6 +1774,8 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map(|v| HealListItem {
|
||||
name: v.name,
|
||||
version_id: v.version_id,
|
||||
mod_time_unix_nanos: v.mod_time_unix_nanos,
|
||||
lifecycle_object_info: v.lifecycle_object_info,
|
||||
is_delete_marker: v.is_delete_marker,
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME;
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME,
|
||||
load_admin_data_usage_from_backend_cached as ecstore_load_admin_data_usage_from_backend_cached,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
|
||||
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
@@ -25,7 +28,9 @@ pub(crate) use rustfs_ecstore::api::disk::{
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{
|
||||
ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext,
|
||||
};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod owner {
|
||||
@@ -34,8 +39,8 @@ pub(crate) mod owner {
|
||||
pub(crate) use super::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError,
|
||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore,
|
||||
ecstore_local_disk_map_read,
|
||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreHealLifecycleExpiryContext,
|
||||
EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -19,11 +19,12 @@ use crate::heal::{
|
||||
resume::{
|
||||
CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match,
|
||||
},
|
||||
storage::{HealStorageAPI, next_heal_listing_token},
|
||||
storage::{HealBucketUsageBaseline, HealStorageAPI, next_heal_listing_token},
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use metrics::{counter, histogram};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -178,6 +179,17 @@ pub enum HealPriority {
|
||||
Urgent = 3,
|
||||
}
|
||||
|
||||
impl HealPriority {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Low => "low",
|
||||
Self::Normal => "normal",
|
||||
Self::High => "high",
|
||||
Self::Urgent => "urgent",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Heal options
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HealOptions {
|
||||
@@ -498,6 +510,61 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_trace_task_state(&self, state: &'static str, duration: Duration, error: Option<&Error>) {
|
||||
trace_emit(|| {
|
||||
let mut event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
|
||||
.with_duration(duration)
|
||||
.with_attr("task_id", self.id.as_str())
|
||||
.with_attr("heal_type", self.heal_type.log_kind())
|
||||
.with_attr("state", state)
|
||||
.with_attr("source", self.source.as_str())
|
||||
.with_attr("priority", self.priority.as_str())
|
||||
.with_attr("retry_attempts", u64::from(self.retry_attempts))
|
||||
.with_attr("dry_run", self.options.dry_run);
|
||||
|
||||
event = match &self.heal_type {
|
||||
HealType::Cluster => event,
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
let event = event.with_bucket(bucket.as_str()).with_object(object.as_str());
|
||||
match version_id {
|
||||
Some(version_id) => event.with_attr("version_id", version_id.as_str()),
|
||||
None => event,
|
||||
}
|
||||
}
|
||||
HealType::Bucket { bucket } => event.with_bucket(bucket.as_str()),
|
||||
HealType::Prefix { bucket, prefix } => event.with_bucket(bucket.as_str()).with_object(prefix.as_str()),
|
||||
HealType::ErasureSet { buckets, set_disk_id } => {
|
||||
let bucket_count = u64::try_from(buckets.len()).unwrap_or(u64::MAX);
|
||||
event
|
||||
.with_attr("set_disk_id", set_disk_id.as_str())
|
||||
.with_attr("bucket_count", bucket_count)
|
||||
}
|
||||
HealType::Metadata { bucket, object } => event.with_bucket(bucket.as_str()).with_object(object.as_str()),
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
let event = event.with_bucket(bucket.as_str()).with_object(object.as_str());
|
||||
match version_id {
|
||||
Some(version_id) => event.with_attr("version_id", version_id.as_str()),
|
||||
None => event,
|
||||
}
|
||||
}
|
||||
HealType::MRF { meta_path } => event.with_object(meta_path.as_str()),
|
||||
};
|
||||
|
||||
match error {
|
||||
Some(error) => event.with_attr("error", error.to_string()),
|
||||
None => event,
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn remaining_timeout(&self) -> Result<Option<Duration>> {
|
||||
if let Some(total) = self.options.timeout {
|
||||
let start_instant = { *self.task_start_instant.read().await };
|
||||
@@ -717,6 +784,7 @@ impl HealTask {
|
||||
queue_delay = ?queue_delay,
|
||||
"Heal task started"
|
||||
});
|
||||
self.emit_trace_task_state("started", Duration::ZERO, None);
|
||||
|
||||
let result = match &self.heal_type {
|
||||
HealType::Cluster => self.heal_cluster().await,
|
||||
@@ -805,6 +873,14 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
|
||||
let terminal_state = match &result {
|
||||
Ok(_) => "completed",
|
||||
Err(Error::TaskCancelled) => "cancelled",
|
||||
Err(Error::TaskTimeout) => "timed_out",
|
||||
Err(_) => "failed",
|
||||
};
|
||||
self.emit_trace_task_state(terminal_state, start_instant.elapsed(), result.as_ref().err());
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
@@ -1535,7 +1611,7 @@ impl HealTask {
|
||||
let (objects, next_token, is_truncated) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref()),
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -1697,6 +1773,23 @@ impl HealTask {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> {
|
||||
let baseline = match self
|
||||
.await_with_control(self.storage.erasure_set_usage_baseline(buckets))
|
||||
.await
|
||||
{
|
||||
Ok(Some(baseline)) => baseline,
|
||||
Ok(None) => return Ok(()),
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(_) => return Ok(()),
|
||||
};
|
||||
|
||||
let HealBucketUsageBaseline { objects_count, bytes } = baseline;
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_total_baseline(objects_count, bytes);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
@@ -2298,6 +2391,8 @@ impl HealTask {
|
||||
None
|
||||
};
|
||||
|
||||
self.apply_erasure_set_usage_baseline(&buckets).await?;
|
||||
|
||||
let healing_marker = format!("{set_disk_id}:{}", self.id);
|
||||
if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() {
|
||||
let state = resume_manager.get_state().await;
|
||||
@@ -2602,7 +2697,8 @@ impl HealTask {
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(4, 4, 0, 0);
|
||||
let bytes_processed = progress.bytes_processed;
|
||||
progress.update_progress(4, 4, 0, bytes_processed);
|
||||
}
|
||||
|
||||
match result {
|
||||
@@ -2658,6 +2754,7 @@ mod tests {
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, HealDiskExt as _, new_disk};
|
||||
use super::*;
|
||||
use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo};
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events};
|
||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::Mutex;
|
||||
@@ -3203,6 +3300,8 @@ mod tests {
|
||||
block_heal_object: Mutex<bool>,
|
||||
resume_disk: Mutex<Option<DiskStore>>,
|
||||
replacement_resume_disk: Mutex<Option<DiskStore>>,
|
||||
usage_baseline: Mutex<Option<HealBucketUsageBaseline>>,
|
||||
usage_baseline_error: Mutex<bool>,
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3265,11 +3364,69 @@ mod tests {
|
||||
assert_eq!(samples_logged, MAX_BUCKET_FAILURE_LOG_SAMPLES);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_emits_heal_trace_task_state() {
|
||||
let mut trace = subscribe_trace_events();
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
|
||||
task.execute().await.expect("mock object heal should complete");
|
||||
|
||||
let started = recv_trace_task_state(&mut trace, &task.id, "started").await;
|
||||
assert_eq!(started.kind, TraceKind::Heal);
|
||||
assert_eq!(started.func, TraceFunc::HealTask);
|
||||
assert_eq!(started.bucket.as_deref(), Some("bucket-a"));
|
||||
assert_eq!(started.object.as_deref(), Some("object-a"));
|
||||
assert_eq!(trace_attr_string(&started, "heal_type").as_deref(), Some("object"));
|
||||
assert_eq!(trace_attr_string(&started, "source").as_deref(), Some("internal"));
|
||||
assert_eq!(trace_attr_string(&started, "version_id").as_deref(), Some("version-a"));
|
||||
|
||||
let completed = recv_trace_task_state(&mut trace, &task.id, "completed").await;
|
||||
assert_eq!(completed.kind, TraceKind::Heal);
|
||||
assert_eq!(completed.func, TraceFunc::HealTask);
|
||||
assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed"));
|
||||
}
|
||||
|
||||
async fn recv_trace_task_state(trace: &mut TraceSubscription, task_id: &str, state: &str) -> TraceEvent {
|
||||
for _ in 0..32 {
|
||||
let event = tokio::time::timeout(Duration::from_secs(1), trace.recv())
|
||||
.await
|
||||
.expect("trace event should arrive")
|
||||
.expect("trace bus should stay open");
|
||||
if trace_attr_string(&event, "task_id").as_deref() == Some(task_id)
|
||||
&& trace_attr_string(&event, "state").as_deref() == Some(state)
|
||||
{
|
||||
return (*event).clone();
|
||||
}
|
||||
}
|
||||
|
||||
panic!("expected trace state {state} for task {task_id}");
|
||||
}
|
||||
|
||||
fn trace_attr_string(event: &TraceEvent, key: &str) -> Option<String> {
|
||||
event.attrs.iter().find_map(|attr| {
|
||||
if attr.key != key {
|
||||
return None;
|
||||
}
|
||||
Some(match &attr.value {
|
||||
TraceVal::Bool(value) => value.to_string(),
|
||||
TraceVal::U64(value) => value.to_string(),
|
||||
TraceVal::I64(value) => value.to_string(),
|
||||
TraceVal::Str(value) => value.to_string(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a latest, non-delete-marker heal list item with no version id.
|
||||
fn heal_item(name: &str) -> HealListItem {
|
||||
HealListItem {
|
||||
name: name.to_string(),
|
||||
version_id: None,
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: false,
|
||||
}
|
||||
}
|
||||
@@ -3357,6 +3514,13 @@ mod tests {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
if *self.usage_baseline_error.lock().unwrap() {
|
||||
return Err(Error::Other("usage baseline unavailable".to_string()));
|
||||
}
|
||||
Ok(*self.usage_baseline.lock().unwrap())
|
||||
}
|
||||
|
||||
async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
@@ -3540,6 +3704,7 @@ mod tests {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.listed_prefixes.lock().unwrap().push(prefix.to_string());
|
||||
if *self.truncate_without_token.lock().unwrap() {
|
||||
@@ -4654,6 +4819,73 @@ mod tests {
|
||||
assert!(storage.object_heal_opts.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_heal_applies_usage_baseline_to_progress() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let storage = Arc::new(MockStorage {
|
||||
resume_disk: Mutex::new(Some(disk)),
|
||||
usage_baseline: Mutex::new(Some(HealBucketUsageBaseline {
|
||||
objects_count: 10,
|
||||
bytes: 8,
|
||||
})),
|
||||
..Default::default()
|
||||
});
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket-a".to_string()],
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage);
|
||||
|
||||
task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string())
|
||||
.await
|
||||
.expect("erasure set heal should complete");
|
||||
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!(progress.objects_total_count, 10);
|
||||
assert_eq!(progress.objects_total_size, 8);
|
||||
assert_eq!(progress.bytes_processed, 2);
|
||||
assert!((progress.progress_percentage - 25.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_heal_ignores_usage_baseline_errors() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let storage = Arc::new(MockStorage {
|
||||
resume_disk: Mutex::new(Some(disk)),
|
||||
usage_baseline_error: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket-a".to_string()],
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage);
|
||||
|
||||
task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string())
|
||||
.await
|
||||
.expect("usage baseline failures should not fail erasure set heal");
|
||||
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!(progress.objects_total_count, 0);
|
||||
assert_eq!(progress.objects_total_size, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resumable_erasure_set_execution_is_cancelled_while_object_heal_is_pending() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
|
||||
@@ -445,6 +445,7 @@ mod tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool), Error> {
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
|
||||
@@ -176,7 +176,7 @@ async fn enumerate_all_versions(heal_storage: &Arc<ECStoreHealStorage>, bucket:
|
||||
let mut token: Option<String> = None;
|
||||
loop {
|
||||
let (page, next, truncated) = heal_storage
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref())
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref(), false)
|
||||
.await
|
||||
.expect("list_objects_for_heal_page failed");
|
||||
items.extend(page);
|
||||
|
||||
@@ -166,7 +166,7 @@ async fn enumerate_b5(heal_storage: &Arc<ECStoreHealStorage>, bucket: &str) -> V
|
||||
let mut token: Option<String> = None;
|
||||
loop {
|
||||
let (page, next, truncated) = heal_storage
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref())
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref(), false)
|
||||
.await
|
||||
.expect("b5 list page failed");
|
||||
items.extend(page);
|
||||
@@ -187,7 +187,7 @@ async fn enumerate_disk_walk(heal_storage: &Arc<ECStoreHealStorage>, bucket: &st
|
||||
let mut token: Option<String> = None;
|
||||
loop {
|
||||
let (page, next, truncated) = heal_storage
|
||||
.list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref())
|
||||
.list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref(), false)
|
||||
.await
|
||||
.expect("disk-walk list page failed");
|
||||
items.extend(page);
|
||||
@@ -418,7 +418,7 @@ mod serial_tests {
|
||||
let mut pages = 0usize;
|
||||
loop {
|
||||
let (versions, next_forward, truncated) = ecstore
|
||||
.heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000)
|
||||
.heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000, false)
|
||||
.await
|
||||
.expect("heal_walk_versions_page failed");
|
||||
pages += 1;
|
||||
|
||||
@@ -242,6 +242,7 @@ fn test_heal_task_status_atomic_update() {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> rustfs_heal::Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
Ok((vec![], None, false))
|
||||
}
|
||||
@@ -385,6 +386,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> rustfs_heal::Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
|
||||
@@ -293,6 +293,15 @@ enum StrictVaultAuthMethod {
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
Kubernetes {
|
||||
role: String,
|
||||
#[serde(default)]
|
||||
mount: Option<String>,
|
||||
#[serde(default)]
|
||||
jwt_path: Option<std::path::PathBuf>,
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
TokenFile {
|
||||
path: std::path::PathBuf,
|
||||
#[serde(default)]
|
||||
@@ -319,6 +328,17 @@ impl From<StrictVaultAuthMethod> for VaultAuthMethod {
|
||||
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
|
||||
refresh_safety_window_secs,
|
||||
},
|
||||
StrictVaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} => Self::Kubernetes {
|
||||
role,
|
||||
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()),
|
||||
jwt_path: jwt_path.unwrap_or_else(|| std::path::PathBuf::from(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)),
|
||||
refresh_safety_window_secs,
|
||||
},
|
||||
StrictVaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -499,6 +519,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
||||
auth_method_type: match &vault_config.auth_method {
|
||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||
},
|
||||
has_stored_credentials: true,
|
||||
@@ -513,6 +534,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
||||
auth_method_type: match &vault_config.auth_method {
|
||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||
},
|
||||
has_stored_credentials: true,
|
||||
@@ -901,6 +923,42 @@ mod tests {
|
||||
assert!(request.to_kms_config().validate().is_ok());
|
||||
}
|
||||
|
||||
/// The admin API reaches Kubernetes auth with the role alone; the mount and
|
||||
/// the projected token path fall back to the cluster defaults, so a Tenant
|
||||
/// manifest carries no credential and no cluster-specific paths.
|
||||
#[test]
|
||||
fn test_deserialize_vault_configure_request_accepts_kubernetes_auth() {
|
||||
let raw = serde_json::json!({
|
||||
"backend_type": "vault-transit",
|
||||
"address": "https://vault.example.com:8200",
|
||||
"mount_path": "rustfs",
|
||||
"auth_method": { "Kubernetes": { "role": "rustfs" } }
|
||||
});
|
||||
|
||||
let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("kubernetes auth should deserialize");
|
||||
let config = request.to_kms_config();
|
||||
config.validate().expect("kubernetes auth must validate");
|
||||
|
||||
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||
let VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} = &vault.auth_method
|
||||
else {
|
||||
panic!("expected Kubernetes auth, got {:?}", vault.auth_method);
|
||||
};
|
||||
assert_eq!(role, "rustfs");
|
||||
assert_eq!(mount, crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||
assert_eq!(jwt_path, std::path::Path::new(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||
|
||||
let unknown_field = serde_json::json!({
|
||||
"backend_type": "vault-transit",
|
||||
"address": "https://vault.example.com:8200",
|
||||
"auth_method": { "Kubernetes": { "role": "rustfs", "service_account": "rustfs" } }
|
||||
});
|
||||
serde_json::from_value::<ConfigureKmsRequest>(unknown_field)
|
||||
.expect_err("an unknown auth field must be rejected rather than silently dropped");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deserialize_aws_configure_request_accepts_type_aliases() {
|
||||
for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
|
||||
|
||||
@@ -550,6 +550,7 @@ impl VaultKmsClient {
|
||||
address: config.address.clone(),
|
||||
namespace: config.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||
};
|
||||
let source = token_source_for(&config.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
@@ -326,6 +326,97 @@ impl fmt::Debug for AppRoleLogin {
|
||||
}
|
||||
}
|
||||
|
||||
/// Token source for [`VaultAuthMethod::Kubernetes`]: exchanges the pod's
|
||||
/// projected ServiceAccount token for a lease-bound Vault token.
|
||||
///
|
||||
/// The JWT is re-read on every login because the kubelet rotates a projected
|
||||
/// token well inside the pod's lifetime; caching it would strand the source on
|
||||
/// an expired assertion once the current Vault token can no longer be renewed.
|
||||
///
|
||||
/// Unlike [`TokenFileSource`], the file mode is not checked: the kubelet owns
|
||||
/// the projected token and mounts it world-readable by default, so rejecting
|
||||
/// group/other bits would refuse every standard pod rather than catch a
|
||||
/// deployment error.
|
||||
pub(crate) struct KubernetesLogin {
|
||||
/// Unauthenticated client used only for the login exchange.
|
||||
login_client: VaultClient,
|
||||
mount: String,
|
||||
role: String,
|
||||
jwt_path: PathBuf,
|
||||
}
|
||||
|
||||
impl KubernetesLogin {
|
||||
pub(crate) fn new(settings: &VaultConnectionSettings, mount: String, role: String, jwt_path: PathBuf) -> Result<Self> {
|
||||
Ok(Self {
|
||||
login_client: settings.build_login_client()?,
|
||||
mount,
|
||||
role,
|
||||
jwt_path,
|
||||
})
|
||||
}
|
||||
|
||||
/// Read the ServiceAccount token for one login attempt.
|
||||
///
|
||||
/// Mirrors [`AppRoleLogin::resolve_secret_id`]: a read failure is fatal for
|
||||
/// the attempt but the refresh loop keeps retrying, so a token the kubelet
|
||||
/// has not projected yet heals the source without a restart.
|
||||
async fn resolve_jwt(&self) -> AttemptResult<SecretString> {
|
||||
let mut raw = tokio::fs::read_to_string(&self.jwt_path)
|
||||
.await
|
||||
.map_err(|error| AttemptError {
|
||||
class: ErrorClass::Fatal,
|
||||
error: KmsError::configuration_error(format!(
|
||||
"Failed to read Kubernetes ServiceAccount token {}: {error}",
|
||||
self.jwt_path.display()
|
||||
)),
|
||||
})?;
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
raw.zeroize();
|
||||
return Err(AttemptError {
|
||||
class: ErrorClass::Fatal,
|
||||
error: KmsError::configuration_error(format!(
|
||||
"Kubernetes ServiceAccount token {} is empty",
|
||||
self.jwt_path.display()
|
||||
)),
|
||||
});
|
||||
}
|
||||
let jwt = SecretString::new(trimmed.to_string());
|
||||
raw.zeroize();
|
||||
Ok(jwt)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TokenSource for KubernetesLogin {
|
||||
async fn acquire(&self) -> AttemptResult<TokenLease> {
|
||||
let jwt = self.resolve_jwt().await?;
|
||||
let auth = vaultrs::auth::kubernetes::login(&self.login_client, &self.mount, &self.role, jwt.expose())
|
||||
.await
|
||||
.map_err(|error| attempt_error("Kubernetes login", error))?;
|
||||
Ok(TokenLease::from_auth(auth))
|
||||
}
|
||||
|
||||
async fn renew(&self, client: &VaultClient) -> AttemptResult<TokenLease> {
|
||||
let auth = vaultrs::token::renew_self(client, None)
|
||||
.await
|
||||
.map_err(|error| attempt_error("token renewal", error))?;
|
||||
Ok(TokenLease::from_auth(auth))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for KubernetesLogin {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
// The login client embeds Vault client settings and must stay out of
|
||||
// Debug output; the role name is not a secret, and the JWT is never held.
|
||||
f.debug_struct("KubernetesLogin")
|
||||
.field("mount", &self.mount)
|
||||
.field("role", &self.role)
|
||||
.field("jwt_path", &self.jwt_path)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed
|
||||
/// token file (for example a Vault Agent auto-auth sink).
|
||||
///
|
||||
@@ -464,6 +555,9 @@ pub(crate) fn token_source_for(
|
||||
secret_id.clone(),
|
||||
secret_id_file.clone(),
|
||||
)?)),
|
||||
VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} => Ok(Box::new(KubernetesLogin::new(settings, mount.clone(), role.clone(), jwt_path.clone())?)),
|
||||
VaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -486,6 +580,9 @@ pub(crate) struct VaultConnectionSettings {
|
||||
pub(crate) namespace: Option<String>,
|
||||
/// Per-attempt HTTP timeout applied to the underlying reqwest client.
|
||||
pub(crate) attempt_timeout: Duration,
|
||||
/// Whether to accept an unverified Vault server certificate. Gated on
|
||||
/// `allow_insecure_dev_defaults` by `KmsConfig::validate`.
|
||||
pub(crate) skip_tls_verify: bool,
|
||||
}
|
||||
|
||||
impl VaultConnectionSettings {
|
||||
@@ -499,6 +596,11 @@ impl VaultConnectionSettings {
|
||||
// operation-level retry policy.
|
||||
settings_builder.timeout(Some(self.attempt_timeout));
|
||||
settings_builder.token(token);
|
||||
// Always set explicitly: left unset, vaultrs derives this from its own
|
||||
// VAULT_SKIP_VERIFY variable, so a stray value in the environment would
|
||||
// disable certificate verification behind the KMS configuration and its
|
||||
// insecure-defaults gate.
|
||||
settings_builder.verify(!self.skip_tls_verify);
|
||||
|
||||
if let Some(namespace) = &self.namespace {
|
||||
settings_builder.namespace(Some(namespace.clone()));
|
||||
@@ -551,6 +653,10 @@ impl VaultCredentialPolicy {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
}
|
||||
| VaultAuthMethod::Kubernetes {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
}
|
||||
| VaultAuthMethod::TokenFile {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
@@ -584,15 +690,25 @@ pub(crate) struct VaultClientHandle {
|
||||
|
||||
impl VaultClientHandle {
|
||||
/// Absolute expiry of this generation's token.
|
||||
///
|
||||
/// `lease.ttl` is built from the `lease_duration` the Vault server sent, so
|
||||
/// a value too large to add to `issued_at` would panic on the bare `+`. A
|
||||
/// TTL that cannot be represented is indistinguishable from no expiry, so it
|
||||
/// collapses to `None` — the same answer already given for the zero-lease
|
||||
/// tokens Vault issues, which keeps the token in use and still fully
|
||||
/// validated by Vault on every call.
|
||||
fn expires_at(&self) -> Option<Instant> {
|
||||
self.lease.map(|lease| self.issued_at + lease.ttl)
|
||||
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl))
|
||||
}
|
||||
|
||||
/// When the renewal task should refresh this generation: half the TTL,
|
||||
/// leaving the second half as budget for retries before the fail-closed
|
||||
/// window is reached.
|
||||
///
|
||||
/// Unrepresentable TTLs collapse to `None` as in [`Self::expires_at`],
|
||||
/// leaving a token that never expires with nothing to renew.
|
||||
fn renew_at(&self) -> Option<Instant> {
|
||||
self.lease.map(|lease| self.issued_at + lease.ttl / 2)
|
||||
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl / 2))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -662,7 +778,7 @@ impl VaultCredentialProvider {
|
||||
let handle = self.current.load_full();
|
||||
if let Some(expires_at) = handle.expires_at() {
|
||||
let now = Instant::now();
|
||||
if now + self.policy.safety_window >= expires_at {
|
||||
if self.inside_safety_window(now, expires_at) {
|
||||
return Err(KmsError::credentials_unavailable(format!(
|
||||
"Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it",
|
||||
handle.generation, self.policy.safety_window
|
||||
@@ -672,6 +788,18 @@ impl VaultCredentialProvider {
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
/// Whether the token expiring at `expires_at` is close enough to refuse.
|
||||
///
|
||||
/// `safety_window` reaches here from persisted configuration, so it is not
|
||||
/// guaranteed to have passed this version's validation: a window too large
|
||||
/// to add to the current instant would panic on the bare `+`. Such a window
|
||||
/// means every token is always inside it, so saturating to "refuse" is both
|
||||
/// the fail-closed answer and the one the arithmetic was reaching for.
|
||||
fn inside_safety_window(&self, now: Instant, expires_at: Instant) -> bool {
|
||||
now.checked_add(self.policy.safety_window)
|
||||
.is_none_or(|deadline| deadline >= expires_at)
|
||||
}
|
||||
|
||||
/// Publish the credential gauges for the generation currently installed.
|
||||
///
|
||||
/// The fail-closed gauge re-evaluates the very gate
|
||||
@@ -683,7 +811,7 @@ impl VaultCredentialProvider {
|
||||
let fail_closed = match handle.expires_at() {
|
||||
Some(expires_at) => {
|
||||
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
|
||||
now + self.policy.safety_window >= expires_at
|
||||
self.inside_safety_window(now, expires_at)
|
||||
}
|
||||
// A generation without an expiry has no remaining TTL to report
|
||||
// and can never lapse, so it can never fail closed either.
|
||||
@@ -860,7 +988,7 @@ impl Drop for CredentialTaskHandle {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::REDACTED_SECRET;
|
||||
use crate::config::{DEFAULT_VAULT_KUBERNETES_MOUNT, REDACTED_SECRET};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
|
||||
const TEST_TOKEN: &str = "vault-token-debug-leak-canary";
|
||||
@@ -871,6 +999,7 @@ mod tests {
|
||||
address: "http://127.0.0.1:8200".to_string(),
|
||||
namespace: Some("team-namespace".to_string()),
|
||||
attempt_timeout: Duration::from_secs(30),
|
||||
skip_tls_verify: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1057,6 +1186,143 @@ mod tests {
|
||||
assert!(format!("{source:?}").contains("AppRoleLogin"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_auth_method_maps_to_login_source() {
|
||||
let settings = test_settings();
|
||||
let source = token_source_for(&VaultAuthMethod::kubernetes("rustfs".to_string()), &settings)
|
||||
.expect("kubernetes auth must map to a login source");
|
||||
|
||||
assert!(format!("{source:?}").contains("KubernetesLogin"));
|
||||
}
|
||||
|
||||
/// `refresh_safety_window_secs` is operator-supplied and reaches the request
|
||||
/// path from persisted configuration, so the fail-closed comparison must
|
||||
/// survive a window too large to add to the current instant. Before the
|
||||
/// checked arithmetic this panicked with "overflow when adding duration to
|
||||
/// instant" on the first request after a lease-bearing login.
|
||||
#[tokio::test]
|
||||
async fn test_current_refuses_rather_than_panics_on_an_unrepresentable_safety_window() {
|
||||
let (provider, _state) = scripted_provider(
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
test_policy(Duration::from_secs(u64::MAX), Duration::from_secs(5)),
|
||||
)
|
||||
.await;
|
||||
|
||||
let error = provider
|
||||
.current()
|
||||
.expect_err("a window wider than any lease must refuse the token");
|
||||
assert!(
|
||||
matches!(error, KmsError::CredentialsUnavailable { .. }),
|
||||
"expected CredentialsUnavailable, got {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// `lease_duration` is a bare u64 straight off the Vault response and forms
|
||||
/// the other side of the same comparison, so an absurd one must not panic
|
||||
/// either. It is indistinguishable from a non-expiring token, which is how
|
||||
/// the zero-lease case already behaves.
|
||||
#[tokio::test]
|
||||
async fn test_an_unrepresentable_lease_is_treated_as_non_expiring() {
|
||||
let (provider, _state) = scripted_provider(
|
||||
Duration::from_secs(u64::MAX),
|
||||
true,
|
||||
test_policy(Duration::from_secs(30), Duration::from_secs(5)),
|
||||
)
|
||||
.await;
|
||||
|
||||
provider
|
||||
.current()
|
||||
.expect("a token whose expiry cannot be represented must stay usable");
|
||||
}
|
||||
|
||||
/// The configured flag has to reach the HTTP client, not just the config
|
||||
/// struct: every generation (authenticated and login) builds its own client,
|
||||
/// and a Vault with a self-signed certificate fails the handshake unless
|
||||
/// each one carries the setting.
|
||||
#[test]
|
||||
fn test_skip_tls_verify_reaches_every_vault_client_generation() {
|
||||
for skip_tls_verify in [false, true] {
|
||||
let settings = VaultConnectionSettings {
|
||||
address: "https://vault.example.com:8200".to_string(),
|
||||
namespace: None,
|
||||
attempt_timeout: Duration::from_secs(30),
|
||||
skip_tls_verify,
|
||||
};
|
||||
|
||||
let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build");
|
||||
assert_eq!(authenticated.settings.verify, !skip_tls_verify);
|
||||
|
||||
let login = settings.build_login_client().expect("login client must build");
|
||||
assert_eq!(login.settings.verify, !skip_tls_verify);
|
||||
}
|
||||
}
|
||||
|
||||
/// vaultrs derives `verify` from its own VAULT_SKIP_VERIFY variable when the
|
||||
/// builder leaves it unset, which would disable certificate verification
|
||||
/// without passing the KMS insecure-defaults gate.
|
||||
#[test]
|
||||
fn test_vaultrs_skip_verify_env_cannot_override_the_configured_setting() {
|
||||
temp_env::with_var("VAULT_SKIP_VERIFY", Some("true"), || {
|
||||
let client = test_settings().build_client(TEST_TOKEN).expect("client must build");
|
||||
assert!(
|
||||
client.settings.verify,
|
||||
"a stray VAULT_SKIP_VERIFY must not disable verification behind the KMS configuration"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// The projected token is read fresh per login attempt and trimmed, so a
|
||||
/// kubelet rotation is picked up without a restart and a trailing newline
|
||||
/// does not corrupt the assertion sent to Vault.
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_login_rereads_and_trims_the_service_account_token() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("token");
|
||||
tokio::fs::write(&path, " first-jwt\n").await.expect("write token");
|
||||
|
||||
let login = KubernetesLogin::new(
|
||||
&test_settings(),
|
||||
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||
"rustfs".to_string(),
|
||||
path.clone(),
|
||||
)
|
||||
.expect("login source must build");
|
||||
|
||||
assert_eq!(login.resolve_jwt().await.expect("first read").expose(), "first-jwt");
|
||||
|
||||
tokio::fs::write(&path, "rotated-jwt").await.expect("rotate token");
|
||||
assert_eq!(
|
||||
login.resolve_jwt().await.expect("second read").expose(),
|
||||
"rotated-jwt",
|
||||
"a rotated projected token must be picked up without a restart"
|
||||
);
|
||||
}
|
||||
|
||||
/// The ServiceAccount token is re-read per attempt, so an unreadable or
|
||||
/// empty one fails that attempt without reaching Vault; the refresh loop
|
||||
/// keeps retrying, which is what lets a late projection heal the source.
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_login_rejects_an_unusable_service_account_token() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let missing = dir.path().join("absent-token");
|
||||
let empty = dir.path().join("empty-token");
|
||||
tokio::fs::write(&empty, " \n").await.expect("write empty token");
|
||||
|
||||
for (path, expected) in [(missing, "Failed to read"), (empty, "is empty")] {
|
||||
let login =
|
||||
KubernetesLogin::new(&test_settings(), DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), "rustfs".to_string(), path)
|
||||
.expect("login source must build");
|
||||
|
||||
let error = login
|
||||
.acquire()
|
||||
.await
|
||||
.expect_err("an unusable ServiceAccount token must fail the attempt");
|
||||
assert!(matches!(error.class, ErrorClass::Fatal));
|
||||
assert!(error.error.to_string().contains(expected), "got {}", error.error);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_renewal_task_renews_at_half_ttl() {
|
||||
let (provider, state) = scripted_provider(
|
||||
|
||||
@@ -415,6 +415,7 @@ impl VaultTransitKmsClient {
|
||||
address: config.address.clone(),
|
||||
namespace: config.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||
};
|
||||
let source = token_source_for(&config.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
@@ -450,6 +450,10 @@ impl VaultRestoreClient {
|
||||
address: target.address.clone(),
|
||||
namespace: target.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
// A restore target carries no TLS settings, so certificates are
|
||||
// always verified: recovery is the last path that should accept an
|
||||
// unauthenticated Vault.
|
||||
skip_tls_verify: false,
|
||||
};
|
||||
let source = token_source_for(&target.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
+295
-54
@@ -25,6 +25,10 @@ use url::Url;
|
||||
|
||||
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
|
||||
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
|
||||
pub const ENV_KMS_VAULT_ADDRESS: &str = "RUSTFS_KMS_VAULT_ADDRESS";
|
||||
pub const ENV_KMS_VAULT_TOKEN: &str = "RUSTFS_KMS_VAULT_TOKEN";
|
||||
pub const ENV_KMS_VAULT_NAMESPACE: &str = "RUSTFS_KMS_VAULT_NAMESPACE";
|
||||
pub const ENV_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH";
|
||||
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
|
||||
@@ -35,6 +39,9 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
|
||||
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
|
||||
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_ROLE: &str = "RUSTFS_KMS_VAULT_KUBERNETES_ROLE";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_MOUNT: &str = "RUSTFS_KMS_VAULT_KUBERNETES_MOUNT";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_JWT_PATH: &str = "RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH";
|
||||
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
|
||||
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
|
||||
/// Age in whole seconds beyond which a key is reported as due for rotation;
|
||||
@@ -45,6 +52,9 @@ pub const ENV_KMS_ROTATION_MAX_WRAPS: &str = "RUSTFS_KMS_ROTATION_MAX_WRAPS";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
|
||||
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
|
||||
pub const DEFAULT_VAULT_KUBERNETES_MOUNT: &str = "kubernetes";
|
||||
/// Where the kubelet projects a pod's ServiceAccount token by default.
|
||||
pub const DEFAULT_VAULT_KUBERNETES_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token";
|
||||
|
||||
/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior.
|
||||
///
|
||||
@@ -84,6 +94,14 @@ fn default_vault_approle_mount() -> String {
|
||||
DEFAULT_VAULT_APPROLE_MOUNT.to_string()
|
||||
}
|
||||
|
||||
fn default_vault_kubernetes_mount() -> String {
|
||||
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()
|
||||
}
|
||||
|
||||
fn default_vault_kubernetes_jwt_path() -> PathBuf {
|
||||
PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH)
|
||||
}
|
||||
|
||||
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
|
||||
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
|
||||
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
|
||||
@@ -490,6 +508,23 @@ pub enum VaultAuthMethod {
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
/// Kubernetes authentication: the pod's ServiceAccount token is exchanged
|
||||
/// for a lease-bound Vault token that is renewed in the background.
|
||||
Kubernetes {
|
||||
/// Vault role bound to this ServiceAccount.
|
||||
role: String,
|
||||
/// Kubernetes auth engine mount path.
|
||||
#[serde(default = "default_vault_kubernetes_mount")]
|
||||
mount: String,
|
||||
/// Projected ServiceAccount token to present. Re-read on every login so
|
||||
/// a token the kubelet rotates is picked up without a restart.
|
||||
#[serde(default = "default_vault_kubernetes_jwt_path")]
|
||||
jwt_path: PathBuf,
|
||||
/// Fail-closed margin in seconds, as on `AppRole`. Defaults to the
|
||||
/// per-attempt timeout.
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
/// Agent-managed token file (for example a Vault Agent auto-auth sink):
|
||||
/// the token is read from `path` and re-read periodically so a token
|
||||
/// rotated by the agent is picked up without a restart.
|
||||
@@ -520,6 +555,16 @@ impl VaultAuthMethod {
|
||||
}
|
||||
}
|
||||
|
||||
/// Kubernetes authentication with the default mount and projected token path.
|
||||
pub fn kubernetes(role: String) -> Self {
|
||||
Self::Kubernetes {
|
||||
role,
|
||||
mount: default_vault_kubernetes_mount(),
|
||||
jwt_path: default_vault_kubernetes_jwt_path(),
|
||||
refresh_safety_window_secs: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Agent-managed token file with the default poll interval.
|
||||
pub fn token_file(path: PathBuf) -> Self {
|
||||
Self::TokenFile {
|
||||
@@ -548,6 +593,20 @@ impl fmt::Debug for VaultAuthMethod {
|
||||
.field("mount", mount)
|
||||
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||
.finish(),
|
||||
// No redaction: the role and mount name a Vault binding, and the
|
||||
// ServiceAccount token itself is never held on this type.
|
||||
Self::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} => f
|
||||
.debug_struct("Kubernetes")
|
||||
.field("role", role)
|
||||
.field("mount", mount)
|
||||
.field("jwt_path", jwt_path)
|
||||
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||
.finish(),
|
||||
Self::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -1028,50 +1087,12 @@ impl KmsConfig {
|
||||
});
|
||||
}
|
||||
KmsBackend::VaultKv2 => {
|
||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
||||
let auth_method = vault_auth_method_from_env()?;
|
||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
||||
|
||||
let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") {
|
||||
Some(path) => {
|
||||
tracing::warn!(
|
||||
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
||||
);
|
||||
path
|
||||
}
|
||||
None => default_vault_kv2_mount_path(),
|
||||
};
|
||||
|
||||
config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig {
|
||||
address,
|
||||
auth_method,
|
||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
||||
mount_path,
|
||||
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
||||
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
||||
tls: vault_tls_config(skip_tls_verify),
|
||||
}));
|
||||
config.backend_config =
|
||||
BackendConfig::VaultKv2(Box::new(vault_kv2_config_from_env(VaultCliOverrides::default())?));
|
||||
}
|
||||
KmsBackend::VaultTransit => {
|
||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
||||
let auth_method = vault_auth_method_from_env()?;
|
||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
||||
|
||||
config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
||||
address,
|
||||
auth_method,
|
||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
||||
mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"),
|
||||
metadata_kv_mount: get_env_str(
|
||||
ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
||||
DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
||||
),
|
||||
metadata_key_prefix: get_env_str(
|
||||
ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX,
|
||||
DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX,
|
||||
),
|
||||
tls: vault_tls_config(skip_tls_verify),
|
||||
}));
|
||||
config.backend_config =
|
||||
BackendConfig::VaultTransit(Box::new(vault_transit_config_from_env(VaultCliOverrides::default())?));
|
||||
}
|
||||
KmsBackend::Static => {
|
||||
// Read from file first, then fall back to direct env var
|
||||
@@ -1202,6 +1223,78 @@ fn is_under_temp_dir(path: &Path) -> bool {
|
||||
path.starts_with(std::env::temp_dir())
|
||||
}
|
||||
|
||||
/// Command-line values that take precedence over the matching environment
|
||||
/// variables when assembling a Vault backend configuration.
|
||||
///
|
||||
/// Every field has a `RUSTFS_KMS_VAULT_*` equivalent that the CLI layer already
|
||||
/// reads, so these are only set when the operator passed an explicit flag.
|
||||
///
|
||||
/// Deliberately not `Debug`: `token` holds the raw Vault token, and the
|
||||
/// redacting `Debug` impls elsewhere in this module exist because a derived one
|
||||
/// would print it. Denying the derive makes a future `{overrides:?}` a compile
|
||||
/// error instead of a leak.
|
||||
#[derive(Default, Clone, Copy)]
|
||||
pub struct VaultCliOverrides<'a> {
|
||||
pub address: Option<&'a str>,
|
||||
pub token: Option<&'a str>,
|
||||
pub mount_path: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Assemble the Vault KV2 backend configuration from the environment.
|
||||
///
|
||||
/// Shared by [`KmsConfig::from_env`] and the server's command-line startup path
|
||||
/// so both resolve the same auth method, namespace, TLS and mount settings.
|
||||
pub fn vault_kv2_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultConfig> {
|
||||
let mount_path = match overrides
|
||||
.mount_path
|
||||
.map(str::to_string)
|
||||
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_MOUNT_PATH))
|
||||
{
|
||||
Some(path) => {
|
||||
tracing::warn!(
|
||||
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
||||
);
|
||||
path
|
||||
}
|
||||
None => default_vault_kv2_mount_path(),
|
||||
};
|
||||
|
||||
Ok(VaultConfig {
|
||||
address: vault_address_from_env(overrides.address),
|
||||
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||
mount_path,
|
||||
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
||||
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
||||
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Assemble the Vault Transit backend configuration from the environment.
|
||||
///
|
||||
/// Companion to [`vault_kv2_config_from_env`]; see there for why both entry
|
||||
/// points share it.
|
||||
pub fn vault_transit_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultTransitConfig> {
|
||||
Ok(VaultTransitConfig {
|
||||
address: vault_address_from_env(overrides.address),
|
||||
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||
mount_path: overrides
|
||||
.mount_path
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_MOUNT_PATH, "transit")),
|
||||
metadata_kv_mount: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT),
|
||||
metadata_key_prefix: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX),
|
||||
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||
})
|
||||
}
|
||||
|
||||
fn vault_address_from_env(override_value: Option<&str>) -> String {
|
||||
override_value
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_ADDRESS, "http://localhost:8200"))
|
||||
}
|
||||
|
||||
/// Resolve the Vault auth method from environment variables.
|
||||
///
|
||||
/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication;
|
||||
@@ -1209,27 +1302,59 @@ fn is_under_temp_dir(path: &Path) -> bool {
|
||||
/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE`
|
||||
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
|
||||
/// file taking precedence. Without a role id the legacy token flow applies.
|
||||
fn vault_auth_method_from_env() -> Result<VaultAuthMethod> {
|
||||
///
|
||||
/// `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` selects Kubernetes authentication, which
|
||||
/// presents the pod's projected ServiceAccount token.
|
||||
///
|
||||
/// `token_override` carries a token supplied on the command line; it stands in
|
||||
/// for `RUSTFS_KMS_VAULT_TOKEN` everywhere below, including the conflict checks,
|
||||
/// so a flag and the variable it mirrors select the same method.
|
||||
fn vault_auth_method_from_env(token_override: Option<&str>) -> Result<VaultAuthMethod> {
|
||||
let token = token_override
|
||||
.map(str::to_string)
|
||||
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_TOKEN));
|
||||
let role_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID);
|
||||
let kubernetes_role = get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_ROLE);
|
||||
|
||||
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
|
||||
// A token file names one authoritative credential source; combining it
|
||||
// with another one would leave the effective identity ambiguous, so
|
||||
// that is a configuration error rather than a precedence rule.
|
||||
if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method"
|
||||
)));
|
||||
for (name, configured) in [
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, role_id.is_some()),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, kubernetes_role.is_some()),
|
||||
(ENV_KMS_VAULT_TOKEN, token.is_some()),
|
||||
] {
|
||||
if configured {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {name}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
}
|
||||
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
|
||||
}
|
||||
|
||||
let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else {
|
||||
if let Some(role) = kubernetes_role {
|
||||
// Unlike a leftover static token, a second login method is never a
|
||||
// stale remnant: both were configured deliberately and neither can be
|
||||
// ranked over the other.
|
||||
if role_id.is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_KUBERNETES_ROLE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
return Ok(VaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount: get_env_str(ENV_KMS_VAULT_KUBERNETES_MOUNT, DEFAULT_VAULT_KUBERNETES_MOUNT),
|
||||
jwt_path: get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_JWT_PATH)
|
||||
.map_or_else(default_vault_kubernetes_jwt_path, PathBuf::from),
|
||||
refresh_safety_window_secs: None,
|
||||
});
|
||||
}
|
||||
|
||||
let Some(role_id) = role_id else {
|
||||
return Ok(VaultAuthMethod::Token {
|
||||
token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"),
|
||||
token: token.unwrap_or_else(|| "dev-token".to_string()),
|
||||
});
|
||||
};
|
||||
|
||||
@@ -1273,6 +1398,22 @@ fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod)
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} => {
|
||||
if role.is_empty() {
|
||||
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes role cannot be empty")));
|
||||
}
|
||||
if mount.is_empty() {
|
||||
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes mount cannot be empty")));
|
||||
}
|
||||
if jwt_path.as_os_str().is_empty() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{backend_name} Kubernetes ServiceAccount token path cannot be empty"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
VaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -1976,6 +2117,106 @@ mod tests {
|
||||
.expect("well-formed token file auth must validate");
|
||||
}
|
||||
|
||||
/// A Kubernetes role alone configures the method: the credential is the
|
||||
/// pod's projected ServiceAccount token, so nothing secret is in the
|
||||
/// environment and the mount and token path fall back to the cluster
|
||||
/// defaults.
|
||||
#[test]
|
||||
fn test_from_env_selects_kubernetes() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||
(ENV_KMS_VAULT_ADDRESS, Some("https://vault.example.com")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_MOUNT, None),
|
||||
(ENV_KMS_VAULT_KUBERNETES_JWT_PATH, None),
|
||||
(ENV_KMS_VAULT_TOKEN, None),
|
||||
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, None),
|
||||
],
|
||||
|| {
|
||||
let config = KmsConfig::from_env().expect("kms config should load from env");
|
||||
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||
let VaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} = &vault.auth_method
|
||||
else {
|
||||
panic!(
|
||||
"a kubernetes role in the environment must select Kubernetes auth, got {:?}",
|
||||
vault.auth_method
|
||||
);
|
||||
};
|
||||
assert_eq!(role, "rustfs");
|
||||
assert_eq!(mount, DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||
assert_eq!(jwt_path, Path::new(DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||
assert_eq!(refresh_safety_window_secs, &None);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_env_kubernetes_is_mutually_exclusive_with_other_auth() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
|
||||
(ENV_KMS_VAULT_TOKEN, None),
|
||||
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||
],
|
||||
|| {
|
||||
let error = KmsConfig::from_env().expect_err("kubernetes combined with approle must be rejected");
|
||||
assert!(error.to_string().contains(ENV_KMS_VAULT_KUBERNETES_ROLE));
|
||||
assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rejects_bad_kubernetes_settings() {
|
||||
let vault_config = |auth_method: VaultAuthMethod| KmsConfig {
|
||||
backend: KmsBackend::VaultTransit,
|
||||
backend_config: BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
||||
address: "https://vault.example.com:8200".to_string(),
|
||||
auth_method,
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let error = vault_config(VaultAuthMethod::kubernetes(String::new()))
|
||||
.validate()
|
||||
.expect_err("an empty kubernetes role must be rejected");
|
||||
assert!(error.to_string().contains("role"), "got {error}");
|
||||
|
||||
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||
role: "rustfs".to_string(),
|
||||
mount: String::new(),
|
||||
jwt_path: PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH),
|
||||
refresh_safety_window_secs: None,
|
||||
})
|
||||
.validate()
|
||||
.expect_err("an empty kubernetes mount must be rejected");
|
||||
assert!(error.to_string().contains("mount"), "got {error}");
|
||||
|
||||
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||
role: "rustfs".to_string(),
|
||||
mount: DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||
jwt_path: PathBuf::new(),
|
||||
refresh_safety_window_secs: None,
|
||||
})
|
||||
.validate()
|
||||
.expect_err("an empty ServiceAccount token path must be rejected");
|
||||
assert!(error.to_string().contains("token path"), "got {error}");
|
||||
|
||||
vault_config(VaultAuthMethod::kubernetes("rustfs".to_string()))
|
||||
.validate()
|
||||
.expect("well-formed kubernetes auth must validate");
|
||||
}
|
||||
|
||||
/// Every KV2 read, write and listing is routed through `kv_mount`, so an
|
||||
/// empty one names a path no Vault engine answers. The Transit backend
|
||||
/// already rejects its own empty mounts; this closes the same gap on the
|
||||
|
||||
@@ -37,7 +37,11 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
humantime.workspace = true
|
||||
http.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
rustfs-signer.workspace = true
|
||||
s3s.workspace = true
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
@@ -49,3 +53,4 @@ doctest = false
|
||||
|
||||
[dev-dependencies]
|
||||
rmp-serde.workspace = true
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net"] }
|
||||
|
||||
@@ -0,0 +1,851 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Admin API HTTP client for heal and scanner management (rustfs/backlog#1869).
|
||||
//!
|
||||
//! [`AdminClient`] speaks the `/rustfs/admin/v3` surface with S3 SigV4
|
||||
//! request signing (the same scheme the server's admin router authenticates),
|
||||
//! so `mc`-style tooling and automation can drive heal start/query/cancel and
|
||||
//! read background-heal / scanner status without hand-rolling HTTP.
|
||||
//!
|
||||
//! Wire structs in this module mirror the server-side shapes
|
||||
//! (`rustfs/src/admin/handlers/heal.rs`, `handlers/scanner.rs`,
|
||||
//! `rustfs-common/src/heal_channel.rs`), following the madmin-go model where
|
||||
//! the SDK owns its own copies and round-trip tests pin the encoding. Deeply
|
||||
//! nested status payloads that the server composes from runtime types are
|
||||
//! carried through as `serde_json::Value` and flattened maps rather than
|
||||
//! duplicated field-for-field, so the client cannot silently drift on fields
|
||||
//! it never interprets.
|
||||
|
||||
use crate::heal_commands::HealResultItem;
|
||||
use http::Method;
|
||||
use serde::{Deserialize, Serialize, de};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Default admin API path prefix on a RustFS endpoint.
|
||||
pub const DEFAULT_ADMIN_API_PREFIX: &str = "/rustfs/admin";
|
||||
/// Default SigV4 region when the server has no explicit region configured.
|
||||
pub const DEFAULT_REGION: &str = "us-east-1";
|
||||
|
||||
/// Scan mode for a heal request, mirroring the server's numeric-or-name wire
|
||||
/// encoding (`0` unknown/default, `1` normal, `2` deep).
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum HealScanMode {
|
||||
/// Server default; behaves as [`HealScanMode::Normal`].
|
||||
#[default]
|
||||
Unknown,
|
||||
/// Metadata-level checks only.
|
||||
Normal,
|
||||
/// Full bitrot verification while healing.
|
||||
Deep,
|
||||
}
|
||||
|
||||
impl HealScanMode {
|
||||
fn wire_number(self) -> u8 {
|
||||
match self {
|
||||
Self::Unknown => 0,
|
||||
Self::Normal => 1,
|
||||
Self::Deep => 2,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_wire_number(value: u8) -> Option<Self> {
|
||||
match value {
|
||||
0 => Some(Self::Unknown),
|
||||
1 => Some(Self::Normal),
|
||||
2 => Some(Self::Deep),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_wire_name(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"unknown" => Some(Self::Unknown),
|
||||
"normal" => Some(Self::Normal),
|
||||
"deep" => Some(Self::Deep),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for HealScanMode {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
serializer.serialize_u8(self.wire_number())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for HealScanMode {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct HealScanModeVisitor;
|
||||
|
||||
impl de::Visitor<'_> for HealScanModeVisitor {
|
||||
type Value = HealScanMode;
|
||||
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("a heal scan mode number or name")
|
||||
}
|
||||
|
||||
fn visit_u64<E: de::Error>(self, value: u64) -> Result<Self::Value, E> {
|
||||
u8::try_from(value)
|
||||
.ok()
|
||||
.and_then(HealScanMode::from_wire_number)
|
||||
.ok_or_else(|| E::custom(format!("unknown heal scan mode number: {value}")))
|
||||
}
|
||||
|
||||
fn visit_str<E: de::Error>(self, value: &str) -> Result<Self::Value, E> {
|
||||
HealScanMode::from_wire_name(value).ok_or_else(|| E::custom(format!("unknown heal scan mode name: {value}")))
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(HealScanModeVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
/// Heal options for an admin heal request (mirror of the server body type).
|
||||
/// Fields default on decode: a client should tolerate a server response whose
|
||||
/// settings object omits fields it never set.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct HealOpts {
|
||||
#[serde(default)]
|
||||
pub recursive: bool,
|
||||
#[serde(rename = "dryRun", default)]
|
||||
pub dry_run: bool,
|
||||
#[serde(default)]
|
||||
pub remove: bool,
|
||||
#[serde(default)]
|
||||
pub recreate: bool,
|
||||
#[serde(rename = "scanMode", default)]
|
||||
pub scan_mode: HealScanMode,
|
||||
#[serde(rename = "updateParity", default)]
|
||||
pub update_parity: bool,
|
||||
#[serde(rename = "nolock", default)]
|
||||
pub no_lock: bool,
|
||||
#[serde(rename = "pool", default)]
|
||||
pub pool: Option<usize>,
|
||||
#[serde(rename = "set", default)]
|
||||
pub set: Option<usize>,
|
||||
}
|
||||
|
||||
/// Successful heal start / path-scoped cancel response.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealStartSuccess {
|
||||
pub client_token: String,
|
||||
pub client_address: String,
|
||||
#[serde(default)]
|
||||
pub start_time: String,
|
||||
}
|
||||
|
||||
/// Heal task status response (query, cancel-with-token, start-then-poll).
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealTaskStatus {
|
||||
/// `running` | `finished` | `stopped` | `notFound`.
|
||||
pub summary: String,
|
||||
/// Failure detail for stopped tasks; empty otherwise.
|
||||
#[serde(rename = "detail", default)]
|
||||
pub failure_detail: String,
|
||||
#[serde(default)]
|
||||
pub start_time: String,
|
||||
#[serde(default)]
|
||||
pub settings: HealOpts,
|
||||
#[serde(default)]
|
||||
pub items: Vec<HealResultItem>,
|
||||
#[serde(default)]
|
||||
pub truncated: bool,
|
||||
/// Live progress snapshot; the exact shape is owned by the heal runtime.
|
||||
#[serde(default)]
|
||||
pub progress: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `POST /v3/background-heal/status` response. Known top-level fields are
|
||||
/// typed; the flattened heal info and operations matrix pass through verbatim.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackgroundHealStatus {
|
||||
/// `disabled` | `uninitialized` | `idle` | `active` | `degraded`.
|
||||
pub state: String,
|
||||
#[serde(default)]
|
||||
pub heal_queue_length: u64,
|
||||
#[serde(default)]
|
||||
pub heal_active_tasks: u64,
|
||||
#[serde(default)]
|
||||
pub cluster_status_complete: bool,
|
||||
#[serde(default)]
|
||||
pub progress: Option<serde_json::Value>,
|
||||
/// Remaining wire fields (flattened `BackgroundHealInfo` plus the
|
||||
/// priority-by-source operations matrix), carried verbatim.
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `GET /v3/scanner/status` response, typed at the fields operators branch
|
||||
/// on; everything else passes through verbatim.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ScannerStatus {
|
||||
pub enabled: bool,
|
||||
/// `fresh` | `stale` | `unknown`; absent when the scanner never completed
|
||||
/// a cycle.
|
||||
#[serde(default)]
|
||||
pub freshness: Option<ScannerFreshness>,
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Freshness block of the scanner status response.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ScannerFreshness {
|
||||
/// `fresh` | `stale` | `unknown`.
|
||||
pub state: String,
|
||||
}
|
||||
|
||||
impl ScannerStatus {
|
||||
/// Convenience accessor for the freshness state string.
|
||||
pub fn freshness(&self) -> &str {
|
||||
self.freshness
|
||||
.as_ref()
|
||||
.map(|freshness| freshness.state.as_str())
|
||||
.unwrap_or("unknown")
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything that can go wrong in an admin client call.
|
||||
#[derive(Debug)]
|
||||
pub enum AdminClientError {
|
||||
/// The endpoint URL could not be parsed.
|
||||
InvalidEndpoint(String),
|
||||
/// Request build/send failed (DNS, connect, timeout, body read).
|
||||
Transport(reqwest::Error),
|
||||
/// The server answered a non-2xx status.
|
||||
HttpStatus { status: u16, body: String },
|
||||
/// The response body did not decode into the expected shape.
|
||||
Decode { message: String },
|
||||
}
|
||||
|
||||
impl std::fmt::Display for AdminClientError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::InvalidEndpoint(message) => write!(f, "invalid admin endpoint: {message}"),
|
||||
Self::Transport(err) => write!(f, "admin request transport failure: {err}"),
|
||||
Self::HttpStatus { status, body } => write!(f, "admin request failed with HTTP {status}: {body}"),
|
||||
Self::Decode { message } => write!(f, "admin response decode failure: {message}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for AdminClientError {}
|
||||
|
||||
impl From<reqwest::Error> for AdminClientError {
|
||||
fn from(err: reqwest::Error) -> Self {
|
||||
Self::Transport(err)
|
||||
}
|
||||
}
|
||||
|
||||
/// A signed client for a RustFS admin API.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AdminClient {
|
||||
endpoint: reqwest::Url,
|
||||
access_key: String,
|
||||
secret_key: String,
|
||||
session_token: String,
|
||||
region: String,
|
||||
api_prefix: String,
|
||||
http: reqwest::Client,
|
||||
}
|
||||
|
||||
impl AdminClient {
|
||||
/// Build a client for `endpoint` (e.g. `http://127.0.0.1:9000`) using root
|
||||
/// or admin credentials. Requests are SigV4-signed with the same scheme
|
||||
/// the server's admin router authenticates.
|
||||
pub fn new(endpoint: &str, access_key: &str, secret_key: &str) -> Result<Self, AdminClientError> {
|
||||
let url = reqwest::Url::parse(endpoint).map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?;
|
||||
if url.host_str().is_none() {
|
||||
return Err(AdminClientError::InvalidEndpoint("endpoint has no host".to_string()));
|
||||
}
|
||||
let http = reqwest::Client::builder()
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.map_err(AdminClientError::Transport)?;
|
||||
Ok(Self {
|
||||
endpoint: url,
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: secret_key.to_string(),
|
||||
session_token: String::new(),
|
||||
region: DEFAULT_REGION.to_string(),
|
||||
api_prefix: DEFAULT_ADMIN_API_PREFIX.to_string(),
|
||||
http,
|
||||
})
|
||||
}
|
||||
|
||||
/// Attach an STS session token (signed as `x-amz-security-token`).
|
||||
pub fn with_session_token(mut self, session_token: impl Into<String>) -> Self {
|
||||
self.session_token = session_token.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the SigV4 region (defaults to `us-east-1`, matching a
|
||||
/// region-less RustFS deployment).
|
||||
pub fn with_region(mut self, region: impl Into<String>) -> Self {
|
||||
self.region = region.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the admin API path prefix (defaults to `/rustfs/admin`).
|
||||
pub fn with_api_prefix(mut self, prefix: impl Into<String>) -> Self {
|
||||
self.api_prefix = prefix.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Start a heal. `bucket` empty and `prefix` empty heals the whole
|
||||
/// deployment (requires `recursive` or a `pool`/`set` pair in `opts`,
|
||||
/// enforced server-side); a bucket alone heals the bucket (the server
|
||||
/// forces `recursive` for bucket heals).
|
||||
pub async fn heal_start(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
prefix: Option<&str>,
|
||||
opts: &HealOpts,
|
||||
force_start: bool,
|
||||
) -> Result<HealStartSuccess, AdminClientError> {
|
||||
let body = serde_json::to_vec(opts).map_err(|err| AdminClientError::Decode {
|
||||
message: err.to_string(),
|
||||
})?;
|
||||
let mut query = Vec::new();
|
||||
if force_start {
|
||||
query.push(("forceStart", "true".to_string()));
|
||||
}
|
||||
self.post_json(&heal_path(bucket, prefix), &query, body).await
|
||||
}
|
||||
|
||||
/// Query the status of the heal identified by `client_token` (the token
|
||||
/// returned by [`Self::heal_start`]) at the path it was started on.
|
||||
pub async fn heal_status(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
prefix: Option<&str>,
|
||||
client_token: &str,
|
||||
) -> Result<HealTaskStatus, AdminClientError> {
|
||||
self.post_json(&heal_path(bucket, prefix), &[("clientToken", client_token.to_string())], Vec::new())
|
||||
.await
|
||||
}
|
||||
|
||||
/// Stop a heal: with a `client_token` only that task is cancelled and its
|
||||
/// final status returned; without one, every heal task at the path is
|
||||
/// cancelled (the server answers with a start-success-shaped receipt).
|
||||
pub async fn heal_stop(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
prefix: Option<&str>,
|
||||
client_token: Option<&str>,
|
||||
) -> Result<HealStopOutcome, AdminClientError> {
|
||||
let mut query = vec![("forceStop", "true".to_string())];
|
||||
if let Some(token) = client_token {
|
||||
query.push(("clientToken", token.to_string()));
|
||||
}
|
||||
match client_token {
|
||||
Some(_) => {
|
||||
let status: HealTaskStatus = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
|
||||
Ok(HealStopOutcome::Stopped(status))
|
||||
}
|
||||
None => {
|
||||
let success: HealStartSuccess = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
|
||||
Ok(HealStopOutcome::PathStopped(success))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cluster-aggregated background heal status.
|
||||
pub async fn background_heal_status(&self) -> Result<BackgroundHealStatus, AdminClientError> {
|
||||
self.get_json("/v3/background-heal/status").await
|
||||
}
|
||||
|
||||
/// Data scanner status (enabled state, freshness, runtime config).
|
||||
pub async fn scanner_status(&self) -> Result<ScannerStatus, AdminClientError> {
|
||||
self.get_json("/v3/scanner/status").await
|
||||
}
|
||||
|
||||
/// ILM expiry worker status. The payload is owned by the expiry
|
||||
/// subsystem and still evolving; returned verbatim.
|
||||
pub async fn ilm_expiry_status(&self) -> Result<serde_json::Value, AdminClientError> {
|
||||
self.get_json("/v3/ilm/expiry/status").await
|
||||
}
|
||||
|
||||
/// Durable replacement-recovery status (admin v4). The payload is owned
|
||||
/// by the heal runtime; returned verbatim.
|
||||
pub async fn replacement_recovery_status(&self) -> Result<serde_json::Value, AdminClientError> {
|
||||
self.get_json("/v4/heal/replacement-recovery").await
|
||||
}
|
||||
|
||||
/// Signed GET returning a decoded JSON body; escape hatch for endpoints
|
||||
/// this client does not wrap yet.
|
||||
pub async fn get_json<T: for<'de> Deserialize<'de>>(&self, path: &str) -> Result<T, AdminClientError> {
|
||||
let url = self.url_for(path, &[])?;
|
||||
let request = self.sign_and_build(Method::GET, url, Vec::new(), None).await?;
|
||||
self.execute(request).await
|
||||
}
|
||||
|
||||
/// Signed POST returning a decoded JSON body.
|
||||
async fn post_json<T: for<'de> Deserialize<'de>>(
|
||||
&self,
|
||||
path: &str,
|
||||
query: &[(&str, String)],
|
||||
body: Vec<u8>,
|
||||
) -> Result<T, AdminClientError> {
|
||||
let content_type = if body.is_empty() { None } else { Some("application/json") };
|
||||
let url = self.url_for(path, query)?;
|
||||
let request = self.sign_and_build(Method::POST, url, body, content_type).await?;
|
||||
self.execute(request).await
|
||||
}
|
||||
|
||||
fn url_for(&self, path: &str, query: &[(&str, String)]) -> Result<reqwest::Url, AdminClientError> {
|
||||
let mut url = self
|
||||
.endpoint
|
||||
.join(&format!("{}{}", self.api_prefix.trim_end_matches('/'), path))
|
||||
.map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?;
|
||||
if !query.is_empty() {
|
||||
let mut pairs = url.query_pairs_mut();
|
||||
for (key, value) in query {
|
||||
pairs.append_pair(key, value);
|
||||
}
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
/// Build a SigV4-signed request via the same signer the server trusts,
|
||||
/// then hand the signed headers to the HTTP client. The signature covers
|
||||
/// method, path, query, and an unsigned-payload marker — the same shape
|
||||
/// RustFS itself sends for peer admin calls.
|
||||
async fn sign_and_build(
|
||||
&self,
|
||||
method: Method,
|
||||
url: reqwest::Url,
|
||||
body: Vec<u8>,
|
||||
content_type: Option<&str>,
|
||||
) -> Result<reqwest::Request, AdminClientError> {
|
||||
let authority = match (url.host_str(), url.port_or_known_default()) {
|
||||
(Some(host), Some(port)) => format!("{host}:{port}"),
|
||||
_ => return Err(AdminClientError::InvalidEndpoint("endpoint has no authority".to_string())),
|
||||
};
|
||||
let mut builder = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(url.as_str())
|
||||
.header(http::header::HOST, &authority)
|
||||
.header("x-amz-content-sha256", rustfs_signer::constants::UNSIGNED_PAYLOAD);
|
||||
if let Some(content_type) = content_type {
|
||||
builder = builder.header(http::header::CONTENT_TYPE, content_type);
|
||||
}
|
||||
let unsigned = builder
|
||||
.body(s3s::Body::empty())
|
||||
.map_err(|err| AdminClientError::InvalidEndpoint(format!("build request failed: {err}")))?;
|
||||
let signed = rustfs_signer::sign_v4(
|
||||
unsigned,
|
||||
body.len() as i64,
|
||||
&self.access_key,
|
||||
&self.secret_key,
|
||||
&self.session_token,
|
||||
&self.region,
|
||||
);
|
||||
|
||||
let mut request = self
|
||||
.http
|
||||
.request(method, url)
|
||||
.body(body)
|
||||
.build()
|
||||
.map_err(AdminClientError::Transport)?;
|
||||
let headers = request.headers_mut();
|
||||
for (name, value) in signed.headers().iter() {
|
||||
// HOST is owned by the HTTP client; the signed value above was
|
||||
// built from the same URL authority, so they always agree.
|
||||
if name == http::header::HOST {
|
||||
continue;
|
||||
}
|
||||
headers.insert(name, value.clone());
|
||||
}
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
async fn execute<T: for<'de> Deserialize<'de>>(&self, request: reqwest::Request) -> Result<T, AdminClientError> {
|
||||
let response = self.http.execute(request).await?;
|
||||
let status = response.status();
|
||||
let bytes = response.bytes().await?;
|
||||
if !status.is_success() {
|
||||
return Err(AdminClientError::HttpStatus {
|
||||
status: status.as_u16(),
|
||||
body: String::from_utf8_lossy(&bytes).into_owned(),
|
||||
});
|
||||
}
|
||||
serde_json::from_slice(&bytes).map_err(|err| AdminClientError::Decode {
|
||||
message: err.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Response of [`AdminClient::heal_stop`]: cancelling a single tokened task
|
||||
/// answers with that task's status, cancelling a whole path answers with a
|
||||
/// start-success-shaped receipt.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum HealStopOutcome {
|
||||
Stopped(HealTaskStatus),
|
||||
PathStopped(HealStartSuccess),
|
||||
}
|
||||
|
||||
fn heal_path(bucket: Option<&str>, prefix: Option<&str>) -> String {
|
||||
match (bucket, prefix) {
|
||||
(Some(bucket), Some(prefix)) if !bucket.is_empty() && !prefix.is_empty() => {
|
||||
format!("/v3/heal/{}/{}", percent_encode_path_segment(bucket), percent_encode_path_segment(prefix))
|
||||
}
|
||||
(Some(bucket), Some(_)) | (Some(bucket), None) if !bucket.is_empty() => {
|
||||
format!("/v3/heal/{}", percent_encode_path_segment(bucket))
|
||||
}
|
||||
_ => "/v3/heal/".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a single path segment (slashes are content, not separators, inside
|
||||
/// bucket/prefix path params).
|
||||
fn percent_encode_path_segment(segment: &str) -> String {
|
||||
let mut out = String::with_capacity(segment.len());
|
||||
for byte in segment.bytes() {
|
||||
match byte {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => out.push(byte as char),
|
||||
_ => out.push_str(&format!("%{byte:02X}")),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
AdminClient, AdminClientError, BackgroundHealStatus, HealOpts, HealScanMode, HealStartSuccess, HealTaskStatus,
|
||||
ScannerStatus, heal_path, percent_encode_path_segment,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[test]
|
||||
fn heal_paths_cover_root_bucket_and_prefix() {
|
||||
assert_eq!(heal_path(None, None), "/v3/heal/");
|
||||
assert_eq!(heal_path(Some(""), Some("")), "/v3/heal/");
|
||||
assert_eq!(heal_path(Some("bucket"), None), "/v3/heal/bucket");
|
||||
assert_eq!(heal_path(Some("bucket"), Some("pre/fix")), "/v3/heal/bucket/pre%2Ffix");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_segments_percent_encode_reserved_characters() {
|
||||
assert_eq!(percent_encode_path_segment("a b"), "a%20b");
|
||||
assert_eq!(percent_encode_path_segment("a/b"), "a%2Fb");
|
||||
assert_eq!(percent_encode_path_segment("ü"), "%C3%BC");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_opts_round_trip_through_the_server_wire_shape() {
|
||||
let opts = HealOpts {
|
||||
recursive: true,
|
||||
dry_run: false,
|
||||
remove: true,
|
||||
recreate: false,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
update_parity: true,
|
||||
no_lock: false,
|
||||
pool: Some(1),
|
||||
set: Some(2),
|
||||
};
|
||||
let wire = serde_json::to_value(&opts).unwrap();
|
||||
assert_eq!(wire["scanMode"], json!(2), "the server body decodes scanMode as a number");
|
||||
let back: HealOpts = serde_json::from_value(wire).unwrap();
|
||||
assert_eq!(back.scan_mode, HealScanMode::Deep);
|
||||
assert_eq!(back.pool, Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_scan_mode_accepts_both_wire_encodings() {
|
||||
assert_eq!(serde_json::from_value::<HealScanMode>(json!(1)).unwrap(), HealScanMode::Normal);
|
||||
assert_eq!(serde_json::from_value::<HealScanMode>(json!("deep")).unwrap(), HealScanMode::Deep);
|
||||
assert!(serde_json::from_value::<HealScanMode>(json!(9)).is_err());
|
||||
assert!(serde_json::from_value::<HealScanMode>(json!("sideways")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_task_status_decodes_the_server_response_shape() {
|
||||
let raw = json!({
|
||||
"summary": "finished",
|
||||
"detail": "",
|
||||
"startTime": "2026-08-17T00:00:00Z",
|
||||
"settings": {"recursive": false, "scanMode": 1},
|
||||
"items": [{
|
||||
"resultId": 1, "type": "object", "bucket": "b", "object": "o", "versionId": "", "detail": "",
|
||||
"parityBlocks": 2, "dataBlocks": 2, "diskCount": 4, "setCount": 1,
|
||||
"before": {"drives": []}, "after": {"drives": []}, "objectSize": 128
|
||||
}],
|
||||
"truncated": false
|
||||
});
|
||||
let status: HealTaskStatus = serde_json::from_value(raw).unwrap();
|
||||
assert_eq!(status.summary, "finished");
|
||||
assert_eq!(status.items.len(), 1);
|
||||
assert_eq!(status.settings.scan_mode, HealScanMode::Normal);
|
||||
assert!(status.progress.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_heal_status_types_known_fields_and_passes_the_rest_through() {
|
||||
let raw = json!({
|
||||
"state": "active",
|
||||
"bitrotStartTime": "t",
|
||||
"healQueueLength": 3,
|
||||
"healActiveTasks": 1,
|
||||
"healOperations": {"queueLength": 3},
|
||||
"clusterStatusComplete": true
|
||||
});
|
||||
let status: BackgroundHealStatus = serde_json::from_value(raw).unwrap();
|
||||
assert_eq!(status.state, "active");
|
||||
assert_eq!(status.heal_queue_length, 3);
|
||||
assert!(status.cluster_status_complete);
|
||||
assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_status_defaults_freshness_to_unknown() {
|
||||
let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}});
|
||||
let status: ScannerStatus = serde_json::from_value(raw).unwrap();
|
||||
assert_eq!(status.freshness(), "stale");
|
||||
let bare: ScannerStatus = serde_json::from_value(json!({"enabled": false})).unwrap();
|
||||
assert_eq!(bare.freshness(), "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_endpoint_is_rejected_without_io() {
|
||||
let err = AdminClient::new("not a url", "ak", "sk").unwrap_err();
|
||||
assert!(matches!(err, AdminClientError::InvalidEndpoint(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn signed_requests_carry_sigv4_authorization_and_correct_target() {
|
||||
let server = TestServer::spawn(r#"{"clientToken":"token-1","clientAddress":"127.0.0.1:9","startTime":"t"}"#, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "minioadmin", "minioadmin")
|
||||
.expect("client builds against the test server");
|
||||
|
||||
let start: HealStartSuccess = client
|
||||
.heal_start(
|
||||
Some("bucket"),
|
||||
None,
|
||||
&HealOpts {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("signed heal start decodes");
|
||||
|
||||
assert_eq!(start.client_token, "token-1");
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket");
|
||||
assert!(!request.query.contains("forceStart"), "absent flags must not be sent");
|
||||
let auth = request.header("authorization").expect("request must be signed");
|
||||
assert!(auth.starts_with("AWS4-HMAC-SHA256"), "SigV4 scheme, got: {auth}");
|
||||
assert!(auth.contains("Credential=minioadmin/"), "credentials must be in the Authorization header");
|
||||
assert_eq!(
|
||||
request.header("x-amz-content-sha256").as_deref(),
|
||||
Some("UNSIGNED-PAYLOAD"),
|
||||
"the client signs the same payload marker RustFS peer calls use"
|
||||
);
|
||||
assert_eq!(request.header("content-type").as_deref(), Some("application/json"));
|
||||
assert!(request.body.contains("\"recursive\":true"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_sends_client_token_on_the_same_path() {
|
||||
let body = r#"{"summary":"running","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#;
|
||||
let server = TestServer::spawn(body, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let status = client
|
||||
.heal_status(Some("bucket"), None, "token-1")
|
||||
.await
|
||||
.expect("status decodes");
|
||||
assert_eq!(status.summary, "running");
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket");
|
||||
assert!(request.query.contains("clientToken=token-1"));
|
||||
assert!(!request.query.contains("forceStop"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_without_token_takes_the_path_cancel_branch() {
|
||||
let server = TestServer::spawn(r#"{"clientToken":"path","clientAddress":"c","startTime":"t"}"#, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let outcome = client.heal_stop(Some("bucket"), None, None).await.expect("path stop decodes");
|
||||
assert!(matches!(outcome, super::HealStopOutcome::PathStopped(_)));
|
||||
let request = server.recorded();
|
||||
assert!(request.query.contains("forceStop=true"));
|
||||
assert!(!request.query.contains("clientToken"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_error_status_maps_to_a_typed_error_with_body() {
|
||||
let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
let err = client.scanner_status().await.unwrap_err();
|
||||
match err {
|
||||
AdminClientError::HttpStatus { status, body } => {
|
||||
assert_eq!(status, 403);
|
||||
assert!(body.contains("AccessDenied"));
|
||||
}
|
||||
other => panic!("expected HttpStatus, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_success_body_maps_to_a_decode_error() {
|
||||
let server = TestServer::spawn("not json", 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
assert!(matches!(client.scanner_status().await.unwrap_err(), AdminClientError::Decode { .. }));
|
||||
}
|
||||
|
||||
/// One recorded request, parsed off the wire with the minimum needed for
|
||||
/// assertions: method, path, query, headers, body.
|
||||
#[derive(Debug, Clone)]
|
||||
struct RecordedRequest {
|
||||
method: String,
|
||||
path: String,
|
||||
query: String,
|
||||
headers: Vec<(String, String)>,
|
||||
body: String,
|
||||
}
|
||||
|
||||
impl RecordedRequest {
|
||||
fn header(&self, name: &str) -> Option<String> {
|
||||
self.headers
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal HTTP/1.1 server: one canned response per connection, every
|
||||
/// request recorded behind an `Arc<Mutex>`. Deliberately dependency-free —
|
||||
/// the assertions only need the raw request bytes.
|
||||
struct TestServer {
|
||||
addr: std::net::SocketAddr,
|
||||
requests: Arc<Mutex<Vec<RecordedRequest>>>,
|
||||
}
|
||||
|
||||
impl TestServer {
|
||||
async fn spawn(response_body: &'static str, status: u16) -> Self {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind ephemeral port");
|
||||
let addr = listener.local_addr().expect("local addr");
|
||||
let requests: Arc<Mutex<Vec<RecordedRequest>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
let recorded = requests.clone();
|
||||
tokio::spawn(async move {
|
||||
let reason = if status == 200 { "OK" } else { "Forbidden" };
|
||||
let response = format!(
|
||||
"HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{response_body}",
|
||||
response_body.len()
|
||||
);
|
||||
// Each request is a fresh connection (connection: close); a
|
||||
// bounded loop serves every call a test makes while letting
|
||||
// the task exit instead of lingering for the whole process.
|
||||
for _ in 0..16 {
|
||||
let Ok((mut stream, _)) = listener.accept().await else {
|
||||
break;
|
||||
};
|
||||
let mut buffer = Vec::with_capacity(2048);
|
||||
let mut chunk = [0u8; 2048];
|
||||
// Read headers plus content-length body, or stop on close.
|
||||
loop {
|
||||
if let Some(end) = find_header_end(&buffer) {
|
||||
let content_length = extract_content_length(&buffer[..end]);
|
||||
if buffer.len() >= end + content_length {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let n = match stream.read(&mut chunk).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
};
|
||||
buffer.extend_from_slice(&chunk[..n]);
|
||||
if buffer.len() > 64 * 1024 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(request) = parse_request(&buffer) {
|
||||
recorded.lock().expect("recorded lock").push(request);
|
||||
}
|
||||
let _ = stream.write_all(response.as_bytes()).await;
|
||||
let _ = stream.shutdown().await;
|
||||
}
|
||||
});
|
||||
|
||||
Self { addr, requests }
|
||||
}
|
||||
|
||||
fn recorded(&self) -> RecordedRequest {
|
||||
self.requests
|
||||
.lock()
|
||||
.expect("recorded lock")
|
||||
.last()
|
||||
.cloned()
|
||||
.expect("the client call must have produced one recorded request")
|
||||
}
|
||||
}
|
||||
|
||||
fn find_header_end(buffer: &[u8]) -> Option<usize> {
|
||||
buffer.windows(4).position(|window| window == b"\r\n\r\n").map(|pos| pos + 4)
|
||||
}
|
||||
|
||||
fn extract_content_length(headers: &[u8]) -> usize {
|
||||
let text = String::from_utf8_lossy(headers).to_ascii_lowercase();
|
||||
text.lines()
|
||||
.find_map(|line| line.strip_prefix("content-length:"))
|
||||
.and_then(|value| value.trim().parse().ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn parse_request(raw: &[u8]) -> Option<RecordedRequest> {
|
||||
let end = find_header_end(raw)?;
|
||||
let head = String::from_utf8_lossy(&raw[..end]);
|
||||
let body = String::from_utf8_lossy(&raw[end..]).into_owned();
|
||||
let mut lines = head.lines();
|
||||
let request_line = lines.next()?;
|
||||
let mut parts = request_line.split_whitespace();
|
||||
let method = parts.next()?.to_string();
|
||||
let target = parts.next()?.to_string();
|
||||
let (path, query) = match target.split_once('?') {
|
||||
Some((path, query)) => (path.to_string(), query.to_string()),
|
||||
None => (target, String::new()),
|
||||
};
|
||||
let headers = lines
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.map(|(name, value)| (name.trim().to_string(), value.trim().to_string()))
|
||||
.collect();
|
||||
Some(RecordedRequest {
|
||||
method,
|
||||
path,
|
||||
query,
|
||||
headers,
|
||||
body,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod client;
|
||||
pub mod group;
|
||||
pub mod heal_commands;
|
||||
pub mod health;
|
||||
@@ -25,6 +26,7 @@ pub mod trace;
|
||||
pub mod user;
|
||||
pub mod utils;
|
||||
|
||||
pub use client::*;
|
||||
pub use group::*;
|
||||
pub use info_commands::*;
|
||||
pub use policy::*;
|
||||
|
||||
@@ -43,7 +43,7 @@ pub struct ServiceTraceOpts {
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ServiceTraceOpts {
|
||||
fn trace_types(&self) -> TraceType {
|
||||
pub fn trace_types(&self) -> TraceType {
|
||||
let mut tt = TraceType::default();
|
||||
tt.set_if(self.s3, &TraceType::S3);
|
||||
tt.set_if(self.internal, &TraceType::INTERNAL);
|
||||
@@ -72,6 +72,14 @@ impl ServiceTraceOpts {
|
||||
tt
|
||||
}
|
||||
|
||||
pub fn only_errors(&self) -> bool {
|
||||
self.only_errors
|
||||
}
|
||||
|
||||
pub fn threshold(&self) -> Duration {
|
||||
self.threshold
|
||||
}
|
||||
|
||||
pub fn parse_params(&mut self, uri: &Uri) -> Result<(), String> {
|
||||
let query_pairs: HashMap<_, _> = uri
|
||||
.query()
|
||||
|
||||
@@ -258,7 +258,7 @@ pub struct SRLDAPUser {
|
||||
pub api_version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct SRIAMUser {
|
||||
#[serde(rename = "accessKey", default)]
|
||||
pub access_key: String,
|
||||
@@ -270,7 +270,7 @@ pub struct SRIAMUser {
|
||||
pub api_version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct SRGroupInfo {
|
||||
#[serde(rename = "updateReq", default)]
|
||||
pub update_req: GroupAddRemove,
|
||||
@@ -346,7 +346,7 @@ pub struct SRCredInfo {
|
||||
pub api_version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct SRIAMItem {
|
||||
#[serde(default)]
|
||||
pub r#type: String,
|
||||
|
||||
@@ -108,6 +108,9 @@ temp-env = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
# Test-only: pins the emitted scanner alert wire names against the canonical
|
||||
# EventName string forms subscribers configure (rustfs/backlog#1868).
|
||||
rustfs-s3-types.workspace = true
|
||||
# Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via
|
||||
# the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6).
|
||||
rustfs-ecstore = { workspace = true, features = ["test-util"] }
|
||||
|
||||
@@ -6,6 +6,8 @@ admission, scanner-driven heal/bitrot checks, and namespace alerts.
|
||||
|
||||
For operator-facing runtime controls, status fields, and tuning workflows, see
|
||||
[Scanner Runtime Controls](../../docs/operations/scanner-runtime-controls.md).
|
||||
For a MinIO `data-scanner` comparison and the improvement contracts, see
|
||||
[MinIO Scanner Compatibility](../../docs/architecture/minio-scanner-compat.md).
|
||||
For repeatable scanner-pressure validation, see
|
||||
[Scanner Benchmark Runbook](../../docs/operations/scanner-benchmark-runbook.md).
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ RustFS Scanner 是后台维护扫描循环,负责用量统计、生命周期
|
||||
|
||||
面向运维人员的运行时控制项、状态字段和调参流程,请参考
|
||||
[Scanner Runtime Controls](../../docs/operations/scanner-runtime-controls.md)。
|
||||
与 MinIO `data-scanner` 的对照和补齐约定请参考
|
||||
[MinIO Scanner Compatibility](../../docs/architecture/minio-scanner-compat.md)。
|
||||
可复现的 scanner 压力验证流程请参考
|
||||
[Scanner Benchmark Runbook](../../docs/operations/scanner-benchmark-runbook.md)。
|
||||
|
||||
|
||||
@@ -28,7 +28,8 @@ use rustfs_common::heal_channel::HealScanMode;
|
||||
use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
|
||||
pub use rustfs_data_usage::{
|
||||
AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
|
||||
DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, TierStats, hash_path,
|
||||
DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry,
|
||||
PrefixUsageQuery, PrefixUsageSummary, TierStats, hash_path, prefix_usage_in_cache,
|
||||
};
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
@@ -430,6 +431,13 @@ pub(crate) enum DataUsageCachePrepareOutcome {
|
||||
}
|
||||
|
||||
impl DataUsageCache {
|
||||
/// Prefix-level usage query over this (writer-side) cache; see
|
||||
/// [`prefix_usage_in_cache`] for the semantics
|
||||
/// (rustfs/backlog#1872).
|
||||
pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option<PrefixUsageQuery> {
|
||||
prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_for_scan(
|
||||
&mut self,
|
||||
name: &str,
|
||||
|
||||
@@ -53,6 +53,7 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
pub mod data_usage_define;
|
||||
pub mod error;
|
||||
pub mod prefix_usage;
|
||||
mod remote_scanner;
|
||||
pub mod runtime_config;
|
||||
pub mod scanner;
|
||||
@@ -64,6 +65,7 @@ pub(crate) mod storage_api;
|
||||
|
||||
pub use data_usage_define::*;
|
||||
pub use error::ScannerError;
|
||||
pub use prefix_usage::{BucketPrefixUsageResponse, bucket_prefix_usage, invalidate_prefix_usage_cache};
|
||||
pub use remote_scanner::{
|
||||
NS_SCANNER_MAX_REQUEST_BODY_SIZE, RemoteScannerAdmission, RemoteScannerRequest, admit_remote_scanner_request,
|
||||
claim_remote_scanner_request, decode_remote_scanner_request, preflight_remote_scanner_request,
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Prefix-level bucket usage for admin/console consumers (rustfs/backlog#1872,
|
||||
//! MinIO `loadPrefixUsageFromBackend` parity).
|
||||
//!
|
||||
//! The per-bucket, per-set `.usage-cache.bin` objects already hold a
|
||||
//! path-keyed prefix tree; this module reads every set's copy through that
|
||||
//! set's own object layer (the hash-routed store path would always land on
|
||||
//! one set), aggregates the overlapping trees, and serves the result from a
|
||||
//! bounded 30-second cache. Bucket writes poke the cache through the
|
||||
//! dirty-usage hook so a fresh scan is visible immediately.
|
||||
|
||||
use crate::data_usage_define::{DATA_USAGE_CACHE_NAME, DataUsageCache};
|
||||
use crate::error::ScannerError;
|
||||
use crate::storage_api::owner::{
|
||||
EcstoreSetDisks, EcstoreStore, ecstore_is_reserved_or_invalid_bucket, ecstore_resolve_object_store_handle,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
use rustfs_data_usage::{PrefixUsageEntry, PrefixUsageSummary};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
const LOG_COMPONENT_SCANNER: &str = "scanner";
|
||||
const LOG_SUBSYSTEM_PREFIX_USAGE: &str = "prefix_usage";
|
||||
const EVENT_PREFIX_USAGE_CACHE_STATE: &str = "prefix_usage_cache_state";
|
||||
|
||||
/// How long a computed breakdown stays fresh. MinIO uses the same 30s for
|
||||
/// its prefix-usage cache; bucket writes additionally invalidate on the spot.
|
||||
const CACHE_TTL: Duration = Duration::from_secs(30);
|
||||
/// Hard entry cap for the result cache; exceeded, expired entries go first
|
||||
/// and the map clears rather than growing past the bound.
|
||||
const CACHE_MAX_ENTRIES: usize = 128;
|
||||
/// Per-set cache read budget. The underlying loader retries for up to a
|
||||
/// minute per attempt on backend errors — far too long for an admin GET, so
|
||||
/// a slow set degrades to "not reporting" instead of stalling the caller.
|
||||
const PER_SET_LOAD_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Aggregated prefix-usage answer across every erasure set.
|
||||
#[derive(Clone, Debug, PartialEq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BucketPrefixUsageResponse {
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub usage: PrefixUsageSummary,
|
||||
/// Every reporting set's prefix entry was compacted: the aggregate is
|
||||
/// valid, the sub-prefix breakdown is empty on disk.
|
||||
pub compacted: bool,
|
||||
/// The sub-prefix breakdown is incomplete: at least one reporting set
|
||||
/// had the prefix compacted (or absent while others found it), so its
|
||||
/// objects cannot be attributed to a sub-prefix.
|
||||
pub sub_prefixes_partial: bool,
|
||||
/// The breakdown exceeded the caller's entry limit; largest remain.
|
||||
pub truncated: bool,
|
||||
pub sub_prefixes: Vec<PrefixUsageEntry>,
|
||||
/// Sets whose cache held this bucket and prefix.
|
||||
pub sets_reporting: usize,
|
||||
pub sets_total: usize,
|
||||
/// Newest `last_update` across reporting sets, unix seconds.
|
||||
pub last_update_unix_secs: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CachedResponse {
|
||||
computed_at: std::time::Instant,
|
||||
response: Arc<BucketPrefixUsageResponse>,
|
||||
}
|
||||
|
||||
/// Cache key: (lowercased bucket, normalized prefix, max entries).
|
||||
type PrefixUsageCacheKey = (String, String, usize);
|
||||
type PrefixUsageCacheMap = Option<HashMap<PrefixUsageCacheKey, CachedResponse>>;
|
||||
|
||||
static PREFIX_USAGE_CACHE: Mutex<PrefixUsageCacheMap> = Mutex::new(None);
|
||||
|
||||
/// Drop cached results for `bucket` (empty string clears everything). Wired
|
||||
/// into the dirty-usage recording path so a write makes the next prefix
|
||||
/// query recompute instead of serving up to `CACHE_TTL` seconds of stale
|
||||
/// numbers.
|
||||
pub fn invalidate_prefix_usage_cache(bucket: &str) {
|
||||
let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
let Some(map) = guard.as_mut() else {
|
||||
return;
|
||||
};
|
||||
if bucket.is_empty() {
|
||||
map.clear();
|
||||
return;
|
||||
}
|
||||
map.retain(|(cached_bucket, ..), _| !cached_bucket.eq_ignore_ascii_case(bucket));
|
||||
}
|
||||
|
||||
/// Query prefix usage for `bucket` (arbitrary `prefix`, empty = whole
|
||||
/// bucket), merging every erasure set's own cache copy. `max_entries` bounds
|
||||
/// the sub-prefix rows (largest first).
|
||||
pub async fn bucket_prefix_usage(
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
max_entries: usize,
|
||||
) -> Result<BucketPrefixUsageResponse, ScannerError> {
|
||||
if ecstore_is_reserved_or_invalid_bucket(bucket, true) {
|
||||
return Err(ScannerError::Other(format!("invalid bucket name: {bucket}")));
|
||||
}
|
||||
let normalized_prefix = prefix.trim_matches('/').to_string();
|
||||
let cache_key = (bucket.to_ascii_lowercase(), normalized_prefix.clone(), max_entries);
|
||||
if let Some(response) = lookup_cached(&cache_key) {
|
||||
return Ok((*response).clone());
|
||||
}
|
||||
|
||||
let store = ecstore_resolve_object_store_handle()
|
||||
.ok_or_else(|| ScannerError::Other("object store is not initialized".to_string()))?;
|
||||
let response = Arc::new(compute_prefix_usage(store, bucket, &normalized_prefix, max_entries).await);
|
||||
store_cached(cache_key, response.clone());
|
||||
Ok((*response).clone())
|
||||
}
|
||||
|
||||
async fn compute_prefix_usage(
|
||||
store: Arc<EcstoreStore>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
max_entries: usize,
|
||||
) -> BucketPrefixUsageResponse {
|
||||
let sets: Vec<Arc<EcstoreSetDisks>> = store.all_set_disks();
|
||||
let sets_total = sets.len();
|
||||
let cache_name = format!("{bucket}/{DATA_USAGE_CACHE_NAME}");
|
||||
|
||||
let per_set = join_all(sets.into_iter().map(|set| {
|
||||
let cache_name = cache_name.clone();
|
||||
async move {
|
||||
let mut cache = DataUsageCache::default();
|
||||
// A set that has never scanned this bucket (or cannot be read
|
||||
// within the budget) reports nothing — the remaining sets still
|
||||
// produce a usable, flagged answer.
|
||||
let loaded = match tokio::time::timeout(PER_SET_LOAD_TIMEOUT, cache.load(set, &cache_name)).await {
|
||||
Ok(Ok(())) => cache,
|
||||
Ok(Err(err)) => {
|
||||
debug!(
|
||||
target: "rustfs::scanner::prefix_usage",
|
||||
event = EVENT_PREFIX_USAGE_CACHE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_PREFIX_USAGE,
|
||||
bucket = %bucket,
|
||||
state = "set_load_failed",
|
||||
error = %err,
|
||||
"Prefix usage set cache load failed"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
Err(_) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner::prefix_usage",
|
||||
event = EVENT_PREFIX_USAGE_CACHE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_PREFIX_USAGE,
|
||||
bucket = %bucket,
|
||||
state = "set_load_timeout",
|
||||
"Prefix usage set cache load timed out"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if loaded.info.name != bucket {
|
||||
// Empty or stale-scoped cache: this set has no data for the bucket.
|
||||
return None;
|
||||
}
|
||||
let last_update = loaded.info.last_update;
|
||||
let query = loaded.prefix_usage(bucket, prefix, max_entries);
|
||||
Some((query, last_update))
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
|
||||
let mut usage = PrefixUsageSummary::default();
|
||||
let mut sub_prefix_map: HashMap<String, PrefixUsageSummary> = HashMap::new();
|
||||
let mut sets_reporting = 0usize;
|
||||
let mut reporting_but_absent = 0usize;
|
||||
let mut any_compacted = false;
|
||||
let mut all_compacted = true;
|
||||
let mut truncated = false;
|
||||
let mut last_update: Option<SystemTime> = None;
|
||||
|
||||
for (query, set_last_update) in per_set.into_iter().flatten() {
|
||||
// last_update counts every set that has scanned the bucket, even
|
||||
// when the prefix itself is absent on that set.
|
||||
if let Some(set_last_update) = set_last_update
|
||||
&& last_update.map(|current| set_last_update > current).unwrap_or(true)
|
||||
{
|
||||
last_update = Some(set_last_update);
|
||||
}
|
||||
let Some(query) = query else {
|
||||
// The set knows the bucket but not this prefix: legitimate when
|
||||
// the prefix's objects all hash to other sets, but it means the
|
||||
// breakdown below cannot attribute that set's (zero) objects.
|
||||
reporting_but_absent += 1;
|
||||
continue;
|
||||
};
|
||||
sets_reporting += 1;
|
||||
usage.merge(&query.usage);
|
||||
if query.compacted {
|
||||
any_compacted = true;
|
||||
} else {
|
||||
all_compacted = false;
|
||||
}
|
||||
truncated |= query.truncated;
|
||||
for entry in query.sub_prefixes {
|
||||
sub_prefix_map.entry(entry.prefix).or_default().merge(&entry.usage);
|
||||
}
|
||||
}
|
||||
|
||||
let mut sub_prefixes: Vec<PrefixUsageEntry> = sub_prefix_map
|
||||
.into_iter()
|
||||
.map(|(prefix, usage)| PrefixUsageEntry { prefix, usage })
|
||||
.collect();
|
||||
sub_prefixes.sort_by(|left, right| {
|
||||
right
|
||||
.usage
|
||||
.size
|
||||
.cmp(&left.usage.size)
|
||||
.then_with(|| left.prefix.cmp(&right.prefix))
|
||||
});
|
||||
// Merged rows can exceed max_entries only when per-set truncation
|
||||
// already flagged; enforce the caller bound on the merged view too.
|
||||
if sub_prefixes.len() > max_entries {
|
||||
truncated = true;
|
||||
sub_prefixes.truncate(max_entries);
|
||||
}
|
||||
|
||||
let found = sets_reporting > 0;
|
||||
BucketPrefixUsageResponse {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: prefix.to_string(),
|
||||
usage,
|
||||
compacted: found && all_compacted,
|
||||
sub_prefixes_partial: any_compacted || reporting_but_absent > 0,
|
||||
truncated,
|
||||
sub_prefixes,
|
||||
sets_reporting,
|
||||
sets_total,
|
||||
last_update_unix_secs: last_update
|
||||
.and_then(|time| time.duration_since(SystemTime::UNIX_EPOCH).ok())
|
||||
.map(|dur| dur.as_secs()),
|
||||
}
|
||||
}
|
||||
|
||||
fn lookup_cached(key: &(String, String, usize)) -> Option<Arc<BucketPrefixUsageResponse>> {
|
||||
let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
let map = guard.as_mut()?;
|
||||
let cached = map.get(key)?;
|
||||
if cached.computed_at.elapsed() > CACHE_TTL {
|
||||
map.remove(key);
|
||||
return None;
|
||||
}
|
||||
Some(cached.response.clone())
|
||||
}
|
||||
|
||||
fn store_cached(key: (String, String, usize), response: Arc<BucketPrefixUsageResponse>) {
|
||||
let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
let map = guard.get_or_insert_with(HashMap::new);
|
||||
// Bound the cache: drop expired entries first, and if the cap is still
|
||||
// exceeded clear wholesale — the next queries recompute in milliseconds.
|
||||
if map.len() >= CACHE_MAX_ENTRIES {
|
||||
map.retain(|_, cached| cached.computed_at.elapsed() <= CACHE_TTL);
|
||||
if map.len() >= CACHE_MAX_ENTRIES {
|
||||
map.clear();
|
||||
}
|
||||
}
|
||||
map.insert(
|
||||
key,
|
||||
CachedResponse {
|
||||
computed_at: std::time::Instant::now(),
|
||||
response,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{CACHE_MAX_ENTRIES, PREFIX_USAGE_CACHE, invalidate_prefix_usage_cache, store_cached};
|
||||
use rustfs_data_usage::PrefixUsageSummary;
|
||||
|
||||
fn response(bucket: &str) -> super::BucketPrefixUsageResponse {
|
||||
super::BucketPrefixUsageResponse {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: String::new(),
|
||||
usage: PrefixUsageSummary::default(),
|
||||
compacted: false,
|
||||
sub_prefixes_partial: false,
|
||||
truncated: false,
|
||||
sub_prefixes: Vec::new(),
|
||||
sets_reporting: 1,
|
||||
sets_total: 1,
|
||||
last_update_unix_secs: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn seed(bucket: &str, prefix: &str) {
|
||||
store_cached(
|
||||
(bucket.to_ascii_lowercase(), prefix.to_string(), 10),
|
||||
std::sync::Arc::new(response(bucket)),
|
||||
);
|
||||
}
|
||||
|
||||
fn contains(bucket: &str, prefix: &str) -> bool {
|
||||
PREFIX_USAGE_CACHE
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner())
|
||||
.as_ref()
|
||||
.is_some_and(|map| map.contains_key(&(bucket.to_ascii_lowercase(), prefix.to_string(), 10)))
|
||||
}
|
||||
|
||||
/// All cache tests run inside one test to keep the process-global map
|
||||
/// free of cross-test ordering (the flake class this module avoids).
|
||||
#[test]
|
||||
fn invalidation_scopes_to_bucket_and_cache_stays_bounded() {
|
||||
invalidate_prefix_usage_cache("");
|
||||
seed("alpha", "x");
|
||||
seed("beta", "y");
|
||||
|
||||
// Case-insensitive bucket scoping.
|
||||
invalidate_prefix_usage_cache("ALPHA");
|
||||
assert!(!contains("alpha", "x"));
|
||||
assert!(contains("beta", "y"));
|
||||
|
||||
// Wholesale clear.
|
||||
invalidate_prefix_usage_cache("");
|
||||
assert!(!contains("beta", "y"));
|
||||
|
||||
// Hard cap: overflow clears rather than grows.
|
||||
for index in 0..=(CACHE_MAX_ENTRIES / 2) {
|
||||
let bucket = format!("cap-bucket-{index}");
|
||||
seed(&bucket, "a");
|
||||
seed(&bucket, "b");
|
||||
}
|
||||
let guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
let map = guard.as_ref().expect("seeded");
|
||||
assert!(map.len() <= CACHE_MAX_ENTRIES, "cache must stay bounded, got {}", map.len());
|
||||
}
|
||||
}
|
||||
@@ -12,10 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::FileType;
|
||||
use std::io::ErrorKind;
|
||||
use std::sync::{Arc, Once};
|
||||
use std::sync::{Arc, Mutex, Once};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use crate::ReplTargetSizeSummary;
|
||||
@@ -32,6 +32,7 @@ use crate::scanner_io::{
|
||||
SCANNER_SKIP_FILE_ERROR, ScannerIODisk as _, is_scanner_metadata_corrupt_error, is_scanner_metadata_transient_error,
|
||||
};
|
||||
use crate::sleeper::DynamicSleeper;
|
||||
use crate::storage_api::owner::{EcstoreEventArgs, ecstore_send_event};
|
||||
use metrics::{counter, describe_counter};
|
||||
use rustfs_common::heal_channel::{
|
||||
HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest,
|
||||
@@ -41,6 +42,7 @@ use rustfs_common::metrics::{
|
||||
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
|
||||
UpdateCurrentPathFn, current_path_updater, global_metrics,
|
||||
};
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit, trace_subscriber_count};
|
||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
|
||||
@@ -97,6 +99,101 @@ const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders
|
||||
const METRIC_SCANNER_PENDING_HEAL_PRUNE_TOTAL: &str = "rustfs_scanner_pending_heal_prune_total";
|
||||
const METRIC_SCANNER_PENDING_HEAL_MALFORMED_TOTAL: &str = "rustfs_scanner_pending_heal_malformed_total";
|
||||
const MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET: usize = 128;
|
||||
|
||||
// --- scanner excess alerts as S3 notification events (rustfs/backlog#1868) --
|
||||
//
|
||||
// The excess-versions / excess-version-size / excess-folders alerts were
|
||||
// metrics-and-logs only; subscribers (consoles, external auditors) had no way
|
||||
// to hear them. MinIO emits s3:ObjectManyVersions / s3:ObjectLargeVersions /
|
||||
// s3:PrefixManyFolders for the same conditions — RustFS carries those as
|
||||
// EventName::Scanner* with the wire names below. Without a cooldown a single
|
||||
// over-threshold object would re-emit on every scan cycle (~a minute), so
|
||||
// emissions are edge-held per (kind, bucket, object) for 24h.
|
||||
|
||||
/// `s3:Scanner:ManyVersions` (MinIO `s3:ObjectManyVersions`).
|
||||
pub const EVENT_SCANNER_MANY_VERSIONS: &str = "s3:Scanner:ManyVersions";
|
||||
/// `s3:Scanner:LargeVersions` (MinIO `s3:ObjectLargeVersions`).
|
||||
pub const EVENT_SCANNER_LARGE_VERSIONS: &str = "s3:Scanner:LargeVersions";
|
||||
/// `s3:Scanner:BigPrefix` (MinIO `s3:PrefixManyFolders`).
|
||||
pub const EVENT_SCANNER_BIG_PREFIX: &str = "s3:Scanner:BigPrefix";
|
||||
const ENV_SCANNER_ALERT_COOLDOWN_SECS: &str = "RUSTFS_SCANNER_ALERT_COOLDOWN_SECS";
|
||||
const DEFAULT_SCANNER_ALERT_COOLDOWN_SECS: u64 = 86_400;
|
||||
/// Hard cap on distinct cooldown keys; a pathological number of over-threshold
|
||||
/// objects clears the map wholesale instead of growing without bound (the
|
||||
/// worst case is one re-emission per still-hot key per scan cycle).
|
||||
const MAX_SCANNER_ALERT_COOLDOWN_KEYS: usize = 4096;
|
||||
|
||||
/// Distinct alert kinds sharing one cooldown map.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
enum ScannerAlertKind {
|
||||
ManyVersions,
|
||||
LargeVersions,
|
||||
BigPrefix,
|
||||
}
|
||||
|
||||
type ScannerAlertCooldownKey = (ScannerAlertKind, String, String);
|
||||
type ScannerAlertCooldownMap = HashMap<ScannerAlertCooldownKey, Instant>;
|
||||
|
||||
static SCANNER_ALERT_EMISSION_COOLDOWN: Mutex<Option<ScannerAlertCooldownMap>> = Mutex::new(None);
|
||||
|
||||
fn scanner_alert_cooldown() -> Duration {
|
||||
let raw = std::env::var(ENV_SCANNER_ALERT_COOLDOWN_SECS)
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok());
|
||||
Duration::from_secs(raw.unwrap_or(DEFAULT_SCANNER_ALERT_COOLDOWN_SECS))
|
||||
}
|
||||
|
||||
/// Edge-held emission gate: returns `true` (and records the cooldown) only
|
||||
/// when this (kind, bucket, object) last fired longer than the cooldown ago —
|
||||
/// or never. Metrics and logs stay level-triggered every cycle; only the
|
||||
/// notification events are held back.
|
||||
fn scanner_alert_emission_allows(kind: ScannerAlertKind, bucket: &str, object: &str, cooldown: Duration) -> bool {
|
||||
let key = (kind, bucket.to_string(), object.to_string());
|
||||
let mut guard = SCANNER_ALERT_EMISSION_COOLDOWN
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner());
|
||||
let guard = guard.get_or_insert_with(ScannerAlertCooldownMap::new);
|
||||
let now = Instant::now();
|
||||
// Expired entries leave first; the cap is still exceeded only when live
|
||||
// keys alone overflow it, in which case a wholesale clear trades one
|
||||
// extra emission per hot key for a hard memory bound.
|
||||
if guard.len() >= MAX_SCANNER_ALERT_COOLDOWN_KEYS {
|
||||
guard.retain(|_, fired_at| now.duration_since(*fired_at) < cooldown);
|
||||
if guard.len() >= MAX_SCANNER_ALERT_COOLDOWN_KEYS {
|
||||
guard.clear();
|
||||
}
|
||||
}
|
||||
match guard.get(&key) {
|
||||
Some(fired_at) if now.duration_since(*fired_at) < cooldown => false,
|
||||
_ => {
|
||||
guard.insert(key, now);
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Emit a scanner alert as an S3 notification event through the standard
|
||||
/// dispatch pipeline. Fire-and-forget: the notify layer owns delivery,
|
||||
/// retry, and target filtering; the scanner never waits on it.
|
||||
fn emit_scanner_alert_event(event_name: &str, bucket: &str, object: &str, size: i64, details: &[(&str, String)]) {
|
||||
let mut req_params = HashMap::with_capacity(details.len());
|
||||
for (key, value) in details {
|
||||
req_params.insert((*key).to_string(), value.clone());
|
||||
}
|
||||
ecstore_send_event(EcstoreEventArgs {
|
||||
event_name: event_name.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: crate::ScannerObjectInfo {
|
||||
bucket: bucket.to_string(),
|
||||
name: object.to_string(),
|
||||
size,
|
||||
..Default::default()
|
||||
},
|
||||
req_params,
|
||||
user_agent: "Scanner".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
const MAX_PENDING_SCANNER_HEALS_PER_BUCKET: usize = 10_000;
|
||||
|
||||
static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new();
|
||||
@@ -430,6 +527,113 @@ fn non_negative_i64_to_u64(value: i64) -> u64 {
|
||||
value.max(0) as u64
|
||||
}
|
||||
|
||||
fn trace_start_instant() -> Option<Instant> {
|
||||
(trace_subscriber_count() > 0).then(Instant::now)
|
||||
}
|
||||
|
||||
fn emit_scanner_folder_trace(root: &str, folder: &str, objects: u64, started_at: Option<Instant>, state: &'static str) {
|
||||
let Some(started_at) = started_at else {
|
||||
return;
|
||||
};
|
||||
|
||||
trace_emit(|| {
|
||||
let (bucket, prefix) = path2_bucket_object_with_base_path(root, folder);
|
||||
TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder)
|
||||
.with_bucket(bucket)
|
||||
.with_object(prefix)
|
||||
.with_duration(started_at.elapsed())
|
||||
.with_attr("state", state)
|
||||
.with_attr("objects", objects)
|
||||
});
|
||||
}
|
||||
|
||||
fn emit_scanner_ilm_action_trace(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
action: IlmAction,
|
||||
count: u64,
|
||||
queued: bool,
|
||||
started_at: Option<Instant>,
|
||||
) {
|
||||
let Some(started_at) = started_at else {
|
||||
return;
|
||||
};
|
||||
|
||||
let state = if queued { "queued" } else { "not_queued" };
|
||||
trace_emit(|| {
|
||||
TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction)
|
||||
.with_bucket(bucket)
|
||||
.with_object(object)
|
||||
.with_duration(started_at.elapsed())
|
||||
.with_attr("state", state)
|
||||
.with_attr("action", action.as_str())
|
||||
.with_attr("count", count)
|
||||
.with_attr("queued", queued)
|
||||
});
|
||||
}
|
||||
|
||||
struct ScannerHealCandidateTraceContext {
|
||||
bucket: String,
|
||||
object: Option<String>,
|
||||
version_id: Option<String>,
|
||||
scan_mode: Option<HealScanMode>,
|
||||
started_at: Instant,
|
||||
}
|
||||
|
||||
fn scanner_heal_candidate_trace_context(request: &HealChannelRequest) -> Option<ScannerHealCandidateTraceContext> {
|
||||
let started_at = trace_start_instant()?;
|
||||
Some(ScannerHealCandidateTraceContext {
|
||||
bucket: request.bucket.clone(),
|
||||
object: request.object_prefix.clone(),
|
||||
version_id: request.object_version_id.clone(),
|
||||
scan_mode: request.scan_mode,
|
||||
started_at,
|
||||
})
|
||||
}
|
||||
|
||||
struct ScannerHealCandidateTrace<'a> {
|
||||
candidate_type: &'static str,
|
||||
bucket: &'a str,
|
||||
object: Option<&'a str>,
|
||||
version_id: Option<&'a str>,
|
||||
priority: HealChannelPriority,
|
||||
scan_mode: Option<HealScanMode>,
|
||||
result: Result<HealAdmissionResult, &'a str>,
|
||||
started_at: Instant,
|
||||
}
|
||||
|
||||
fn emit_scanner_heal_candidate_trace(trace: ScannerHealCandidateTrace<'_>) {
|
||||
trace_emit(|| {
|
||||
let (state, admission, error) = match trace.result {
|
||||
Ok(result) if result.is_admitted() => ("admitted", describe_heal_admission(result), None),
|
||||
Ok(result) => ("not_admitted", describe_heal_admission(result), None),
|
||||
Err(error) => ("submit_failed", "channel_error".to_string(), Some(error)),
|
||||
};
|
||||
let mut event = TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerHealCandidate)
|
||||
.with_bucket(trace.bucket)
|
||||
.with_duration(trace.started_at.elapsed())
|
||||
.with_attr("state", state)
|
||||
.with_attr("candidate_type", trace.candidate_type)
|
||||
.with_attr("priority", heal_priority_label(trace.priority))
|
||||
.with_attr("admission", admission);
|
||||
|
||||
if let Some(object) = trace.object {
|
||||
event = event.with_object(object);
|
||||
}
|
||||
if let Some(version_id) = trace.version_id {
|
||||
event = event.with_attr("version_id", version_id);
|
||||
}
|
||||
if let Some(scan_mode) = trace.scan_mode {
|
||||
event = event.with_attr("scan_mode", scan_mode.as_str());
|
||||
}
|
||||
if let Some(error) = error {
|
||||
event = event.with_attr("error", error);
|
||||
}
|
||||
|
||||
event
|
||||
});
|
||||
}
|
||||
|
||||
fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary) {
|
||||
into.size = into.size.saturating_add(summary.total_size);
|
||||
into.versions = into.versions.saturating_add(summary.versions);
|
||||
@@ -677,9 +881,22 @@ async fn send_scanner_heal_request(
|
||||
request: HealChannelRequest,
|
||||
) -> Result<HealAdmissionResult, ScannerError> {
|
||||
let priority = request.priority;
|
||||
let trace_context = scanner_heal_candidate_trace_context(&request);
|
||||
match send_heal_request_with_admission(request).await {
|
||||
Ok(result) => {
|
||||
record_heal_candidate_admission(candidate_type, priority, result);
|
||||
if let Some(trace_context) = trace_context.as_ref() {
|
||||
emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace {
|
||||
candidate_type,
|
||||
bucket: &trace_context.bucket,
|
||||
object: trace_context.object.as_deref(),
|
||||
version_id: trace_context.version_id.as_deref(),
|
||||
priority,
|
||||
scan_mode: trace_context.scan_mode,
|
||||
result: Ok(result),
|
||||
started_at: trace_context.started_at,
|
||||
});
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -690,6 +907,18 @@ async fn send_scanner_heal_request(
|
||||
"result" => "channel_error".to_string()
|
||||
)
|
||||
.increment(1);
|
||||
if let Some(trace_context) = trace_context.as_ref() {
|
||||
emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace {
|
||||
candidate_type,
|
||||
bucket: &trace_context.bucket,
|
||||
object: trace_context.object.as_deref(),
|
||||
version_id: trace_context.version_id.as_deref(),
|
||||
priority,
|
||||
scan_mode: trace_context.scan_mode,
|
||||
result: Err(err.as_str()),
|
||||
started_at: trace_context.started_at,
|
||||
});
|
||||
}
|
||||
Err(ScannerError::Other(err))
|
||||
}
|
||||
}
|
||||
@@ -905,7 +1134,9 @@ impl ScannerItem {
|
||||
"Scanner lifecycle action dispatched"
|
||||
);
|
||||
let done_ilm = Metrics::time_ilm(event.action);
|
||||
let trace_started_at = trace_start_instant();
|
||||
let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
|
||||
emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at);
|
||||
if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
|
||||
done_ilm(1)();
|
||||
remaining_versions = 0;
|
||||
@@ -957,7 +1188,9 @@ impl ScannerItem {
|
||||
"Scanner lifecycle action dispatched"
|
||||
);
|
||||
let done_ilm = Metrics::time_ilm(event.action);
|
||||
let trace_started_at = trace_start_instant();
|
||||
let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
|
||||
emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at);
|
||||
if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
|
||||
done_ilm(1)();
|
||||
if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction {
|
||||
@@ -995,7 +1228,9 @@ impl ScannerItem {
|
||||
"Scanner lifecycle action dispatched"
|
||||
);
|
||||
let done_ilm = Metrics::time_ilm(event.action);
|
||||
let trace_started_at = trace_start_instant();
|
||||
let queued = apply_transition_rule(event, &LcEventSrc::Scanner, oi).await;
|
||||
emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at);
|
||||
if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
|
||||
done_ilm(1)();
|
||||
}
|
||||
@@ -1019,7 +1254,21 @@ impl ScannerItem {
|
||||
let action = event.action;
|
||||
let count = u64::try_from(to_delete_objs.len()).unwrap_or(u64::MAX);
|
||||
let done_ilm = Metrics::time_ilm(action);
|
||||
let trace_started_at = trace_start_instant();
|
||||
let queued = enqueue_runtime_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner).await;
|
||||
if let Some(trace_started_at) = trace_started_at {
|
||||
let state = if queued { "queued" } else { "not_queued" };
|
||||
trace_emit(|| {
|
||||
TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction)
|
||||
.with_bucket(self.bucket.as_str())
|
||||
.with_object(self.object_path())
|
||||
.with_duration(trace_started_at.elapsed())
|
||||
.with_attr("state", state)
|
||||
.with_attr("action", action.as_str())
|
||||
.with_attr("count", count)
|
||||
.with_attr("queued", queued)
|
||||
});
|
||||
}
|
||||
if record_scanner_ilm_action_if_queued(global_metrics(), action, count, queued) {
|
||||
done_ilm(count)();
|
||||
remaining_versions = remaining_versions.saturating_sub(noncurrent_accounting.len());
|
||||
@@ -1197,6 +1446,7 @@ impl ScannerItem {
|
||||
fn alert_excessive_versions(&self, remaining_versions: usize, cumulative_size: i64) {
|
||||
ensure_scanner_alert_metrics_registered();
|
||||
let (too_many_versions, too_large_versions) = should_alert_excessive_versions(remaining_versions, cumulative_size);
|
||||
let object_path = self.object_path();
|
||||
if too_many_versions {
|
||||
global_metrics().record_scanner_source_executed(ScannerWorkSource::Alerts, 1);
|
||||
counter!(
|
||||
@@ -1204,13 +1454,26 @@ impl ScannerItem {
|
||||
"bucket" => self.bucket.clone()
|
||||
)
|
||||
.increment(1);
|
||||
if scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, &self.bucket, &object_path, scanner_alert_cooldown())
|
||||
{
|
||||
emit_scanner_alert_event(
|
||||
EVENT_SCANNER_MANY_VERSIONS,
|
||||
&self.bucket,
|
||||
&object_path,
|
||||
cumulative_size,
|
||||
&[
|
||||
("versions", remaining_versions.to_string()),
|
||||
("threshold", scanner_excess_versions_threshold().to_string()),
|
||||
],
|
||||
);
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_ALERT_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
bucket = %self.bucket,
|
||||
object = %self.object_path(),
|
||||
object = %object_path,
|
||||
versions = remaining_versions,
|
||||
threshold = scanner_excess_versions_threshold(),
|
||||
state = "excess_versions",
|
||||
@@ -1224,13 +1487,31 @@ impl ScannerItem {
|
||||
"bucket" => self.bucket.clone()
|
||||
)
|
||||
.increment(1);
|
||||
if scanner_alert_emission_allows(
|
||||
ScannerAlertKind::LargeVersions,
|
||||
&self.bucket,
|
||||
&object_path,
|
||||
scanner_alert_cooldown(),
|
||||
) {
|
||||
emit_scanner_alert_event(
|
||||
EVENT_SCANNER_LARGE_VERSIONS,
|
||||
&self.bucket,
|
||||
&object_path,
|
||||
cumulative_size,
|
||||
&[
|
||||
("versions", remaining_versions.to_string()),
|
||||
("cumulativeSize", cumulative_size.to_string()),
|
||||
("threshold", scanner_excess_version_size_threshold().to_string()),
|
||||
],
|
||||
);
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_ALERT_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
bucket = %self.bucket,
|
||||
object = %self.object_path(),
|
||||
object = %object_path,
|
||||
versions = remaining_versions,
|
||||
cumulative_size,
|
||||
threshold = scanner_excess_version_size_threshold(),
|
||||
@@ -1611,6 +1892,15 @@ impl FolderScanner {
|
||||
"root" => self.root.clone()
|
||||
)
|
||||
.increment(1);
|
||||
if scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, &self.root, folder, scanner_alert_cooldown()) {
|
||||
emit_scanner_alert_event(
|
||||
EVENT_SCANNER_BIG_PREFIX,
|
||||
&self.root,
|
||||
folder,
|
||||
0,
|
||||
&[("folders", total_folders.to_string()), ("threshold", threshold.to_string())],
|
||||
);
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_ALERT_STATE,
|
||||
@@ -1830,6 +2120,7 @@ impl FolderScanner {
|
||||
into: &mut DataUsageEntry,
|
||||
) -> Result<(), ScannerError> {
|
||||
let done_folder = Metrics::time(Metric::ScanFolder);
|
||||
let trace_started_at = trace_start_instant();
|
||||
|
||||
if ctx.is_cancelled() {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
@@ -2895,6 +3186,8 @@ impl FolderScanner {
|
||||
}
|
||||
|
||||
done_folder();
|
||||
let scanned_objects = u64::try_from(into.objects).unwrap_or(u64::MAX);
|
||||
emit_scanner_folder_trace(&self.root, &folder.name, scanned_objects, trace_started_at, "completed");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -3076,6 +3369,90 @@ mod tests {
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
use std::sync::Mutex;
|
||||
|
||||
/// Reset the process-global alert cooldown map; test-only.
|
||||
fn reset_alert_cooldowns() {
|
||||
*SCANNER_ALERT_EMISSION_COOLDOWN
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner()) = Some(ScannerAlertCooldownMap::new());
|
||||
}
|
||||
|
||||
/// The emitted event-name strings must be exactly what `EventName`
|
||||
/// serializes, or a bucket notification subscribed to the documented name
|
||||
/// would silently never match (rustfs/backlog#1868).
|
||||
#[test]
|
||||
fn scanner_alert_wire_names_match_canonical_event_names() {
|
||||
use rustfs_s3_types::EventName;
|
||||
assert_eq!(EVENT_SCANNER_MANY_VERSIONS, EventName::ScannerManyVersions.to_string());
|
||||
assert_eq!(EVENT_SCANNER_LARGE_VERSIONS, EventName::ScannerLargeVersions.to_string());
|
||||
assert_eq!(EVENT_SCANNER_BIG_PREFIX, EventName::ScannerBigPrefix.to_string());
|
||||
}
|
||||
|
||||
fn cooldown_map_len() -> usize {
|
||||
SCANNER_ALERT_EMISSION_COOLDOWN
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner())
|
||||
.as_ref()
|
||||
.map(|map| map.len())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Backdate every recorded cooldown so the next check fires again.
|
||||
fn expire_all_alert_cooldowns(cooldown: Duration) {
|
||||
let now = Instant::now();
|
||||
let mut guard = SCANNER_ALERT_EMISSION_COOLDOWN
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner());
|
||||
if let Some(map) = guard.as_mut() {
|
||||
for fired_at in map.values_mut() {
|
||||
if let Some(expired) = now.checked_sub(cooldown + Duration::from_secs(1)) {
|
||||
*fired_at = expired;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The emission gate is the only thing standing between an over-threshold
|
||||
/// object and one S3 event per scan cycle, so its edge semantics get
|
||||
/// pinned directly. All scenarios share one #[test] because the cooldown
|
||||
/// map is process-global and parallel tests would read each other's
|
||||
/// firings.
|
||||
#[test]
|
||||
fn scanner_alert_emission_is_edge_held_per_key_and_bounded() {
|
||||
reset_alert_cooldowns();
|
||||
let cooldown = Duration::from_secs(3600);
|
||||
|
||||
// First firing allows, an immediate re-check is held.
|
||||
assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown));
|
||||
assert!(!scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown));
|
||||
|
||||
// Different kind, object, and bucket are independent keys.
|
||||
assert!(scanner_alert_emission_allows(ScannerAlertKind::LargeVersions, "bkt", "obj", cooldown));
|
||||
assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "other", cooldown));
|
||||
assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "other", "obj", cooldown));
|
||||
assert_eq!(cooldown_map_len(), 4);
|
||||
|
||||
// After the cooldown elapses the same key fires again.
|
||||
expire_all_alert_cooldowns(cooldown);
|
||||
assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown));
|
||||
|
||||
// A zero cooldown degenerates to always-emit (operators may want that).
|
||||
assert!(scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", "dir", Duration::ZERO));
|
||||
assert!(scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", "dir", Duration::ZERO));
|
||||
|
||||
// Hard bound: overflow the cap with zero-cooldown keys and confirm the
|
||||
// map clears rather than growing past it.
|
||||
reset_alert_cooldowns();
|
||||
for index in 0..=(MAX_SCANNER_ALERT_COOLDOWN_KEYS + 8) {
|
||||
let _ = scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", &format!("dir-{index}"), Duration::ZERO);
|
||||
}
|
||||
assert!(
|
||||
cooldown_map_len() <= MAX_SCANNER_ALERT_COOLDOWN_KEYS,
|
||||
"cooldown map must stay bounded, got {}",
|
||||
cooldown_map_len()
|
||||
);
|
||||
}
|
||||
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use temp_env::{with_var, with_var_unset};
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
@@ -4400,6 +4777,104 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_trace_helpers_emit_expected_events() {
|
||||
let mut trace = rustfs_common::trace_bus::subscribe_trace_events();
|
||||
|
||||
emit_scanner_folder_trace(
|
||||
"/tmp/rustfs-scanner-trace",
|
||||
"/tmp/rustfs-scanner-trace/bucket-a/folder-a",
|
||||
7,
|
||||
Some(Instant::now()),
|
||||
"completed",
|
||||
);
|
||||
let folder = recv_scanner_trace_event(
|
||||
&mut trace,
|
||||
TraceFunc::ScannerFolder,
|
||||
Some("bucket-a"),
|
||||
Some("folder-a"),
|
||||
Some("completed"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(trace_attr_string(&folder, "objects").as_deref(), Some("7"));
|
||||
|
||||
emit_scanner_ilm_action_trace("bucket-a", "object-a", IlmAction::DeleteAction, 2, true, Some(Instant::now()));
|
||||
let ilm = recv_scanner_trace_event(
|
||||
&mut trace,
|
||||
TraceFunc::ScannerIlmAction,
|
||||
Some("bucket-a"),
|
||||
Some("object-a"),
|
||||
Some("queued"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(trace_attr_string(&ilm, "action").as_deref(), Some("delete"));
|
||||
assert_eq!(trace_attr_string(&ilm, "count").as_deref(), Some("2"));
|
||||
assert_eq!(trace_attr_string(&ilm, "queued").as_deref(), Some("true"));
|
||||
|
||||
emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace {
|
||||
candidate_type: "object",
|
||||
bucket: "bucket-a",
|
||||
object: Some("object-a"),
|
||||
version_id: Some("version-a"),
|
||||
priority: HealChannelPriority::High,
|
||||
scan_mode: Some(HealScanMode::Deep),
|
||||
result: Ok(HealAdmissionResult::Merged),
|
||||
started_at: Instant::now(),
|
||||
});
|
||||
let heal_candidate = recv_scanner_trace_event(
|
||||
&mut trace,
|
||||
TraceFunc::ScannerHealCandidate,
|
||||
Some("bucket-a"),
|
||||
Some("object-a"),
|
||||
Some("admitted"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(trace_attr_string(&heal_candidate, "candidate_type").as_deref(), Some("object"));
|
||||
assert_eq!(trace_attr_string(&heal_candidate, "priority").as_deref(), Some("high"));
|
||||
assert_eq!(trace_attr_string(&heal_candidate, "scan_mode").as_deref(), Some("deep"));
|
||||
assert_eq!(trace_attr_string(&heal_candidate, "version_id").as_deref(), Some("version-a"));
|
||||
assert_eq!(trace_attr_string(&heal_candidate, "admission").as_deref(), Some("merged"));
|
||||
}
|
||||
|
||||
async fn recv_scanner_trace_event(
|
||||
trace: &mut rustfs_common::trace_bus::TraceSubscription,
|
||||
func: TraceFunc,
|
||||
bucket: Option<&str>,
|
||||
object: Option<&str>,
|
||||
state: Option<&str>,
|
||||
) -> TraceEvent {
|
||||
for _ in 0..32 {
|
||||
let event = tokio::time::timeout(Duration::from_secs(1), trace.recv())
|
||||
.await
|
||||
.expect("scanner trace event should arrive")
|
||||
.expect("trace bus should stay open");
|
||||
if event.kind == TraceKind::Scanner
|
||||
&& event.func == func
|
||||
&& event.bucket.as_deref() == bucket
|
||||
&& event.object.as_deref() == object
|
||||
&& state.is_none_or(|state| trace_attr_string(&event, "state").as_deref() == Some(state))
|
||||
{
|
||||
return (*event).clone();
|
||||
}
|
||||
}
|
||||
|
||||
panic!("expected scanner trace event {func:?} for bucket {bucket:?} object {object:?}");
|
||||
}
|
||||
|
||||
fn trace_attr_string(event: &TraceEvent, key: &str) -> Option<String> {
|
||||
event.attrs.iter().find_map(|attr| {
|
||||
if attr.key != key {
|
||||
return None;
|
||||
}
|
||||
Some(match &attr.value {
|
||||
rustfs_common::trace_bus::TraceVal::Bool(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::U64(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::I64(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::Str(value) => value.to_string(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_high_priority_heal_admission_error_contains_context() {
|
||||
let err = build_high_priority_heal_admission_error(
|
||||
|
||||
@@ -231,6 +231,10 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets));
|
||||
// A write invalidates this bucket's prefix-usage answers on the spot so
|
||||
// admin/console consumers never ride the full TTL after a change
|
||||
// (rustfs/backlog#1872).
|
||||
crate::prefix_usage::invalidate_prefix_usage_cache(bucket);
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
|
||||
@@ -78,6 +78,7 @@ pub(crate) use rustfs_ecstore::api::disk::{
|
||||
pub(crate) use rustfs_ecstore::api::error::{
|
||||
Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_event as ecstore_send_event};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::layout::{
|
||||
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
|
||||
@@ -110,8 +111,8 @@ pub(crate) mod owner {
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS,
|
||||
ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys,
|
||||
EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions,
|
||||
EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc,
|
||||
EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts,
|
||||
EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreEventArgs,
|
||||
EcstoreLcEventSrc, EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts,
|
||||
EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard,
|
||||
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi,
|
||||
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
|
||||
@@ -121,7 +122,7 @@ pub(crate) mod owner {
|
||||
ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
|
||||
ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
|
||||
ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle,
|
||||
ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
|
||||
ecstore_save_config, ecstore_send_event, scanner_replication_config_for_lifecycle_eval,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -46,6 +46,7 @@ Two rules keep this directory healthy:
|
||||
- [s3-tables-support-matrix.md](s3-tables-support-matrix.md)
|
||||
- [minio-rustfs-router-compatibility.md](minio-rustfs-router-compatibility.md)
|
||||
- [minio-file-format-compat.md](minio-file-format-compat.md)
|
||||
- [minio-scanner-compat.md](minio-scanner-compat.md) — MinIO data-scanner parity: cycle/heal/ILM/usage gaps and improvement contracts
|
||||
|
||||
## Inventories & baselines (snapshots that feed migration work)
|
||||
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
# MinIO ↔ RustFS Data Scanner Comparison
|
||||
|
||||
Assesses how closely the RustFS background data scanner matches MinIO's
|
||||
`cmd/data-scanner.go` implementation: cycle leadership, namespace walk,
|
||||
usage accounting, ILM admission, replication repair, heal/bitrot selection,
|
||||
alerts, and operator surfaces. This is a **durable gap analysis**. It changes
|
||||
no scanner code. Every claim cites the code that backs it.
|
||||
|
||||
MinIO sources below are the public `minio/minio` `master` tree as of
|
||||
2026-08-18 (`cmd/data-scanner.go`, `cmd/erasure.go`, `cmd/xl-storage.go`,
|
||||
`cmd/data-usage-cache.go`, `internal/config/scanner/scanner.go`). They are
|
||||
not files in this repository.
|
||||
|
||||
Operator runtime knobs already documented here stay in
|
||||
[scanner-runtime-controls.md](../operations/scanner-runtime-controls.md).
|
||||
This page does not duplicate that runbook.
|
||||
|
||||
## Executive Summary
|
||||
|
||||
The two scanners share the same skeleton. Both run one cluster-wide leader
|
||||
loop, persist a cycle counter in `.bloomcycle.bin`, walk folders with a
|
||||
1-in-16 compacted-leaf schedule, select objects for heal with a 1-in-1024
|
||||
hash, compact usage trees at the same child thresholds, evaluate ILM through
|
||||
a lifecycle evaluator, enqueue replication heals, emit excess-version and
|
||||
excess-folder alerts, and throttle with a proportional sleeper.
|
||||
|
||||
The remaining gaps are not "the scanner is missing". They are **heal
|
||||
fidelity**, **cross-set scheduling**, **on-disk cache interop**, and
|
||||
**operator/notification wire names**. The highest-severity difference is
|
||||
that MinIO heals a selected object inline and then cleans abandoned parts,
|
||||
while RustFS admits a low-priority heal request that can be dropped and
|
||||
never calls `check_abandoned_parts` on that path.
|
||||
|
||||
RustFS also has several load-bearing additions MinIO does not: dirty-usage
|
||||
fast wake, cycle budgets, leader-epoch fencing, remote NS-scanner protocol
|
||||
v6, checkpoint resume, and clean-idle backoff. Those should be preserved.
|
||||
|
||||
| Area | Verdict | Why it matters |
|
||||
|---|---|---|
|
||||
| Cycle / leadership / bitrot mode | Close | Same `.bloomcycle.bin` counter, same deep-scan window of `healObjectSelectProb` cycles. |
|
||||
| Folder walk, compact, 1/16 + 1/1024 selection | Close | Constants and `mod` / `modAlt` schedule match. |
|
||||
| ILM eval + expiry/transition enqueue | Close | Same action set; RustFS additionally gates metrics on queue admission. |
|
||||
| Bucket replication repair | Close | Both call `queueReplicationHeal` / `queue_heal`. |
|
||||
| Scanner-selected object heal | **Gap** | MinIO `HealObject` is synchronous; RustFS async admission can drop the check. |
|
||||
| Abandoned-part cleanup on selected objects | **Gap** | MinIO calls `CheckAbandonedParts` after heal; RustFS scanner path does not. |
|
||||
| Bucket order across erasure sets | **Gap** | MinIO shuffles per set; RustFS is deterministic dirty→new→existing. |
|
||||
| `.usage-cache.bin` bytes | **Incompatible** | MinIO is zstd+msgp v8; RustFS is raw MessagePack. Reconstructable, not reusable. |
|
||||
| Excess-folder default | Differs | MinIO `50000`; RustFS `65538`. |
|
||||
| Alert event names | Differs | MinIO `s3:ObjectManyVersions`; RustFS `s3:Scanner:ManyVersions`. |
|
||||
|
||||
---
|
||||
|
||||
## Architecture Overlay
|
||||
|
||||
Both stacks are `init → leader lock → cycle → NSScanner → per-set disk walk →
|
||||
scanDataFolder → applyActions`.
|
||||
|
||||
```text
|
||||
initDataScanner / init_data_scanner
|
||||
│
|
||||
▼
|
||||
runDataScanner / run_data_scanner (cluster leader lock)
|
||||
│
|
||||
├─ load .bloomcycle.bin cycle state
|
||||
├─ getCycleScanMode (Normal vs Deep bitrot)
|
||||
└─ NSScanner(wantCycle, scanMode)
|
||||
│
|
||||
▼
|
||||
per erasure set (MinIO: er.nsScanner; RustFS: scanner_io)
|
||||
│
|
||||
├─ load set .usage-cache.bin
|
||||
├─ bucket order (new first, then existing)
|
||||
└─ per disk: NSScanner / scan_data_folder
|
||||
│
|
||||
├─ lifecycle + replication config
|
||||
├─ folder walk, compact, 1/16 skip
|
||||
├─ getSize → applyActions
|
||||
│ ├─ ILM eval
|
||||
│ ├─ heal selected versions
|
||||
│ └─ healReplication
|
||||
└─ abandoned-children heal walk
|
||||
```
|
||||
|
||||
| Stage | MinIO | RustFS |
|
||||
|---|---|---|
|
||||
| Startup | `initDataScanner` goroutine, random sleep ≥ 1s between `runDataScanner` calls | `init_data_scanner` in `crates/scanner/src/scanner.rs`; optional cold-cache / replication skip of start delay |
|
||||
| Leader | `globalLeaderLock.GetLock` (blocks) | `leader.lock` write lock with timeout; contended cycle returns and retries |
|
||||
| Cycle persist | LE `uint64` + msgp `currentScannerCycle` at `.bloomcycle.bin` | LE `uint64` + optional `RSCYC001` epoch header + msgpack `CurrentCycle` |
|
||||
| Set walk | `erasureObjects.nsScanner` in MinIO `cmd/erasure.go` | `crates/scanner/src/scanner_io.rs` |
|
||||
| Folder walk | `folderScanner.scanFolder` | `crates/scanner/src/scanner_folder.rs` |
|
||||
| Object actions | `scannerItem.applyActions` | `ScannerItem::apply_actions` |
|
||||
| Usage publish | `storeDataUsageInBackend` ← `.usage.json` | `store_data_usage_in_backend` ← `.usage.v2.json` (legacy `.usage.json` read-only; `scanner-usage-v2` in [compat-cleanup-register.md](compat-cleanup-register.md)) |
|
||||
|
||||
---
|
||||
|
||||
## What Already Matches
|
||||
|
||||
These are not gaps. Treat regressions here as MinIO-parity bugs.
|
||||
|
||||
### Cycle constants and heal selection
|
||||
|
||||
| Constant | MinIO | RustFS | Evidence |
|
||||
|---|---|---|---|
|
||||
| Folder sleep quantum | 1ms | sleeper `MIN_SLEEP` 1ms | MinIO `dataScannerSleepPerFolder`; `crates/scanner/src/sleeper.rs` |
|
||||
| Compacted-leaf visit period | 16 | 16 (`RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES`) | MinIO `dataUsageUpdateDirCycles`; `crates/scanner/src/scanner_folder.rs` `DATA_USAGE_UPDATE_DIR_CYCLES` |
|
||||
| Heal object probability | 1024 | 1024 (`RUSTFS_HEAL_OBJECT_SELECT_PROB`) | MinIO `healObjectSelectProb`; `DEFAULT_HEAL_OBJECT_SELECT_PROB` |
|
||||
| Compact least objects | 500 | 500 | both `dataScannerCompactLeastObject` / `DATA_SCANNER_COMPACT_LEAST_OBJECT` |
|
||||
| Compact at children | 10000 | 10000 | both |
|
||||
| Compact at folders | 2500 | 2500 | `children/4` |
|
||||
| Force compact folders | 250000 | 250000 | both |
|
||||
| Start delay default | 1 minute | speed-preset derived (default 1 minute) | MinIO `dataScannerStartDelay`; RustFS speed preset |
|
||||
| Excess versions | 100 | 100 | MinIO `scannerExcessObjectVersions`; `DEFAULT_SCANNER_ALERT_EXCESS_VERSIONS` |
|
||||
| Excess version size | 1 TiB | 1 TiB | MinIO `scannerExcessObjectVersionsTotalSize`; `DEFAULT_SCANNER_ALERT_EXCESS_VERSION_SIZE` |
|
||||
|
||||
Folder skip uses `hash.mod(nextCycle, 16)`. Object heal uses
|
||||
`hash.modAlt(nextCycle/div, healObjectSelect/div)`. Compacted folders raise
|
||||
`objectHealProbDiv` to 16 so the 1/1024 overall probability still holds.
|
||||
RustFS copies this in `scan_folder` (`mod_` + `object_heal_prob_div`).
|
||||
|
||||
Path identity is the cleaned path string, not a digest. MinIO `hashPath` is
|
||||
`path.Clean`; RustFS `hash_path` in `crates/data-usage/src/data_usage.rs`
|
||||
cleans the same way. `xxhash` is only used in MinIO `mod` / `modAlt`.
|
||||
|
||||
### Bitrot cycle window
|
||||
|
||||
Both enter Deep scan when:
|
||||
|
||||
- bitrot cycle is `0` (always deep), or
|
||||
- `current - bitrotStartCycle < healObjectSelectProb`, or
|
||||
- wall time since `bitrotStartTime` exceeds the configured bitrot cycle.
|
||||
|
||||
MinIO: `getCycleScanMode` in `cmd/data-scanner.go`. RustFS:
|
||||
`get_cycle_scan_mode` in `crates/scanner/src/scanner.rs`. Both persist
|
||||
`backgroundHealInfo` / `BackgroundHealInfo` and skip it on single-disk
|
||||
(`globalIsErasureSD` / `scanner_is_erasure_sd`).
|
||||
|
||||
### ILM action coverage
|
||||
|
||||
`apply_actions` covers the same action enum MinIO does: delete, delete
|
||||
version, delete restored, delete-all, del-marker-delete-all, transition,
|
||||
and none (heal + replication). Evaluator is constructed with lock
|
||||
retention and replication config. Free versions are swept through
|
||||
`enqueue_free_version` / `enqueue_runtime_free_version`. Noncurrent
|
||||
versions batch through `enqueueNoncurrentVersions` /
|
||||
`enqueue_runtime_newer_noncurrent`.
|
||||
|
||||
### Speed presets
|
||||
|
||||
`fastest` / `fast` / `default` / `slow` / `slowest` map to the same delay,
|
||||
max-wait, and cycle defaults MinIO `LookupConfig` uses (`0/0/1s`,
|
||||
`1/100ms/1m`, `2/1s/1m`, `10/15s/1m`, `100/15s/30m`).
|
||||
|
||||
### Abandoned-children folder heal
|
||||
|
||||
When a previously cached child is missing from the current directory listing,
|
||||
both scanners quorum-list the prefix and enqueue bucket/object heals. RustFS
|
||||
keeps this walk in `scan_folder` after the new/existing folder scans.
|
||||
|
||||
### Read-path heal still exists
|
||||
|
||||
MinIO also heals from GET/HEAD and MRF; the scanner is not the only heal
|
||||
source (MinIO PR 18050). RustFS GET decode errors enqueue
|
||||
`HealRequestSource::ReadRepair` in `crates/ecstore/src/set_disk/read.rs`.
|
||||
Scanner-heal gaps therefore delay *background* repair, not all repair.
|
||||
|
||||
---
|
||||
|
||||
## Gaps
|
||||
|
||||
Severity is the operator-visible failure if the gap is left as-is.
|
||||
|
||||
### G1 — Scanner object heal is async and droppable (high)
|
||||
|
||||
MinIO `applyHealing` calls `ObjectLayer.HealObject` and waits. The folder
|
||||
walker then treats `getSize` as having already healed the object
|
||||
(`cmd/data-scanner.go`, comment on `abandonedChildren` deletion). After a
|
||||
successful heal it always runs `CheckAbandonedParts` with
|
||||
`Remove: healDeleteDangling`.
|
||||
|
||||
RustFS `heal_actions` always returns the original `actual_size` and, when
|
||||
heal is selected, calls `enqueue_heal` → `send_heal_request_with_admission`
|
||||
at `HealChannelPriority::Low`. `Full` and `Dropped` admissions are logged
|
||||
and skipped. `RUSTFS_SCANNER_INLINE_HEAL_ENABLE` only warns
|
||||
`inline_heal_rollback_unsupported` (`warn_inline_heal_compat_requested` in
|
||||
`crates/scanner/src/scanner_folder.rs`).
|
||||
|
||||
**Failure:** a 1/1024-selected object with a missing shard can remain
|
||||
unhealed for many more cycles if the heal channel is full. Bitrot Deep
|
||||
selection has the same drop window. Usage accounting is unchanged by heal
|
||||
outcome, so a reconstructed size never replaces the pre-heal size in that
|
||||
cycle.
|
||||
|
||||
**Do not "fix" this by making every scanner heal inline on the walk
|
||||
goroutine.** MinIO can afford that because `HealObject` is the storage
|
||||
layer. RustFS already has a heal worker pool and admission. The missing
|
||||
contract is: scanner-selected heals must be durable (pending_heals retry)
|
||||
and must not be silent-dropped without a later guaranteed retry.
|
||||
|
||||
Pending heals already exist for some metadata/abandoned-child failures
|
||||
(`PendingScannerHeal` in the usage cache). Object-selection heals that hit
|
||||
`HealAdmissionResult::Full` do not currently join that retry list.
|
||||
|
||||
### G2 — No `CheckAbandonedParts` on the scanner object-heal path (high)
|
||||
|
||||
`check_abandoned_parts` is implemented on the store
|
||||
(`crates/ecstore/src/store/heal.rs`, `crates/ecstore/src/set_disk/ops/heal.rs`)
|
||||
and is in the object API. The scanner never calls it. The heal task
|
||||
processor (`crates/heal/src/heal/task.rs`) also does not call it after a
|
||||
scanner-originated `heal_object`.
|
||||
|
||||
MinIO records this as `scannerMetricCleanAbandoned`. RustFS defines
|
||||
`Metric::CleanAbandoned` in `crates/common/src/metrics.rs` but the scanner
|
||||
crate never records it.
|
||||
|
||||
**Failure:** leftover `part.N` files after a successful object heal stay
|
||||
until some other heal path notices them. Disk usage and bitrot surface
|
||||
area remain inflated.
|
||||
|
||||
### G3 — Erasure-set bucket order is not shuffled (medium)
|
||||
|
||||
MinIO `nsScanner` builds a permutation of buckets, emits *new* buckets
|
||||
(absent from the old cache) first in that random order, then existing
|
||||
buckets in that random order. Comment: otherwise the same buckets are
|
||||
scanned across every erasure set at the same time.
|
||||
|
||||
RustFS `bucket_usage_scan_order` in `crates/scanner/src/scanner_io.rs` is
|
||||
deterministic: dirty buckets, then cache-miss (new) buckets, then
|
||||
cache-hit buckets, preserving `ListBuckets` order.
|
||||
|
||||
Dirty-first is a RustFS improvement (MinIO has no dirty-usage wake). The
|
||||
gap is the *existing* bucket tail: under many buckets and several sets,
|
||||
RustFS lock-steps ILM/heal/replication load onto the same prefixes.
|
||||
|
||||
### G4 — Alert event names and audit channel (medium)
|
||||
|
||||
MinIO emits `event.ObjectManyVersions`, `event.ObjectLargeVersions`,
|
||||
`event.PrefixManyFolders`, plus `auditLogInternal` events
|
||||
`scanner:manyversions` / `scanner:largeversions` / `scanner:manyprefixes`.
|
||||
|
||||
RustFS emits `s3:Scanner:ManyVersions`, `s3:Scanner:LargeVersions`,
|
||||
`s3:Scanner:BigPrefix` (`EVENT_SCANNER_*` in
|
||||
`crates/scanner/src/scanner_folder.rs`; wire names in
|
||||
`crates/s3-types/src/event_name.rs`). Notifications are edge-held 24h
|
||||
(MinIO re-emits every cycle). There is no scanner audit-log counterpart.
|
||||
|
||||
**Failure:** notification destinations configured for MinIO event names
|
||||
miss RustFS scanner alerts. Audit pipelines that key on
|
||||
`scanner:manyversions` see nothing.
|
||||
|
||||
### G5 — Excess-folder default differs (low)
|
||||
|
||||
MinIO `scannerExcessFolders` default is `50000`
|
||||
(`internal/config/scanner/scanner.go`). RustFS
|
||||
`DEFAULT_SCANNER_ALERT_EXCESS_FOLDERS` is `65538`
|
||||
(`crates/config/src/constants/scanner.rs`).
|
||||
|
||||
**Failure:** the same prefix is silent on MinIO and noisy on RustFS (or
|
||||
the reverse if an operator copied MinIO runbooks).
|
||||
|
||||
### G6 — `.usage-cache.bin` is not MinIO-readable (medium for migration, low otherwise)
|
||||
|
||||
MinIO writes one version byte (`dataUsageCacheVerCurrent = 8`) plus zstd
|
||||
plus msgp (`cmd/data-usage-cache.go` `serializeTo`). RustFS
|
||||
`DataUsageCache::save_inner` writes uncompressed `rmp_serde` with no
|
||||
version byte (`crates/scanner/src/data_usage_define.rs`).
|
||||
|
||||
Both use the same object name `.usage-cache.bin` and a `.bkp` sibling.
|
||||
A MinIO disk set attached to RustFS rebuilds the tree on first scan; the
|
||||
bytes are not reused. The inverse is also true.
|
||||
|
||||
`.bloomcycle.bin` is closer: both start with a little-endian `u64` next
|
||||
cycle. RustFS additionally writes `RSCYC001` + leader epoch when fencing
|
||||
is active, and still reads a bare 8-byte or 8-byte+msgpack MinIO blob
|
||||
(`decode_scanner_cycle_state`). MinIO cannot consume the fenced form.
|
||||
|
||||
Cluster usage snapshots diverge on purpose: MinIO `.usage.json`, RustFS
|
||||
authoritative `.usage.v2.json`. That is already a compat register item,
|
||||
not a scanner-logic bug.
|
||||
|
||||
### G7 — Heal-selected usage size ignores heal result (low)
|
||||
|
||||
MinIO `healActions` replaces `actualSz` with `HealObject`'s
|
||||
`res.ObjectSize` when positive. RustFS `heal_actions` always returns
|
||||
`actual_size`. Wrong sizes persist until the *next* cycle that both
|
||||
selects the object *and* observes healed metadata.
|
||||
|
||||
This is secondary to G1: without a completed heal there is no new size.
|
||||
|
||||
### G8 — Operator metric names and `mc admin scanner info` (low)
|
||||
|
||||
MinIO `scannerMetric.String()` is PascalCase (`ReadMetadata`, `ScanObject`,
|
||||
`ILM`). RustFS `Metric::as_str` is snake_case (`read_metadata`,
|
||||
`scan_object`, `ilm`). `mc admin scanner info` against a RustFS
|
||||
`ScannerMetrics.life_time_ops` map will not match MinIO dashboard keys.
|
||||
|
||||
RustFS exposes a richer `/v3/scanner/status` (freshness, runtime config
|
||||
sources, cycle schedule, admission). That is the supported operator
|
||||
surface; MinIO `mc` scanner info is not a compatibility target unless
|
||||
explicitly added.
|
||||
|
||||
### G9 — Unversioned replication heal gate (low)
|
||||
|
||||
MinIO `healReplication` returns immediately when `oi.VersionID == ""`.
|
||||
RustFS allows the call when the object is a delete marker or has a
|
||||
version-purge status even if `version_id` is none/nil
|
||||
(`ScannerItem::heal_replication`). This is likely *more* correct for
|
||||
purge/delete-marker repair on unversioned-looking entries, but it is a
|
||||
behavioral difference worth pinning with a test rather than copying
|
||||
MinIO's empty-VersionID skip blindly.
|
||||
|
||||
---
|
||||
|
||||
## RustFS-Only Behavior To Keep
|
||||
|
||||
These are not MinIO gaps. Removing them to "match MinIO" would be a
|
||||
regression.
|
||||
|
||||
| Addition | Where | Why keep it |
|
||||
|---|---|---|
|
||||
| Dirty-usage fast wake + superseded retry (5s base) | `crates/scanner/src/scanner.rs`, `scanner_io.rs` `record_dirty_usage_bucket` | Quota/usage lag after write bursts; MinIO waits a full cycle. |
|
||||
| Cycle object/directory/runtime budgets | `crates/scanner/src/scanner_budget.rs` | Bounds scanner blast radius; MinIO only sleeps. |
|
||||
| Leader epoch + CAS persist | `encode_scanner_cycle_state` | Split-brain cycle counters after lock loss. |
|
||||
| Remote NS-scanner protocol v6 | `crates/scanner/src/remote_scanner.rs`; compat `ns-scanner-rpc-v3` | Distributed disk walks with fencing. |
|
||||
| Scan checkpoints / resume hints | `DataUsageScanCheckpoint` | Partial cycles after budget cancel. |
|
||||
| Clean-idle backoff (single-disk / erasure) | `ScannerCleanIdleBackoff` | Stops minute-cadence full walks on idle namespaces. |
|
||||
| Heal/replication admission metrics | `HealAdmissionResult`, `ScannerReplicationQueueAdmission` | Makes G1 observable; MinIO has no equivalent queue. |
|
||||
| Alert emission cooldown | 24h edge-hold | Avoids notification storms MinIO still has. |
|
||||
|
||||
---
|
||||
|
||||
## Improvement Workstreams
|
||||
|
||||
These are contracts, not a checklist. Each workstream is independently
|
||||
shippable. Do not couple them into one "make scanner like MinIO" rewrite.
|
||||
|
||||
### W1 — Durable scanner-selected heal (closes G1, G7)
|
||||
|
||||
**Invariant:** if an object is selected by `modAlt` in a cycle that
|
||||
`should_heal()`, that object/version is either healed, recorded in
|
||||
`pending_heals` for a later cycle, or the cycle is marked incomplete for
|
||||
heal work. Silent `Full`/`Dropped` is not a success.
|
||||
|
||||
**Shape:** keep the heal channel. On `Full`/`Dropped`, persist
|
||||
`PendingScannerHeal` (object, version, scan mode) the same way abandoned
|
||||
metadata heals already persist. Retry at high or at least non-droppable
|
||||
priority next cycle. When a heal *completes*, optionally replace the
|
||||
accounted size with the healed size (G7).
|
||||
|
||||
**Do not:** call `HealObject` inline from `scan_folder` as a default. The
|
||||
unsupported `RUSTFS_SCANNER_INLINE_HEAL_ENABLE` warning exists because
|
||||
that rollback fights the worker pool. An opt-in inline path is only
|
||||
justified if a measured admission-drop rate stays high after durable
|
||||
retry.
|
||||
|
||||
**Tests:** (a) selected object missing one shard, heal channel full →
|
||||
pending_heals non-empty, next cycle heals it; (b) Deep mode + recent
|
||||
mtime stays Normal (existing cooldown); (c) usage size updates only after
|
||||
heal success; (d) revert of pending_heals-on-drop fails the test.
|
||||
|
||||
### W2 — Abandoned-part cleanup after scanner object heal (closes G2)
|
||||
|
||||
**Invariant:** a scanner-selected object heal that succeeds (or that the
|
||||
heal worker reports as already consistent) runs `check_abandoned_parts`
|
||||
with dangling removal, matching MinIO `healDeleteDangling = true`.
|
||||
|
||||
**Shape:** call it from the heal worker when `source == Scanner`, not
|
||||
from the folder walk. That keeps IO off the scanner hot path. Record
|
||||
`Metric::CleanAbandoned` so last-minute scanner metrics are not a dead
|
||||
enum.
|
||||
|
||||
**Tests:** object with an extra `part.N` after a valid heal → part
|
||||
removed; dry-run heal does not delete (existing set_disk tests stay
|
||||
authoritative); `CleanAbandoned` lifetime counter increments.
|
||||
|
||||
### W3 — Per-set shuffle of existing buckets (closes G3)
|
||||
|
||||
**Invariant:** dirty and new buckets still go first (RustFS dirty-usage
|
||||
contract). The existing-bucket tail is shuffled per erasure set per
|
||||
cycle so sets do not scan the same prefix concurrently.
|
||||
|
||||
**Shape:** smallest change is `bucket_usage_scan_order` taking a
|
||||
per-set RNG seed (cycle + pool + set). Do not shuffle dirty buckets;
|
||||
that would delay quota/usage repair.
|
||||
|
||||
**Tests:** two sets, three existing buckets, same cycle → different
|
||||
existing tails; dirty bucket always index 0.
|
||||
|
||||
### W4 — Notification and audit aliases (closes G4, optionally G5)
|
||||
|
||||
**Invariant:** a destination subscribed to MinIO names
|
||||
`s3:ObjectManyVersions` / `s3:ObjectLargeVersions` /
|
||||
`s3:PrefixManyFolders` receives RustFS scanner alerts. Keep the current
|
||||
`s3:Scanner:*` names as aliases, not replacements, until clients migrate.
|
||||
|
||||
**Shape:** dual-name parse in `crates/s3-types/src/event_name.rs` (already
|
||||
comments "corresponding to Go") plus dual emit, or a compatibility
|
||||
mapping at notify dispatch. Audit events are optional and should reuse
|
||||
the existing audit pipeline rather than a scanner-specific logger.
|
||||
|
||||
Align `DEFAULT_SCANNER_ALERT_EXCESS_FOLDERS` to `50000` only with a
|
||||
release note; 65538 is not a bug, it is a silent default drift.
|
||||
|
||||
### W5 — Cache-format interop (closes G6 only if migration requires it)
|
||||
|
||||
**Invariant for RustFS-only clusters:** none. Rebuilding `.usage-cache.bin`
|
||||
on first scan is acceptable.
|
||||
|
||||
**Invariant if MinIO disk import is a product goal:** either detect MinIO
|
||||
v8 zstd+msgp and ignore/rebuild, or implement a one-shot importer.
|
||||
Writing MinIO-shaped cache from RustFS is not required for serving
|
||||
objects.
|
||||
|
||||
Document in operations that `.usage-cache.bin` is not a migration
|
||||
artifact. `.bloomcycle.bin` 8-byte prefix already round-trips.
|
||||
|
||||
### W6 — Operator surface (closes G8)
|
||||
|
||||
Keep `/v3/scanner/status` as the source of truth. If `mc admin scanner
|
||||
info` support is required, add a madmin-shaped projection with PascalCase
|
||||
`life_time_ops` keys *in addition to* snake_case, behind a documented
|
||||
compat flag. Do not rename RustFS metrics; Prometheus and status JSON
|
||||
already use snake_case.
|
||||
|
||||
---
|
||||
|
||||
## Suggested Verification (when a workstream ships)
|
||||
|
||||
Scanner changes are high-risk under AGENTS.md (lifecycle/tiering,
|
||||
heal, S3-visible usage). A workstream PR should run:
|
||||
|
||||
- `cargo fmt --all --check`
|
||||
- `cargo test -p rustfs-scanner` (and heal tests for W2)
|
||||
- the crate's lifecycle integration tests when ILM admission changes
|
||||
- `make doc-paths-check` if this file's citations move
|
||||
|
||||
Do not run `make pre-pr` for documentation-only edits of this page.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
RustFS:
|
||||
|
||||
- `crates/scanner/src/scanner.rs` — leader loop, cycle fencing, bitrot mode
|
||||
- `crates/scanner/src/scanner_folder.rs` — folder walk, ILM, heal, alerts
|
||||
- `crates/scanner/src/scanner_io.rs` — NSScanner, bucket order, dirty usage
|
||||
- `crates/scanner/src/scanner_budget.rs` — cycle budgets
|
||||
- `crates/scanner/src/sleeper.rs` — proportional throttle
|
||||
- `crates/scanner/src/data_usage_define.rs` — cache persist
|
||||
- `crates/scanner/src/runtime_config.rs` — env/config resolution
|
||||
- `crates/config/src/constants/scanner.rs` — defaults
|
||||
- `crates/common/src/metrics.rs` — metric enum (MinIO-shaped)
|
||||
- `rustfs/src/admin/handlers/scanner.rs` — `/v3/scanner/status`
|
||||
- [compat-cleanup-register.md](compat-cleanup-register.md) — `scanner-usage-v2`, `ns-scanner-rpc-v3`
|
||||
|
||||
MinIO (`minio/minio` master, 2026-08-18):
|
||||
|
||||
- `cmd/data-scanner.go` — init/run, applyActions, healReplication, sleeper
|
||||
- `cmd/data-scanner-metric.go` — metric enum and `mc` report
|
||||
- `cmd/erasure.go` — `nsScanner` shuffle and per-disk walk
|
||||
- `cmd/xl-storage.go` — disk `NSScanner` / getSize
|
||||
- `cmd/data-usage-cache.go` — hash mod, zstd+msgp cache
|
||||
- `cmd/data-usage.go` — `.usage.json` / `.bloomcycle.bin` names
|
||||
- `internal/config/scanner/scanner.go` — speed presets and alert defaults
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone.
|
||||
|
||||
For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first.
|
||||
For how the Vault backends authenticate (static token, AppRole, Kubernetes, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first.
|
||||
|
||||
## Backend comparison
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# Scanner Excess Alerts: Metrics, S3 Events, and Thresholds
|
||||
|
||||
> 中文版:[scanner-excess-alerts_zh.md](scanner-excess-alerts_zh.md)
|
||||
|
||||
Date: 2026-08-18 (rustfs/backlog#1868 / HS-04; includes the HS-15 threshold-delta notes)
|
||||
|
||||
The background scanner detects three classes of "excess" conditions while it walks buckets and surfaces them as alerts. This page documents each alert's trigger condition, the subscribable S3 event, the cooldown semantics, and the threshold differences versus MinIO — for operators debugging alerts and for event consumers wiring up subscriptions.
|
||||
|
||||
## The three alerts
|
||||
|
||||
| Alert | Trigger (per scan cycle) | Metric | S3 event (RustFS wire name) | MinIO event name |
|
||||
|---|---|---|---|---|
|
||||
| Excess versions | Retained versions of one object ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` |
|
||||
| Excess version size | Cumulative bytes of all versions of one object ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` |
|
||||
| Excess folders | Direct subfolders of one directory > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` |
|
||||
|
||||
Subscribe like any bucket notification: configure a notification on the target bucket with the RustFS wire name above (or the `s3:Scanner:*` wildcard). Events carry `UserAgent: Scanner` as their origin marker, and `req_params` holds the observed value and the threshold (`versions` / `cumulativeSize` / `folders` / `threshold`), so consumers can judge severity directly.
|
||||
|
||||
## Metrics and events fire on different cadences
|
||||
|
||||
- **Metrics and structured logs are level-triggered**: as long as the object stays over the threshold, every scan cycle counts and logs it (default cycle ≈ 60s; see `scanner:speed`).
|
||||
- **S3 events are edge-triggered with a cooldown**: the same (alert kind, bucket, object) emits at most once per cooldown window — 24 hours by default (`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`; set it to 0 to emit every cycle). When the window lapses and the object is still over the threshold, the event fires again. The cooldown table lives in process memory with a 4096-entry hard cap; on overflow it is cleared and rebuilt (worst case: one extra emission per still-hot key).
|
||||
- A process restart resets the cooldown (every still-over-threshold object emits once more after a restart) — deliberately: restarts usually accompany incident response, and the re-emission buys visibility.
|
||||
|
||||
## Threshold defaults and the MinIO deltas (HS-15)
|
||||
|
||||
| Config key | ENV | RustFS default | MinIO default | Notes |
|
||||
|---|---|---|---|---|
|
||||
| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | Identical |
|
||||
| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | Same order of magnitude; different unit basis (TiB vs TB) |
|
||||
| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **Deliberate divergence**: 65538 tolerates the Proxmox Backup Server chunk layout (65536 chunks per directory plus the directory's own entries); MinIO's 50000 would fire continuously for PBS users. Set it to 50000 explicitly to match MinIO behavior |
|
||||
|
||||
All three keys accept both env and admin config (`PUT /rustfs/admin/v3/config`, `scanner` subsystem); hot updates take effect immediately.
|
||||
|
||||
## Why the event names are mapped
|
||||
|
||||
RustFS's event enum (`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`) keeps the repo's established `s3:Scanner:*` wire names (literally different from MinIO's `s3:ObjectManyVersions`; the enum comments preserve the mapping). Subscribers should use the RustFS wire names in this page. If you need MinIO-literal compatibility, map the names on the console/consumer side — do not change the published wire names.
|
||||
@@ -0,0 +1,37 @@
|
||||
# Scanner 超限告警:指标、S3 事件与阈值
|
||||
|
||||
> English version: [scanner-excess-alerts.md](scanner-excess-alerts.md)
|
||||
|
||||
日期:2026-08-18(rustfs/backlog#1868 / HS-04,含 HS-15 阈值差异说明)
|
||||
|
||||
后台 scanner 在扫描过程中检测三类"超限"状态并对外告警。本文说明每类告警的触发条件、可订阅的 S3 事件、冷却语义,以及与 MinIO 的阈值差异,供运维排障与事件消费方对接。
|
||||
|
||||
## 三类告警
|
||||
|
||||
| 告警 | 触发条件(任一扫描周期) | 指标 | S3 事件(RustFS wire 名) | MinIO 对应事件名 |
|
||||
|---|---|---|---|---|
|
||||
| 版本数超限 | 单对象保留版本数 ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` |
|
||||
| 版本总大小超限 | 单对象全部版本累计字节 ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` |
|
||||
| 子目录数超限 | 单目录直接子目录数 > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` |
|
||||
|
||||
订阅方式与普通桶通知一致:对目标桶配置 notification,事件名填上表 RustFS wire 名(或通配 `s3:Scanner:*`)。事件以 `UserAgent: Scanner` 标记来源,`req_params` 携带实际值与阈值(`versions` / `cumulativeSize` / `folders` / `threshold`),便于消费方直接判断严重程度。
|
||||
|
||||
## 指标与事件的触发节奏不同
|
||||
|
||||
- **指标与结构化日志是电平触发**:只要对象仍在阈值之上,每个扫描周期都会计数/打日志(默认周期约 60s,见 `scanner:speed`)。
|
||||
- **S3 事件是边沿触发 + 冷却**:同一 (告警类型, 桶, 对象) 在冷却窗口内只发一次,默认 24 小时(`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`,设 0 表示每周期都发)。窗口过后对象仍超限会再次发出。冷却表在进程内有 4096 条硬顶,超限清空重建(最坏情况是每个仍超限的 key 多发一次)。
|
||||
- 进程重启会重置冷却(重启后每个仍超限的对象会再发一次)——这是有意为之:重启常伴随排障,重发提供可见性。
|
||||
|
||||
## 阈值默认值与 MinIO 差异(HS-15)
|
||||
|
||||
| 配置键 | ENV | RustFS 默认 | MinIO 默认 | 差异说明 |
|
||||
|---|---|---|---|---|
|
||||
| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | 一致 |
|
||||
| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | 语义同量级,单位口径不同(TiB vs TB) |
|
||||
| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **有意差异**:65538 兼容 Proxmox Backup Server 的 chunk 布局(每目录 65536 个 chunk + 目录自身条目),按 MinIO 的 50000 会对 PBS 用户持续误报。如需与 MinIO 行为一致可显式配置为 50000 |
|
||||
|
||||
三个键均支持 env 与 admin config(`PUT /rustfs/admin/v3/config` 的 `scanner` 子系统)双通道,热更新即时生效。
|
||||
|
||||
## 事件名映射的由来
|
||||
|
||||
RustFS 的事件枚举(`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`)沿用仓库既有 wire 名 `s3:Scanner:*`(与 MinIO 的 `s3:ObjectManyVersions` 字面不同,枚举注释中保留了映射关系)。订阅方应以本文的 RustFS wire 名为准;如需 MinIO 字面兼容,请在 console/消费侧做名称映射,不要修改已发布的 wire 名。
|
||||
@@ -8,9 +8,12 @@ This runbook covers how the RustFS Vault KMS backends (KV2 and Transit) authenti
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Static token | `Token` | Whatever the operator provisioned; RustFS never renews it | None | Development; short-lived experiments |
|
||||
| AppRole | `AppRole` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production without a Vault Agent sidecar |
|
||||
| Kubernetes | `Kubernetes` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production on Kubernetes, with no credential to distribute |
|
||||
| Agent token file | `TokenFile` | Owned by Vault Agent; RustFS only re-reads the sink file | File re-read once per poll interval | Production with a Vault Agent (or equivalent) managing auth |
|
||||
|
||||
Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` or an explicit `RUSTFS_KMS_VAULT_TOKEN` is rejected at startup with a configuration error, because the effective identity would be ambiguous.
|
||||
Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with any other method, or `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID`, is rejected at startup with a configuration error, because the effective identity would be ambiguous. A leftover `RUSTFS_KMS_VAULT_TOKEN` alongside a configured login method is tolerated and ignored, so a stale variable cannot silently downgrade the identity.
|
||||
|
||||
All of these are read the same way whether the service is started with `RUSTFS_KMS_ENABLE=true` or configured later through `POST /rustfs/admin/v3/kms/configure`.
|
||||
|
||||
The default `dev-token` fallback for `RUSTFS_KMS_VAULT_TOKEN` is rejected outside explicit development mode (`RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS=true`), as are plain-HTTP Vault addresses and disabled TLS verification.
|
||||
|
||||
@@ -56,7 +59,44 @@ Deliver the SecretID out of band — a secrets-manager-mounted file, an init-con
|
||||
|
||||
The secret_id file is re-read on every login attempt, so rotating the SecretID is a two-step operation with no restart: generate a new SecretID (`vault write -f auth/approle/role/rustfs-kms/secret-id`), atomically replace the file, then revoke the old SecretID accessor. The already-issued token keeps renewing; the new SecretID is only needed at the next full re-login.
|
||||
|
||||
An empty or missing secret_id file fails the login attempt immediately (no Vault round trip) and is retried on the normal refresh cadence, so repairing the file heals the backend without a restart.
|
||||
An empty or missing secret_id file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a file missing at boot is recovered by restarting the process, not by an in-process retry. Once RustFS is running, the same failure is retried on the normal refresh cadence, so repairing the file mid-run heals the backend without a restart.
|
||||
|
||||
## Kubernetes authentication
|
||||
|
||||
On Kubernetes this is the method to prefer: the pod's own ServiceAccount is the identity, so there is no credential to distribute, rotate, or leak into a Secret.
|
||||
|
||||
### Vault-side setup
|
||||
|
||||
```shell
|
||||
vault auth enable kubernetes
|
||||
|
||||
vault write auth/kubernetes/config \
|
||||
kubernetes_host="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT"
|
||||
|
||||
vault write auth/kubernetes/role/rustfs \
|
||||
bound_service_account_names=rustfs \
|
||||
bound_service_account_namespaces=rustfs \
|
||||
token_policies=rustfs-kms \
|
||||
token_ttl=1h
|
||||
```
|
||||
|
||||
As with AppRole, keep `token_ttl` comfortably above the RustFS per-attempt timeout (default 30s).
|
||||
|
||||
### RustFS configuration
|
||||
|
||||
```shell
|
||||
RUSTFS_KMS_BACKEND=vault-transit # or "vault" for the KV2 backend
|
||||
RUSTFS_KMS_VAULT_ADDRESS=https://vault.vault.svc.cluster.local:8200
|
||||
RUSTFS_KMS_VAULT_KUBERNETES_ROLE=rustfs
|
||||
# Optional, defaults to "kubernetes":
|
||||
# RUSTFS_KMS_VAULT_KUBERNETES_MOUNT=kubernetes
|
||||
# Optional, defaults to the kubelet's projected token path:
|
||||
# RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
```
|
||||
|
||||
RustFS logs in at startup and renews the token at half its TTL, falling back to a fresh login exactly as AppRole does. The ServiceAccount token is re-read from disk on every login rather than cached, so a projected token the kubelet rotates is picked up without a restart.
|
||||
|
||||
A missing or empty token file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a token projected late during a slow pod start is recovered by the pod restart loop, not by an in-process retry. Once RustFS is running, a token file that goes missing or turns empty is retried on the normal refresh cadence and heals the backend on its own.
|
||||
|
||||
## Vault Agent token file
|
||||
|
||||
@@ -101,13 +141,13 @@ If the agent stops refreshing the file that is fine — RustFS re-reads the same
|
||||
|
||||
## Fail-closed window
|
||||
|
||||
For lease-bound credentials (AppRole tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side.
|
||||
For lease-bound credentials (AppRole and Kubernetes tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side.
|
||||
|
||||
- Default window: one per-attempt timeout (`RUSTFS_KMS_TIMEOUT_SECS`, default 30s) — a request issued now can legitimately stay in flight that long, so the token must outlive it.
|
||||
- Override: `refresh_safety_window_secs` on the `AppRole` or `TokenFile` auth configuration.
|
||||
- Override: `refresh_safety_window_secs` on the `AppRole`, `Kubernetes` or `TokenFile` auth configuration.
|
||||
- Static tokens never trip the window: they carry no lease and are assumed valid until Vault says otherwise.
|
||||
|
||||
The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole) or two poll intervals (token file).
|
||||
The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole, Kubernetes) or two poll intervals (token file).
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
@@ -117,6 +157,8 @@ The window is a symptom threshold, not the fault itself: by the time it trips, r
|
||||
| Renewal succeeded but re-login later fails | `Vault token renewal failed; falling back to a fresh login` followed by login errors | SecretID expired/revoked or AppRole role changed; rotate the secret_id file |
|
||||
| Token file mode error at startup or during polls | `has insecure permissions` in the error | Fix the sink `mode` (0600) and the file owner; the next poll heals the provider |
|
||||
| Token file missing/empty errors | `Failed to read Vault token file` / `token file ... is empty` | Vault Agent down or sink misconfigured; restart the agent, the next poll heals the provider |
|
||||
| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, token file |
|
||||
| Kubernetes login fails with a permission error | `Vault Kubernetes login failed` | The pod's ServiceAccount is not in the role's `bound_service_account_names`/`_namespaces`, or `auth/kubernetes/config` names the wrong API server |
|
||||
| Kubernetes ServiceAccount token errors | `Failed to read Kubernetes ServiceAccount token` / `ServiceAccount token ... is empty` | The token is not projected into the pod (check `automountServiceAccountToken` and the volume mount); the next refresh cycle heals the provider |
|
||||
| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, Kubernetes, token file |
|
||||
|
||||
When diagnosing, confirm three clocks/lifetimes in order: the Vault token TTL (`vault token lookup` with the token's accessor), the RustFS refresh cadence (half TTL or the poll interval), and the fail-closed window. The renewal task logs every failed cycle, so a silent gap in warnings combined with `CredentialsUnavailable` errors points at the process clock or a paused runtime rather than Vault.
|
||||
|
||||
@@ -59,7 +59,7 @@
|
||||
{
|
||||
default = rustPlatform.buildRustPackage {
|
||||
pname = "rustfs";
|
||||
version = "1.0.0-rc.3";
|
||||
version = "1.0.0-rc.2";
|
||||
|
||||
src = ./.;
|
||||
|
||||
|
||||
@@ -273,6 +273,10 @@ uer. `ClusterIssuer` or `Issuer`. |
|
||||
| gatewayApi.listeners.http.port| int | `8000` | Gateway API http listener port. |
|
||||
| gatewayApi.listeners.https.name | string | `websecure` | Gateway API https listener name. |
|
||||
| gatewayApi.listeners.https.port| int | `8443` | Gateway API https listener port. |
|
||||
| gatewayApi.listeners.tls.enabled | bool | `false` | Enable a TLS passthrough listener and generate a TLSRoute. |
|
||||
| gatewayApi.listeners.tls.name | string | `tls` | Gateway API TLS passthrough listener name. |
|
||||
| gatewayApi.listeners.tls.port | int | `443` | Gateway API TLS passthrough listener port. |
|
||||
| gatewayApi.listeners.tls.backendPort | int | `null` | Backend service port that terminates TLS; defaults to the console port. |
|
||||
| gatewayApi.hostname | string | Hostname to access RustFS via gateway api. |
|
||||
| gatewayApi.secretName | string | Secret tls to via RustFS using HTTPS. |
|
||||
| gatewayApi.existingGateway.name | string | `""` | The existing gateway name, instead of creating a new one. |
|
||||
@@ -447,6 +451,8 @@ rustfs-route ["example.rustfs.com"] 172m
|
||||
|
||||
Then, via RustFS instance via `https://example.rustfs.com` or `http://example.rustfs.com`.
|
||||
|
||||
For end-to-end encryption, set `gatewayApi.listeners.tls.enabled` to `true`. The chart then adds a `TLS` listener with `tls.mode: Passthrough` to the `Gateway` and generates a `TLSRoute` that forwards the encrypted stream to the RustFS service, where TLS is terminated on the backend side. Note that backend TLS termination must be configured on RustFS itself (for example `RUSTFS_TLS_PATH` pointing to server certificates), and the installed Gateway API CRDs must include `TLSRoute`.
|
||||
|
||||
# Uninstall
|
||||
|
||||
Uninstalling the rustfs installation with command,
|
||||
|
||||
@@ -2,8 +2,8 @@ apiVersion: v2
|
||||
name: rustfs
|
||||
description: RustFS helm chart to deploy RustFS on kubernetes cluster.
|
||||
type: application
|
||||
version: "1.0.0-rc.3"
|
||||
appVersion: "1.0.0-rc.3"
|
||||
version: "1.0.0-rc.2"
|
||||
appVersion: "1.0.0-rc.2"
|
||||
home: https://rustfs.com
|
||||
icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg
|
||||
maintainers:
|
||||
|
||||
@@ -26,5 +26,15 @@ spec:
|
||||
- name: {{ include "rustfs.fullname" $ }}-tls
|
||||
kind: Secret
|
||||
{{- end }}
|
||||
{{- if .tls.enabled }}
|
||||
- name: {{ .tls.name }}
|
||||
port: {{ .tls.port }}
|
||||
protocol: TLS
|
||||
tls:
|
||||
mode: Passthrough
|
||||
allowedRoutes:
|
||||
namespaces:
|
||||
from: Same
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
{{- if and .Values.gatewayApi.enabled .Values.gatewayApi.listeners.tls.enabled }}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: TLSRoute
|
||||
metadata:
|
||||
name: {{ include "rustfs.fullname" . }}-tlsroute
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
parentRefs:
|
||||
{{- if .Values.gatewayApi.existingGateway.name }}
|
||||
- name: {{ .Values.gatewayApi.existingGateway.name }}
|
||||
{{- if .Values.gatewayApi.existingGateway.namespace }}
|
||||
namespace: {{ .Values.gatewayApi.existingGateway.namespace }}
|
||||
{{- end }}
|
||||
sectionName: {{ .Values.gatewayApi.listeners.tls.name }}
|
||||
{{- else }}
|
||||
- name: {{ include "rustfs.fullname" $ }}-gateway
|
||||
sectionName: {{ .Values.gatewayApi.listeners.tls.name }}
|
||||
{{- end }}
|
||||
hostnames:
|
||||
- {{ .Values.gatewayApi.hostname }}
|
||||
rules:
|
||||
- backendRefs:
|
||||
- name: {{ include "rustfs.fullname" . }}-svc
|
||||
port: {{ .Values.gatewayApi.listeners.tls.backendPort | default .Values.service.console.port }}
|
||||
{{- end }}
|
||||
@@ -369,6 +369,12 @@ gatewayApi:
|
||||
https:
|
||||
name: websecure
|
||||
port: 8443
|
||||
tls: # Optional TLS passthrough listener; renders a TLSRoute so TLS terminates at the RustFS backend.
|
||||
enabled: false
|
||||
name: tls
|
||||
port: 443
|
||||
# Service port that terminates TLS on the backend; defaults to the console port.
|
||||
backendPort: null
|
||||
hostname: example.rustfs.com
|
||||
httpToHttpsRedirect: true
|
||||
existingGateway:
|
||||
|
||||
+2
-5
@@ -1,9 +1,9 @@
|
||||
%global _enable_debug_packages 0
|
||||
%global _empty_manifest_terminate_build 0
|
||||
%global prerelease rc.3
|
||||
%global prerelease rc.2
|
||||
Name: rustfs
|
||||
Version: 1.0.0
|
||||
Release: rc.3
|
||||
Release: rc.2
|
||||
Summary: High-performance distributed object storage for MinIO alternative
|
||||
|
||||
License: Apache-2.0
|
||||
@@ -58,9 +58,6 @@ install %_builddir/%{name}-%{version}-%{prerelease}/target/%_arch/%_arch-unknown
|
||||
%_bindir/rustfs
|
||||
|
||||
%changelog
|
||||
* Mon Aug 17 2026 唐小鸭 <tangtang1251@qq.com>
|
||||
- Update RPM package to RustFS 1.0.0-rc.3
|
||||
|
||||
* Fri Aug 14 2026 overtrue <anzhengchao@gmail.com>
|
||||
- Update RPM package to RustFS 1.0.0-rc.2
|
||||
|
||||
|
||||
@@ -286,6 +286,7 @@ fn auth_method_kind(auth: &VaultAuthMethod) -> String {
|
||||
match auth {
|
||||
VaultAuthMethod::Token { .. } => "token",
|
||||
VaultAuthMethod::AppRole { .. } => "approle",
|
||||
VaultAuthMethod::Kubernetes { .. } => "kubernetes",
|
||||
VaultAuthMethod::TokenFile { .. } => "token-file",
|
||||
}
|
||||
.to_string()
|
||||
@@ -484,7 +485,10 @@ fn business_trust_root_secrets(config: &KmsConfig) -> Vec<Zeroizing<String>> {
|
||||
secrets.push(Zeroizing::new(role_id.clone()));
|
||||
secrets.push(Zeroizing::new(secret_id.clone()));
|
||||
}
|
||||
VaultAuthMethod::TokenFile { .. } => {}
|
||||
// Kubernetes and TokenFile hold no inline plaintext credential: the
|
||||
// ServiceAccount token and the agent-managed token live in files, and
|
||||
// the role names a Vault binding rather than half a credential pair.
|
||||
VaultAuthMethod::Kubernetes { .. } | VaultAuthMethod::TokenFile { .. } => {}
|
||||
};
|
||||
|
||||
match &config.backend_config {
|
||||
|
||||
@@ -64,6 +64,7 @@ mod target_descriptor;
|
||||
pub mod tier;
|
||||
pub mod tls_debug;
|
||||
pub mod trace;
|
||||
pub mod usage_prefix;
|
||||
pub mod user;
|
||||
pub mod user_iam;
|
||||
pub mod user_lifecycle;
|
||||
|
||||
@@ -23,18 +23,24 @@ use futures::{Stream, StreamExt};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
use regex::Regex;
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceKind, TraceVal, subscribe_trace_events};
|
||||
use rustfs_madmin::service_commands::ServiceTraceOpts;
|
||||
use rustfs_madmin::trace::TraceType;
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use s3s::header::CONTENT_TYPE;
|
||||
use s3s::stream::{ByteStream, DynByteStream};
|
||||
use s3s::{Body, S3Request, S3Response, S3Result, StdError, s3_error};
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tracing::error;
|
||||
use url::form_urlencoded;
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ProfileStatus {
|
||||
@@ -206,16 +212,164 @@ impl Stream for TraceStream {
|
||||
|
||||
impl ByteStream for TraceStream {}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct TraceKindFilter {
|
||||
heal: bool,
|
||||
scanner: bool,
|
||||
}
|
||||
|
||||
impl TraceKindFilter {
|
||||
const ALL_SUPPORTED: Self = Self {
|
||||
heal: true,
|
||||
scanner: true,
|
||||
};
|
||||
|
||||
fn from_request(uri: &hyper::Uri, trace_types: TraceType) -> S3Result<Self> {
|
||||
let mut has_kind = false;
|
||||
let mut filter = Self {
|
||||
heal: false,
|
||||
scanner: false,
|
||||
};
|
||||
|
||||
for (key, value) in trace_query_pairs(uri) {
|
||||
if key != "kind" {
|
||||
continue;
|
||||
}
|
||||
has_kind = true;
|
||||
for item in value.split(',') {
|
||||
match item.trim().to_ascii_lowercase().as_str() {
|
||||
"heal" | "healing" => filter.heal = true,
|
||||
"scanner" => filter.scanner = true,
|
||||
"all" => return Ok(Self::ALL_SUPPORTED),
|
||||
_ => return Err(s3_error!(InvalidRequest, "invalid trace kind")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if has_kind {
|
||||
return Ok(filter);
|
||||
}
|
||||
|
||||
if trace_types.mask() == 0 || trace_query_flag(uri, "all") {
|
||||
return Ok(Self::ALL_SUPPORTED);
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
heal: trace_types.overlaps(&TraceType::HEALING),
|
||||
scanner: trace_types.overlaps(&TraceType::SCANNER),
|
||||
})
|
||||
}
|
||||
|
||||
const fn matches(self, kind: TraceKind) -> bool {
|
||||
match kind {
|
||||
TraceKind::Heal => self.heal,
|
||||
TraceKind::Scanner => self.scanner,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TraceStreamFilter {
|
||||
kinds: TraceKindFilter,
|
||||
regex: Option<Regex>,
|
||||
threshold: Duration,
|
||||
}
|
||||
|
||||
impl TraceStreamFilter {
|
||||
fn from_request(uri: &hyper::Uri, opts: &ServiceTraceOpts) -> S3Result<Self> {
|
||||
if opts.only_errors() {
|
||||
return Err(s3_error!(
|
||||
InvalidRequest,
|
||||
"trace error-only filter is not supported for heal/scanner trace"
|
||||
));
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
kinds: TraceKindFilter::from_request(uri, opts.trace_types())?,
|
||||
regex: trace_regex_filter(uri)?,
|
||||
threshold: opts.threshold(),
|
||||
})
|
||||
}
|
||||
|
||||
fn matches_kind(&self, kind: TraceKind) -> bool {
|
||||
self.kinds.matches(kind)
|
||||
}
|
||||
|
||||
fn matches_record(&self, record: &TraceWireRecord) -> bool {
|
||||
record.duration >= self.threshold && self.regex.as_ref().is_none_or(|regex| record.matches_regex(regex))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct TraceWireRecord {
|
||||
#[serde(rename = "type")]
|
||||
trace_type: u64,
|
||||
#[serde(rename = "nodename")]
|
||||
node_name: String,
|
||||
#[serde(rename = "funcname")]
|
||||
func_name: String,
|
||||
#[serde(rename = "time")]
|
||||
time: String,
|
||||
#[serde(rename = "path")]
|
||||
path: String,
|
||||
#[serde(rename = "dur")]
|
||||
duration: Duration,
|
||||
#[serde(rename = "bytes", skip_serializing_if = "Option::is_none")]
|
||||
bytes: Option<i64>,
|
||||
#[serde(rename = "msg", skip_serializing_if = "Option::is_none")]
|
||||
message: Option<String>,
|
||||
#[serde(rename = "custom", skip_serializing_if = "Option::is_none")]
|
||||
custom: Option<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
impl TraceWireRecord {
|
||||
fn from_event(node_name: &str, event: &TraceEvent) -> Self {
|
||||
Self {
|
||||
trace_type: trace_type_mask(event.kind),
|
||||
node_name: node_name.to_owned(),
|
||||
func_name: event.func.as_str().to_owned(),
|
||||
time: trace_time_string(event.time),
|
||||
path: trace_path(event),
|
||||
duration: event.duration,
|
||||
bytes: trace_bytes(event.bytes),
|
||||
message: None,
|
||||
custom: trace_custom_attrs(event),
|
||||
}
|
||||
}
|
||||
|
||||
fn dropped(node_name: &str, dropped: u64) -> Self {
|
||||
let mut custom = HashMap::new();
|
||||
custom.insert("dropped_events".to_string(), dropped.to_string());
|
||||
|
||||
Self {
|
||||
trace_type: 0,
|
||||
node_name: node_name.to_owned(),
|
||||
func_name: "trace.Dropped".to_string(),
|
||||
time: trace_time_string(SystemTime::now()),
|
||||
path: String::new(),
|
||||
duration: Duration::ZERO,
|
||||
bytes: None,
|
||||
message: Some("trace subscriber lagged".to_string()),
|
||||
custom: Some(custom),
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_regex(&self, regex: &Regex) -> bool {
|
||||
regex.is_match(&self.func_name)
|
||||
|| regex.is_match(&self.path)
|
||||
|| self.message.as_ref().is_some_and(|message| regex.is_match(message))
|
||||
|| self
|
||||
.custom
|
||||
.as_ref()
|
||||
.is_some_and(|custom| custom.iter().any(|(key, value)| regex.is_match(key) || regex.is_match(value)))
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /v3/trace` — stream real-time server trace events.
|
||||
///
|
||||
/// RustFS emits diagnostics through the `tracing` pipeline but does not expose
|
||||
/// an in-process subscriber that can fan trace events out to an admin client
|
||||
/// (there is no request-trace broadcast channel). Rather than return an opaque
|
||||
/// `501` — which would make `mc admin trace` fail to connect — this honors the
|
||||
/// streaming NDJSON contract: it validates the requested trace filters, opens
|
||||
/// the stream, emits a single capability record explaining that live tracing is
|
||||
/// not wired, then holds the connection open with keep-alives. No fabricated
|
||||
/// trace records are ever sent.
|
||||
/// RustFS currently publishes heal and scanner diagnostics through the common
|
||||
/// trace bus. The admin endpoint exposes those events as MinIO-shaped NDJSON
|
||||
/// records while keeping unsupported trace classes filtered out.
|
||||
pub struct TraceHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -228,24 +382,13 @@ impl Operation for TraceHandler {
|
||||
let mut opts = ServiceTraceOpts::default();
|
||||
opts.parse_params(&req.uri)
|
||||
.map_err(|_| s3_error!(InvalidRequest, "invalid trace parameters"))?;
|
||||
let filter = TraceStreamFilter::from_request(&req.uri, &opts)?;
|
||||
|
||||
let node_name = sysinfo::System::host_name().unwrap_or_else(|| "rustfs".to_string());
|
||||
let (tx, rx) = mpsc::channel::<Result<Bytes, StdError>>(8);
|
||||
let mut subscription = subscribe_trace_events();
|
||||
let (tx, rx) = mpsc::channel::<Result<Bytes, StdError>>(64);
|
||||
|
||||
spawn_traced(async move {
|
||||
let notice = serde_json::json!({
|
||||
"nodename": node_name,
|
||||
"funcname": "admin.Trace",
|
||||
"msg": "RustFS does not expose an in-process trace-event subscriber; live tracing is not yet available",
|
||||
"err": "trace_streaming_unsupported",
|
||||
});
|
||||
if let Ok(mut encoded) = serde_json::to_vec(¬ice) {
|
||||
encoded.push(b'\n');
|
||||
if tx.send(Ok(Bytes::from(encoded))).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let mut ticker = tokio::time::interval(Duration::from_secs(15));
|
||||
ticker.tick().await;
|
||||
loop {
|
||||
@@ -256,6 +399,26 @@ impl Operation for TraceHandler {
|
||||
break;
|
||||
}
|
||||
}
|
||||
received = subscription.recv() => {
|
||||
match received {
|
||||
Ok(event) => {
|
||||
if !filter.matches_kind(event.kind) {
|
||||
continue;
|
||||
}
|
||||
let record = TraceWireRecord::from_event(&node_name, &event);
|
||||
if filter.matches_record(&record) && send_trace_record(&tx, &record).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(dropped)) => {
|
||||
let record = TraceWireRecord::dropped(&node_name, dropped);
|
||||
if send_trace_record(&tx, &record).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -269,17 +432,115 @@ impl Operation for TraceHandler {
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_trace_record(tx: &mpsc::Sender<Result<Bytes, StdError>>, record: &TraceWireRecord) -> Result<(), ()> {
|
||||
let Some(encoded) = encode_ndjson(record) else {
|
||||
return Ok(());
|
||||
};
|
||||
tx.send(Ok(encoded)).await.map_err(|_| ())
|
||||
}
|
||||
|
||||
fn encode_ndjson(value: &impl Serialize) -> Option<Bytes> {
|
||||
let mut encoded = serde_json::to_vec(value).ok()?;
|
||||
encoded.push(b'\n');
|
||||
Some(Bytes::from(encoded))
|
||||
}
|
||||
|
||||
fn trace_query_pairs(uri: &hyper::Uri) -> impl Iterator<Item = (String, String)> + '_ {
|
||||
uri.query()
|
||||
.into_iter()
|
||||
.flat_map(|query| form_urlencoded::parse(query.as_bytes()))
|
||||
.map(|(key, value)| (key.into_owned(), value.into_owned()))
|
||||
}
|
||||
|
||||
fn trace_query_flag(uri: &hyper::Uri, flag: &str) -> bool {
|
||||
trace_query_pairs(uri).any(|(key, value)| key == flag && value == "true")
|
||||
}
|
||||
|
||||
fn trace_regex_filter(uri: &hyper::Uri) -> S3Result<Option<Regex>> {
|
||||
trace_query_pairs(uri)
|
||||
.find_map(|(key, value)| {
|
||||
if key == "filter" && !value.is_empty() {
|
||||
Some(value)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.map(|pattern| Regex::new(&pattern).map_err(|_| s3_error!(InvalidRequest, "invalid trace filter")))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn trace_type_mask(kind: TraceKind) -> u64 {
|
||||
match kind {
|
||||
TraceKind::Heal => TraceType::HEALING.mask(),
|
||||
TraceKind::Scanner => TraceType::SCANNER.mask(),
|
||||
}
|
||||
}
|
||||
|
||||
fn trace_time_string(time: SystemTime) -> String {
|
||||
match OffsetDateTime::from(time).format(&Rfc3339) {
|
||||
Ok(value) => value,
|
||||
Err(_) => "1970-01-01T00:00:00Z".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn trace_path(event: &TraceEvent) -> String {
|
||||
match (event.bucket.as_deref(), event.object.as_deref()) {
|
||||
(Some(bucket), Some(object)) if !object.is_empty() => format!("{bucket}/{object}"),
|
||||
(Some(bucket), _) => bucket.to_owned(),
|
||||
(None, Some(object)) => object.to_owned(),
|
||||
(None, None) => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn trace_bytes(bytes: u64) -> Option<i64> {
|
||||
if bytes == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
match i64::try_from(bytes) {
|
||||
Ok(value) => Some(value),
|
||||
Err(_) => Some(i64::MAX),
|
||||
}
|
||||
}
|
||||
|
||||
fn trace_custom_attrs(event: &TraceEvent) -> Option<HashMap<String, String>> {
|
||||
if event.attrs.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(
|
||||
event
|
||||
.attrs
|
||||
.iter()
|
||||
.map(|attr| (attr.key.to_string(), trace_value_string(&attr.value)))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
fn trace_value_string(value: &TraceVal) -> String {
|
||||
match value {
|
||||
TraceVal::Bool(value) => value.to_string(),
|
||||
TraceVal::U64(value) => value.to_string(),
|
||||
TraceVal::I64(value) => value.to_string(),
|
||||
TraceVal::Str(value) => value.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
ProfileControlHandler, ProfileHandler, ProfileStatusHandler, ProfilingDownloadHandler, ProfilingStartHandler,
|
||||
TraceHandler,
|
||||
TraceHandler, TraceKindFilter, TraceStreamFilter, TraceWireRecord,
|
||||
};
|
||||
use crate::admin::router::Operation;
|
||||
use http::{Extensions, HeaderMap, Uri};
|
||||
use hyper::Method;
|
||||
use matchit::Params;
|
||||
use s3s::{Body, S3ErrorCode, S3Request};
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind};
|
||||
use rustfs_madmin::service_commands::ServiceTraceOpts;
|
||||
use rustfs_madmin::trace::TraceType;
|
||||
use s3s::{Body, S3ErrorCode, S3Request, S3Result};
|
||||
use std::time::{Duration, UNIX_EPOCH};
|
||||
|
||||
fn build_profile_request(uri: &'static str) -> S3Request<Body> {
|
||||
S3Request {
|
||||
@@ -295,6 +556,13 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn build_trace_stream_filter(uri: &'static str) -> S3Result<TraceStreamFilter> {
|
||||
let uri = Uri::from_static(uri);
|
||||
let mut opts = ServiceTraceOpts::default();
|
||||
opts.parse_params(&uri).expect("test trace params should parse");
|
||||
TraceStreamFilter::from_request(&uri, &opts)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn profile_handler_rejects_missing_credentials() {
|
||||
let result = ProfileHandler {}
|
||||
@@ -358,4 +626,108 @@ mod tests {
|
||||
.expect_err("trace must reject anonymous requests");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_kind_filter_supports_kind_query() {
|
||||
let uri = Uri::from_static("/rustfs/admin/v3/trace?kind=heal");
|
||||
let filter = TraceKindFilter::from_request(&uri, TraceType::default()).expect("kind filter should parse");
|
||||
|
||||
assert!(filter.matches(TraceKind::Heal));
|
||||
assert!(!filter.matches(TraceKind::Scanner));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_kind_filter_defaults_to_supported_events_without_type_flags() {
|
||||
let uri = Uri::from_static("/rustfs/admin/v3/trace");
|
||||
let filter = TraceKindFilter::from_request(&uri, TraceType::default()).expect("empty filter should parse");
|
||||
|
||||
assert!(filter.matches(TraceKind::Heal));
|
||||
assert!(filter.matches(TraceKind::Scanner));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_kind_filter_rejects_unknown_kind() {
|
||||
let uri = Uri::from_static("/rustfs/admin/v3/trace?kind=s3");
|
||||
let err = TraceKindFilter::from_request(&uri, TraceType::default()).expect_err("unknown kind should fail");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_stream_filter_matches_regex_against_path_and_attrs() {
|
||||
let filter = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&filter=data/.%2Bxl.meta")
|
||||
.expect("regex filter should parse");
|
||||
let event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject)
|
||||
.with_bucket("data")
|
||||
.with_object("dir/xl.meta")
|
||||
.with_attr("dry_run", true);
|
||||
let record = TraceWireRecord::from_event("node-a", &event);
|
||||
|
||||
assert!(filter.matches_kind(event.kind));
|
||||
assert!(filter.matches_record(&record));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_stream_filter_rejects_invalid_regex() {
|
||||
let err = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&filter=[").expect_err("invalid regex should fail");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_stream_filter_applies_threshold() {
|
||||
let filter =
|
||||
build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&threshold=10ms").expect("threshold should parse");
|
||||
let short = TraceWireRecord::from_event(
|
||||
"node-a",
|
||||
&TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject).with_duration(Duration::from_millis(9)),
|
||||
);
|
||||
let long = TraceWireRecord::from_event(
|
||||
"node-a",
|
||||
&TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject).with_duration(Duration::from_millis(10)),
|
||||
);
|
||||
|
||||
assert!(!filter.matches_record(&short));
|
||||
assert!(filter.matches_record(&long));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_stream_filter_rejects_error_only_filter() {
|
||||
let err = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&err=true").expect_err("err filter should fail");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_wire_record_contains_madmin_trace_fields() {
|
||||
let event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject)
|
||||
.with_bucket("bucket")
|
||||
.with_object("object")
|
||||
.with_duration(Duration::from_millis(3))
|
||||
.with_bytes(17)
|
||||
.with_attr("dry", true);
|
||||
let mut record = TraceWireRecord::from_event("node-a", &event);
|
||||
record.time = "1970-01-01T00:00:00Z".to_string();
|
||||
|
||||
let value = serde_json::to_value(&record).expect("trace record should serialize");
|
||||
|
||||
assert_eq!(value["type"], TraceType::HEALING.mask());
|
||||
assert_eq!(value["nodename"], "node-a");
|
||||
assert_eq!(value["funcname"], "heal.Object");
|
||||
assert_eq!(value["time"], "1970-01-01T00:00:00Z");
|
||||
assert_eq!(value["path"], "bucket/object");
|
||||
assert_eq!(value["bytes"], 17);
|
||||
assert_eq!(value["custom"]["dry"], "true");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_wire_record_formats_epoch_time() {
|
||||
let event = TraceEvent {
|
||||
time: UNIX_EPOCH,
|
||||
..TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder)
|
||||
};
|
||||
let record = TraceWireRecord::from_event("node-a", &event);
|
||||
|
||||
assert_eq!(record.time, "1970-01-01T00:00:00Z");
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -417,6 +417,13 @@ struct SystemAdminDiscovery {
|
||||
struct ServerInfoResponse {
|
||||
info: InfoMessage,
|
||||
admin_discovery: SystemAdminDiscovery,
|
||||
/// Startup bitrot algorithm self-test outcome (rustfs/backlog#1873):
|
||||
/// `passed` (algorithms verified at boot), `failed` (a drifted hash
|
||||
/// implementation — the process is serving with degraded integrity
|
||||
/// checking unless `RUSTFS_BITROT_SELFTEST_STRICT` aborted it), or
|
||||
/// `unknown` (not yet run or disabled).
|
||||
#[serde(rename = "bitrotSelftest")]
|
||||
bitrot_selftest: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -433,6 +440,14 @@ fn system_admin_discovery(usecase: &DefaultAdminUsecase) -> SystemAdminDiscovery
|
||||
}
|
||||
}
|
||||
|
||||
fn bitrot_selftest_status_str() -> &'static str {
|
||||
match crate::bitrot_selftest::bitrot_selftest_passed() {
|
||||
Some(true) => "passed",
|
||||
Some(false) => "failed",
|
||||
None => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for ServerInfoHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
@@ -464,6 +479,7 @@ impl Operation for ServerInfoHandler {
|
||||
let response = ServerInfoResponse {
|
||||
info,
|
||||
admin_discovery: system_admin_discovery(&usecase),
|
||||
bitrot_selftest: bitrot_selftest_status_str(),
|
||||
};
|
||||
|
||||
let data = serde_json::to_vec(&response).map_err(|e| {
|
||||
@@ -1142,10 +1158,10 @@ impl Operation for RuntimeCapabilitiesHandler {
|
||||
}
|
||||
}
|
||||
|
||||
/// Authorization gate for GET datausageinfo: any-of the dedicated admin action
|
||||
/// OR the bucket listing action. Pinned by a unit test so the gate cannot
|
||||
/// silently narrow or widen (rustfs/backlog#1306).
|
||||
fn data_usage_info_gate_actions() -> Vec<Action> {
|
||||
/// Authorization gate for GET datausageinfo (and prefix usage): any-of the
|
||||
/// dedicated admin action OR the bucket listing action. Pinned by a unit test
|
||||
/// so the gate cannot silently narrow or widen (rustfs/backlog#1306).
|
||||
pub(crate) fn data_usage_info_gate_actions() -> Vec<Action> {
|
||||
vec![
|
||||
Action::AdminAction(AdminAction::DataUsageInfoAdminAction),
|
||||
Action::S3Action(S3Action::ListBucketAction),
|
||||
@@ -1535,6 +1551,18 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The startup bitrot self-test outcome must surface in server info as one
|
||||
/// of three closed-set strings, never an internal enum or a null
|
||||
/// (rustfs/backlog#1873). This test pins the string mapping; whether the
|
||||
/// process-global cell holds Some(true)/Some(false)/None is owned by
|
||||
/// `crate::bitrot_selftest`'s own tests.
|
||||
#[test]
|
||||
fn bitrot_selftest_status_str_is_a_closed_set_of_operators_strings() {
|
||||
let rendered = super::bitrot_selftest_status_str();
|
||||
assert!(matches!(rendered, "passed" | "failed" | "unknown"));
|
||||
assert_eq!(super::bitrot_selftest_status_str(), rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_info_response_exposes_admin_discovery_paths() {
|
||||
let usecase = DefaultAdminUsecase::without_context();
|
||||
@@ -1556,6 +1584,7 @@ mod tests {
|
||||
pools: None,
|
||||
},
|
||||
admin_discovery: system_admin_discovery(&usecase),
|
||||
bitrot_selftest: super::bitrot_selftest_status_str(),
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(response).expect("server info response should serialize");
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Prefix-level bucket usage admin handler (rustfs/backlog#1872).
|
||||
//!
|
||||
//! `GET /rustfs/admin/v3/usage/{bucket}?prefix=&max-entries=` answers
|
||||
//! "what does this bucket / this prefix hold" from the scanner's per-set
|
||||
//! usage caches, with a one-level sub-prefix breakdown — the data console
|
||||
//! buckets view MinIO serves from `loadPrefixUsageFromBackend`.
|
||||
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::handlers::system::data_usage_info_gate_actions;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use hyper::Method;
|
||||
use matchit::Params;
|
||||
use s3s::header::CONTENT_TYPE;
|
||||
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
|
||||
const JSON_CONTENT_TYPE: &str = "application/json";
|
||||
const DEFAULT_MAX_ENTRIES: usize = 1000;
|
||||
const MAX_ENTRIES_LIMIT: usize = 10_000;
|
||||
|
||||
pub struct BucketPrefixUsageHandler {}
|
||||
|
||||
pub fn register_usage_prefix_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
|
||||
r.insert(
|
||||
Method::GET,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/usage/{bucket}").as_str(),
|
||||
AdminOperation(&BucketPrefixUsageHandler {}),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Parse `prefix` and `max-entries` from the query string. Unknown keys are
|
||||
/// rejected so a typo'd parameter cannot silently change the answer's shape.
|
||||
fn parse_usage_prefix_query(query: Option<&str>) -> S3Result<(String, usize)> {
|
||||
let mut prefix: Option<String> = None;
|
||||
let mut max_entries: Option<usize> = None;
|
||||
for (key, value) in url::form_urlencoded::parse(query.unwrap_or_default().as_bytes()) {
|
||||
match key.as_ref() {
|
||||
"prefix" => prefix = Some(value.into_owned()),
|
||||
"max-entries" => {
|
||||
max_entries = Some(
|
||||
value
|
||||
.parse::<usize>()
|
||||
.map_err(|_| s3_error!(InvalidArgument, "max-entries must be a positive integer"))?,
|
||||
);
|
||||
}
|
||||
other => return Err(s3_error!(InvalidArgument, "unknown query parameter: {other}")),
|
||||
}
|
||||
}
|
||||
let max_entries = max_entries.unwrap_or(DEFAULT_MAX_ENTRIES).clamp(1, MAX_ENTRIES_LIMIT);
|
||||
Ok((prefix.unwrap_or_default(), max_entries))
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for BucketPrefixUsageHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let Some(input_cred) = req.credentials else {
|
||||
return Err(s3_error!(InvalidRequest, "get cred failed"));
|
||||
};
|
||||
|
||||
let (cred, owner) =
|
||||
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
|
||||
|
||||
let remote_addr = req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0));
|
||||
validate_admin_request(&req.headers, &cred, owner, false, data_usage_info_gate_actions(), remote_addr).await?;
|
||||
|
||||
let bucket = params.get("bucket").unwrap_or_default().to_string();
|
||||
if bucket.is_empty() {
|
||||
return Err(s3_error!(InvalidRequest, "bucket path parameter is required"));
|
||||
}
|
||||
let (prefix, max_entries) = parse_usage_prefix_query(req.uri.query())?;
|
||||
|
||||
// Authorization is bucket-scoped by the same any-of gate as the
|
||||
// datausageinfo route; the bucket name itself is validated by the
|
||||
// scanner layer, which rejects reserved/invalid names.
|
||||
let response = rustfs_scanner::bucket_prefix_usage(&bucket, &prefix, max_entries)
|
||||
.await
|
||||
.map_err(|err| s3_error!(InvalidArgument, "{}", err))?;
|
||||
|
||||
let data = serde_json::to_vec(&response)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "parse prefix usage failed"))?;
|
||||
let mut header = HeaderMap::new();
|
||||
header.insert(CONTENT_TYPE, HeaderValue::from_static(JSON_CONTENT_TYPE));
|
||||
|
||||
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{DEFAULT_MAX_ENTRIES, MAX_ENTRIES_LIMIT, parse_usage_prefix_query};
|
||||
use s3s::S3Error;
|
||||
|
||||
fn query(raw: &str) -> Result<(String, usize), S3Error> {
|
||||
parse_usage_prefix_query(Some(raw))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_apply_when_no_query_is_given() {
|
||||
assert_eq!(parse_usage_prefix_query(None).unwrap(), (String::new(), DEFAULT_MAX_ENTRIES));
|
||||
assert_eq!(query("").unwrap(), (String::new(), DEFAULT_MAX_ENTRIES));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_round_trips_url_encoded_characters() {
|
||||
let (prefix, _) = query("prefix=pre%2Ffix%20name").unwrap();
|
||||
assert_eq!(prefix, "pre/fix name");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn max_entries_parses_and_clamps_to_documented_bounds() {
|
||||
assert_eq!(query("max-entries=5").unwrap().1, 5);
|
||||
assert_eq!(query("max-entries=0").unwrap().1, 1, "zero must clamp up, not mean unlimited");
|
||||
assert_eq!(query("max-entries=99999999").unwrap().1, MAX_ENTRIES_LIMIT);
|
||||
assert!(query("max-entries=-3").is_err());
|
||||
assert!(query("max-entries=abc").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_parameters_are_rejected_not_ignored() {
|
||||
assert!(
|
||||
query("prefixes=x").is_err(),
|
||||
"a typo'd parameter must fail the request, not widen the query"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -40,7 +40,8 @@ use handlers::{
|
||||
audit, batch_job, bucket_meta, cluster_snapshot, config_admin, diagnostics, durability as durability_handler, extensions,
|
||||
heal, health, idp_compat, ilm_transition, inspect_archive, kms, module_switch, object_data_cache, object_zip_download, oidc,
|
||||
plugins_catalog, plugins_instances, pools, profile_admin, quota as quota_handler, rebalance,
|
||||
replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, user,
|
||||
replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, usage_prefix,
|
||||
user,
|
||||
};
|
||||
use router::{AdminOperation, S3Router};
|
||||
use s3s::route::S3Route;
|
||||
@@ -80,6 +81,7 @@ fn register_admin_routes(r: &mut S3Router<AdminOperation>) -> std::io::Result<()
|
||||
bucket_meta::register_bucket_meta_route(r)?;
|
||||
config_admin::register_config_route(r)?;
|
||||
scanner::register_scanner_route(r)?;
|
||||
usage_prefix::register_usage_prefix_route(r)?;
|
||||
ilm_transition::register_ilm_transition_route(r)?;
|
||||
object_data_cache::register_object_data_cache_route(r)?;
|
||||
audit::register_audit_target_route(r)?;
|
||||
|
||||
@@ -1558,6 +1558,11 @@ pub const DEFERRED_ADMIN_ROUTE_POLICIES: &[DeferredAdminRoutePolicy] = &[
|
||||
"/rustfs/admin/v3/datausageinfo",
|
||||
DeferredRoutePolicyReason::MultipleActions,
|
||||
),
|
||||
deferred(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/v3/usage/{bucket}",
|
||||
DeferredRoutePolicyReason::MultipleActions,
|
||||
),
|
||||
deferred(
|
||||
HttpMethod::Post,
|
||||
"/rustfs/admin/v3/object-zip-downloads",
|
||||
|
||||
@@ -172,6 +172,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
admin_route(Method::POST, "/v4/inspect/archive"),
|
||||
admin_route(Method::GET, "/v3/storageinfo"),
|
||||
admin_route(Method::GET, "/v3/datausageinfo"),
|
||||
admin_route_sample(Method::GET, "/v3/usage/{bucket}", "/v3/usage/test-bucket"),
|
||||
admin_route(Method::GET, "/v3/metrics"),
|
||||
admin_route(Method::GET, "/v3/object-data-cache/stats"),
|
||||
admin_route(Method::POST, "/v3/object-data-cache/flush"),
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Startup bitrot algorithm self-test (rustfs/backlog#1873).
|
||||
//!
|
||||
//! A drifted hash implementation fails silently in production: every shard
|
||||
//! reads back "corrupt", heal rewrites healthy data, and cross-platform
|
||||
//! clusters disagree about which copy is good. [`run_startup_bitrot_self_test`]
|
||||
//! pins the algorithms once at process start — the check itself runs in well
|
||||
//! under a millisecond on 4 KiB, so it executes inline before background
|
||||
//! services come up and the result is published before the server accepts
|
||||
//! traffic.
|
||||
//!
|
||||
//! Outcome surface:
|
||||
//! - one structured `bitrot_selftest` log event (`passed`/`failed`/`skipped`),
|
||||
//! - the `rustfs_bitrot_selftest_status` gauge (1=passed, 0=failed, 2=skipped),
|
||||
//! - [`bitrot_selftest_passed`] for admin/health surfaces,
|
||||
//! - `RUSTFS_BITROT_SELFTEST_STRICT=on` turns a failure into a startup error
|
||||
//! (MinIO `bitrotSelfTest` Fatal parity); the default only degrades the
|
||||
//! status so a bad build cannot brick an existing fleet on upgrade.
|
||||
|
||||
use crate::storage_api::startup::background::{BitrotSelfTestError, bitrot_self_test};
|
||||
use metrics::gauge;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
use std::time::Instant;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
const LOG_COMPONENT_MAIN: &str = "main";
|
||||
const LOG_SUBSYSTEM_STARTUP: &str = "startup";
|
||||
const EVENT_BITROT_SELFTEST: &str = "bitrot_selftest";
|
||||
const METRIC_BITROT_SELFTEST_STATUS: &str = "rustfs_bitrot_selftest_status";
|
||||
|
||||
/// Gauge values for [`METRIC_BITROT_SELFTEST_STATUS`].
|
||||
const STATUS_PASSED: f64 = 1.0;
|
||||
const STATUS_FAILED: f64 = 0.0;
|
||||
const STATUS_SKIPPED: f64 = 2.0;
|
||||
|
||||
/// Internal cell values for [`BITROT_SELF_TEST_STATUS`].
|
||||
const STATUS_CELL_UNSET: u8 = 0;
|
||||
const STATUS_CELL_PASSED: u8 = 1;
|
||||
const STATUS_CELL_FAILED: u8 = 2;
|
||||
|
||||
static BITROT_SELF_TEST_STATUS: AtomicU8 = AtomicU8::new(STATUS_CELL_UNSET);
|
||||
|
||||
/// Last recorded self-test outcome: `None` before the first run, then
|
||||
/// `Some(true)` on a passing check and `Some(false)` on a failed one (a
|
||||
/// skipped check never publishes, so it cannot read as a pass). The cell is
|
||||
/// last-writer-wins rather than set-once: production runs the self-test once,
|
||||
/// and last-writer-wins keeps tests that exercise both outcomes
|
||||
/// order-independent.
|
||||
pub fn bitrot_selftest_passed() -> Option<bool> {
|
||||
match BITROT_SELF_TEST_STATUS.load(Ordering::Acquire) {
|
||||
STATUS_CELL_UNSET => None,
|
||||
STATUS_CELL_PASSED => Some(true),
|
||||
STATUS_CELL_FAILED => Some(false),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the bitrot self-test and publish the outcome. In strict mode a failure
|
||||
/// is returned as an error so the caller aborts startup.
|
||||
pub(crate) async fn run_startup_bitrot_self_test(enabled: bool, strict: bool) -> io::Result<()> {
|
||||
run_startup_bitrot_self_test_with(enabled, strict, bitrot_self_test).await
|
||||
}
|
||||
|
||||
async fn run_startup_bitrot_self_test_with<F, Fut>(enabled: bool, strict: bool, run_check: F) -> io::Result<()>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Result<(), BitrotSelfTestError>>,
|
||||
{
|
||||
if !enabled {
|
||||
gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_SKIPPED);
|
||||
debug!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_BITROT_SELFTEST,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
state = "skipped",
|
||||
reason = "disabled",
|
||||
"Bitrot self-test skipped"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let started = Instant::now();
|
||||
match run_check().await {
|
||||
Ok(()) => {
|
||||
BITROT_SELF_TEST_STATUS.store(STATUS_CELL_PASSED, Ordering::Release);
|
||||
gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_PASSED);
|
||||
info!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_BITROT_SELFTEST,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
state = "passed",
|
||||
duration_us = started.elapsed().as_micros() as u64,
|
||||
"Bitrot self-test passed"
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
BITROT_SELF_TEST_STATUS.store(STATUS_CELL_FAILED, Ordering::Release);
|
||||
gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_FAILED);
|
||||
error!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_BITROT_SELFTEST,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
state = "failed",
|
||||
duration_us = started.elapsed().as_micros() as u64,
|
||||
error = %err,
|
||||
"Bitrot self-test failed"
|
||||
);
|
||||
if strict {
|
||||
return Err(io::Error::other(format!("bitrot self-test failed: {err}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{BITROT_SELF_TEST_STATUS, STATUS_CELL_UNSET, bitrot_selftest_passed, run_startup_bitrot_self_test_with};
|
||||
use crate::storage_api::startup::background::BitrotSelfTestError;
|
||||
use std::future::ready;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
fn failing_check() -> impl Future<Output = Result<(), BitrotSelfTestError>> {
|
||||
ready(Err(BitrotSelfTestError::RoundtripReadback {
|
||||
algorithm: "HighwayHash256S",
|
||||
}))
|
||||
}
|
||||
|
||||
/// All scenarios run sequentially inside one test: the status cell is
|
||||
/// process-global, so parallel per-scenario tests would race the reset and
|
||||
/// read each other's outcomes (the exact order-dependent flake class this
|
||||
/// module exists to avoid).
|
||||
#[tokio::test]
|
||||
async fn startup_self_test_publishes_outcome_and_strict_gates_abort() {
|
||||
BITROT_SELF_TEST_STATUS.store(STATUS_CELL_UNSET, Ordering::Release);
|
||||
|
||||
// Skipped: publishes nothing, never fails, never aborts.
|
||||
run_startup_bitrot_self_test_with(false, true, || async { Ok(()) })
|
||||
.await
|
||||
.expect("a disabled self-test must not fail even in strict mode");
|
||||
assert_eq!(bitrot_selftest_passed(), None, "a skipped run must leave the status unset");
|
||||
|
||||
// Passing: publishes Some(true), never fails.
|
||||
run_startup_bitrot_self_test_with(true, false, || async { Ok(()) })
|
||||
.await
|
||||
.expect("a passing check must never fail startup");
|
||||
assert_eq!(bitrot_selftest_passed(), Some(true), "a passing run must publish Some(true)");
|
||||
|
||||
// Failing, non-strict: publishes Some(false) but startup continues.
|
||||
run_startup_bitrot_self_test_with(true, false, failing_check)
|
||||
.await
|
||||
.expect("a failed check must not abort startup in non-strict mode");
|
||||
assert_eq!(bitrot_selftest_passed(), Some(false), "a failing run must publish Some(false)");
|
||||
|
||||
// Failing, strict: startup error carries the failure and the published
|
||||
// outcome stays a failure.
|
||||
let err = run_startup_bitrot_self_test_with(true, true, failing_check)
|
||||
.await
|
||||
.expect_err("strict mode must turn a failed check into a startup error");
|
||||
assert!(err.to_string().contains("bitrot self-test failed"));
|
||||
assert_eq!(bitrot_selftest_passed(), Some(false));
|
||||
}
|
||||
}
|
||||
+178
-37
@@ -304,30 +304,37 @@ fn build_local_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
|
||||
Ok(kms_config)
|
||||
}
|
||||
|
||||
/// Collect the Vault settings the command line owns.
|
||||
///
|
||||
/// Everything else — auth method, namespace, TLS, KV mount and metadata paths —
|
||||
/// is resolved from the environment by the KMS crate, so this path and
|
||||
/// [`rustfs_kms::config::KmsConfig::from_env`] cannot drift apart. The address
|
||||
/// stays required here so a missing one is still named instead of silently
|
||||
/// falling back to the crate's localhost default.
|
||||
fn vault_cli_overrides<'a>(
|
||||
cfg: &'a config::Config,
|
||||
backend_name: &str,
|
||||
) -> std::io::Result<rustfs_kms::config::VaultCliOverrides<'a>> {
|
||||
let address = cfg
|
||||
.kms_vault_address
|
||||
.as_deref()
|
||||
.ok_or_else(|| Error::other(format!("Vault address is required for {backend_name} backend")))?;
|
||||
|
||||
Ok(rustfs_kms::config::VaultCliOverrides {
|
||||
address: Some(address),
|
||||
token: cfg.kms_vault_token.as_deref(),
|
||||
mount_path: cfg.kms_vault_mount_path.as_deref(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Build KMS configuration for Vault backend
|
||||
fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::config::KmsConfig> {
|
||||
let vault_address = cfg
|
||||
.kms_vault_address
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("Vault address is required for vault backend"))?;
|
||||
let vault_token = cfg
|
||||
.kms_vault_token
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("Vault token is required for vault backend"))?;
|
||||
let backend_config = rustfs_kms::config::vault_kv2_config_from_env(vault_cli_overrides(cfg, "vault")?)
|
||||
.map_err(|e| Error::other(format!("Vault KMS configuration failed: {e}")))?;
|
||||
|
||||
let kms_config = rustfs_kms::config::KmsConfig {
|
||||
backend: rustfs_kms::config::KmsBackend::VaultKv2,
|
||||
backend_config: rustfs_kms::config::BackendConfig::VaultKv2(Box::new(rustfs_kms::config::VaultConfig {
|
||||
address: vault_address.clone(),
|
||||
auth_method: rustfs_kms::config::VaultAuthMethod::Token {
|
||||
token: vault_token.clone(),
|
||||
},
|
||||
namespace: None,
|
||||
mount_path: cfg.kms_vault_mount_path.clone().unwrap_or_else(|| "transit".to_string()),
|
||||
kv_mount: "secret".to_string(),
|
||||
key_path_prefix: "rustfs/kms/keys".to_string(),
|
||||
tls: None,
|
||||
})),
|
||||
backend_config: rustfs_kms::config::BackendConfig::VaultKv2(Box::new(backend_config)),
|
||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||
default_key_id: cfg.kms_default_key_id.clone(),
|
||||
@@ -344,26 +351,12 @@ fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
|
||||
|
||||
/// Build KMS configuration for Vault Transit backend
|
||||
fn build_vault_transit_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::config::KmsConfig> {
|
||||
let vault_address = cfg
|
||||
.kms_vault_address
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("Vault address is required for vault-transit backend"))?;
|
||||
let vault_token = cfg
|
||||
.kms_vault_token
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("Vault token is required for vault-transit backend"))?;
|
||||
let backend_config = rustfs_kms::config::vault_transit_config_from_env(vault_cli_overrides(cfg, "vault-transit")?)
|
||||
.map_err(|e| Error::other(format!("Vault Transit KMS configuration failed: {e}")))?;
|
||||
|
||||
let kms_config = rustfs_kms::config::KmsConfig {
|
||||
backend: rustfs_kms::config::KmsBackend::VaultTransit,
|
||||
backend_config: rustfs_kms::config::BackendConfig::VaultTransit(Box::new(rustfs_kms::config::VaultTransitConfig {
|
||||
address: vault_address.clone(),
|
||||
auth_method: rustfs_kms::config::VaultAuthMethod::Token {
|
||||
token: vault_token.clone(),
|
||||
},
|
||||
namespace: None,
|
||||
mount_path: cfg.kms_vault_mount_path.clone().unwrap_or_else(|| "transit".to_string()),
|
||||
..rustfs_kms::config::VaultTransitConfig::default()
|
||||
})),
|
||||
backend_config: rustfs_kms::config::BackendConfig::VaultTransit(Box::new(backend_config)),
|
||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||
default_key_id: cfg.kms_default_key_id.clone(),
|
||||
@@ -1405,7 +1398,10 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{build_aws_kms_config, notification_config_to_event_rules, resolve_buffer_profile_config};
|
||||
use super::{
|
||||
build_aws_kms_config, build_vault_kms_config, build_vault_transit_kms_config, notification_config_to_event_rules,
|
||||
resolve_buffer_profile_config,
|
||||
};
|
||||
use crate::config::{BufferConfig, WorkloadProfile};
|
||||
use rustfs_config::KI_B;
|
||||
use rustfs_s3_types::EventName;
|
||||
@@ -1499,6 +1495,151 @@ mod tests {
|
||||
assert!(err.to_string().contains("Invalid ARN"), "unexpected error: {err}");
|
||||
}
|
||||
|
||||
fn vault_kms_test_config(backend: &str) -> crate::config::Config {
|
||||
let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-vault-kms".to_string()]);
|
||||
config.kms_enable = true;
|
||||
config.kms_backend = backend.to_string();
|
||||
config.kms_vault_address = Some("https://vault.example.com:8200".to_string());
|
||||
config
|
||||
}
|
||||
|
||||
/// The Vault auth method and the settings the CLI has no flag for come from
|
||||
/// the environment, so startup and `KmsConfig::from_env` cannot disagree.
|
||||
/// Regression: startup used to hardcode token auth and require a token,
|
||||
/// which made every non-token method unreachable through `RUSTFS_KMS_ENABLE`.
|
||||
#[test]
|
||||
fn build_vault_transit_kms_config_resolves_auth_and_mounts_from_env() {
|
||||
let config = temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_KMS_VAULT_TOKEN", None),
|
||||
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
|
||||
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None),
|
||||
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", Some("env-role-id")),
|
||||
("RUSTFS_KMS_VAULT_APPROLE_SECRET_ID", Some("env-secret-id")),
|
||||
("RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE", None),
|
||||
("RUSTFS_KMS_VAULT_NAMESPACE", Some("team-a")),
|
||||
("RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT", Some("rustfs-kv")),
|
||||
],
|
||||
|| {
|
||||
build_vault_transit_kms_config(&vault_kms_test_config("vault-transit"))
|
||||
.expect("vault transit KMS configuration should build")
|
||||
},
|
||||
);
|
||||
|
||||
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||
let rustfs_kms::config::VaultAuthMethod::AppRole { role_id, secret_id, .. } = &vault.auth_method else {
|
||||
panic!("approle in the environment must select AppRole auth, got {:?}", vault.auth_method);
|
||||
};
|
||||
assert_eq!(role_id, "env-role-id");
|
||||
assert_eq!(secret_id, "env-secret-id");
|
||||
assert_eq!(vault.namespace.as_deref(), Some("team-a"));
|
||||
assert_eq!(vault.metadata_kv_mount, "rustfs-kv");
|
||||
}
|
||||
|
||||
/// Kubernetes auth needs no credential in the environment at all: the role
|
||||
/// selects it and the pod's projected ServiceAccount token supplies the rest.
|
||||
#[test]
|
||||
fn build_vault_transit_kms_config_selects_kubernetes_auth() {
|
||||
let config = temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_KMS_VAULT_TOKEN", None),
|
||||
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
|
||||
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
|
||||
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", Some("rustfs")),
|
||||
("RUSTFS_KMS_VAULT_KUBERNETES_MOUNT", None),
|
||||
("RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH", None),
|
||||
],
|
||||
|| {
|
||||
build_vault_transit_kms_config(&vault_kms_test_config("vault-transit"))
|
||||
.expect("vault transit KMS configuration should build")
|
||||
},
|
||||
);
|
||||
|
||||
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||
let rustfs_kms::config::VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} = &vault.auth_method
|
||||
else {
|
||||
panic!(
|
||||
"a kubernetes role in the environment must select Kubernetes auth, got {:?}",
|
||||
vault.auth_method
|
||||
);
|
||||
};
|
||||
assert_eq!(role, "rustfs");
|
||||
assert_eq!(mount, rustfs_kms::config::DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||
assert_eq!(jwt_path, std::path::Path::new(rustfs_kms::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||
}
|
||||
|
||||
/// Two credential sources leave the effective identity ambiguous, so
|
||||
/// startup refuses rather than picking one.
|
||||
#[test]
|
||||
fn build_vault_kms_config_refuses_two_auth_methods() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_KMS_VAULT_TOKEN", None),
|
||||
("RUSTFS_KMS_VAULT_TOKEN_FILE", Some("/run/vault-agent/token")),
|
||||
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
|
||||
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", Some("rustfs")),
|
||||
],
|
||||
|| {
|
||||
let error = build_vault_kms_config(&vault_kms_test_config("vault"))
|
||||
.expect_err("two Vault auth methods must not start the server");
|
||||
assert!(error.to_string().contains("exactly one"), "unexpected error: {error}");
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// The KV2 backend has its own builder, so the key-location settings have
|
||||
/// to be proven separately from the Transit one: pointing at the wrong KV
|
||||
/// mount or prefix makes existing keys look absent.
|
||||
#[test]
|
||||
fn build_vault_kms_config_resolves_kv_mount_and_prefix_from_env() {
|
||||
let config = temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_KMS_VAULT_TOKEN", Some("a-real-token")),
|
||||
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
|
||||
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
|
||||
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None),
|
||||
("RUSTFS_KMS_VAULT_KV_MOUNT", Some("rustfs-kv")),
|
||||
("RUSTFS_KMS_VAULT_KEY_PREFIX", Some("tenant/keys")),
|
||||
],
|
||||
|| build_vault_kms_config(&vault_kms_test_config("vault")).expect("vault KV2 KMS configuration should build"),
|
||||
);
|
||||
|
||||
let vault = config.vault_config().expect("vault kv2 backend config");
|
||||
assert_eq!(vault.kv_mount, "rustfs-kv");
|
||||
assert_eq!(vault.key_path_prefix, "tenant/keys");
|
||||
}
|
||||
|
||||
/// Skipping TLS verification was silently dropped on this path before, so
|
||||
/// an operator who asked for it still got a verified connection. Now that it
|
||||
/// is honoured it must fail closed without the development opt-in, rather
|
||||
/// than quietly downgrading the Vault connection.
|
||||
#[test]
|
||||
fn build_vault_transit_kms_config_refuses_skip_tls_verify_without_opt_in() {
|
||||
let vars = [
|
||||
("RUSTFS_KMS_VAULT_TOKEN", Some("a-real-token")),
|
||||
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
|
||||
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
|
||||
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None),
|
||||
("RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY", Some("true")),
|
||||
];
|
||||
|
||||
temp_env::with_vars(vars, || {
|
||||
let error = build_vault_transit_kms_config(&vault_kms_test_config("vault-transit"))
|
||||
.expect_err("skipping TLS verification must not start the server");
|
||||
assert!(error.to_string().contains("TLS"), "unexpected error: {error}");
|
||||
});
|
||||
|
||||
temp_env::with_vars(vars, || {
|
||||
let mut cfg = vault_kms_test_config("vault-transit");
|
||||
cfg.kms_allow_insecure_dev_defaults = true;
|
||||
let config = build_vault_transit_kms_config(&cfg).expect("the development opt-in should accept skip-verify");
|
||||
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||
assert!(vault.tls.as_ref().is_some_and(|tls| tls.skip_verify));
|
||||
});
|
||||
}
|
||||
|
||||
fn aws_kms_test_config() -> crate::config::Config {
|
||||
let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-aws-kms".to_string()]);
|
||||
config.kms_enable = true;
|
||||
|
||||
@@ -76,6 +76,7 @@ pub mod allocator_reclaim;
|
||||
pub mod app;
|
||||
pub mod auth;
|
||||
pub mod auth_keystone;
|
||||
pub(crate) mod bitrot_selftest;
|
||||
pub mod capacity;
|
||||
pub mod cluster_snapshot;
|
||||
pub mod config;
|
||||
|
||||
@@ -33,6 +33,8 @@ pub(crate) const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED";
|
||||
pub(crate) const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER";
|
||||
pub(crate) const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED";
|
||||
pub(crate) const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL";
|
||||
pub(crate) const ENV_BITROT_SELFTEST_ENABLE: &str = "RUSTFS_BITROT_SELFTEST_ENABLE";
|
||||
pub(crate) const ENV_BITROT_SELFTEST_STRICT: &str = "RUSTFS_BITROT_SELFTEST_STRICT";
|
||||
|
||||
static AUDIT_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_AUDIT_ENABLE);
|
||||
static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE);
|
||||
@@ -47,6 +49,18 @@ pub(crate) fn heal_enabled_from_env() -> bool {
|
||||
get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true)
|
||||
}
|
||||
|
||||
/// Whether the startup bitrot algorithm self-test runs, defaulting to on
|
||||
/// (rustfs/backlog#1873).
|
||||
pub(crate) fn bitrot_selftest_enabled_from_env() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_BITROT_SELFTEST_ENABLE, true)
|
||||
}
|
||||
|
||||
/// Whether a failed bitrot self-test aborts startup instead of only logging
|
||||
/// and exposing a failed status, defaulting to off.
|
||||
pub(crate) fn bitrot_selftest_strict_from_env() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_BITROT_SELFTEST_STRICT, false)
|
||||
}
|
||||
|
||||
/// Last published audit-module state.
|
||||
pub fn is_audit_module_enabled() -> bool {
|
||||
AUDIT_MODULE_ENABLED.load(Ordering::Relaxed)
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::module_switches::{heal_enabled_from_env, scanner_enabled_from_env};
|
||||
use crate::bitrot_selftest::run_startup_bitrot_self_test;
|
||||
use crate::module_switches::{
|
||||
bitrot_selftest_enabled_from_env, bitrot_selftest_strict_from_env, heal_enabled_from_env, scanner_enabled_from_env,
|
||||
};
|
||||
use crate::storage_api::startup::background::{ECStore, set_workload_admission_snapshot_provider};
|
||||
use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider;
|
||||
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
|
||||
@@ -27,6 +30,12 @@ const LOG_SUBSYSTEM_STARTUP: &str = "startup";
|
||||
const EVENT_BACKGROUND_SERVICES_CONFIGURED: &str = "background_services_configured";
|
||||
|
||||
pub(crate) async fn init_background_service_runtime(store: Arc<ECStore>) -> Result<bool> {
|
||||
// Pin the bitrot algorithms before anything can write or verify a shard:
|
||||
// the check costs well under a millisecond, and in strict mode a drifted
|
||||
// build must abort here rather than after it has touched data
|
||||
// (rustfs/backlog#1873).
|
||||
run_startup_bitrot_self_test(bitrot_selftest_enabled_from_env(), bitrot_selftest_strict_from_env()).await?;
|
||||
|
||||
let _ = create_ahm_services_cancel_token();
|
||||
|
||||
let enable_scanner = scanner_enabled_from_env();
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user