gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-06 05:17:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): normalize admin telemetry and error messages (#3430)

Browse Source
This commit is contained in:
houseme
2026-06-14 13:27:10 +08:00
committed by GitHub
parent dc82efbab4
commit e8012bd1ba
70 changed files with 4807 additions and 1445 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+123 -116
View File
@@ -35,9 +35,9 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]]
name = "aead"
version = "0.6.0-rc.10"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b657e772794c6b04730ea897b66a058ccd866c16d1967da05eeeecec39043fe"
checksum = "ef60ac202874e574ce7a7158cc8bca7313dd344322482e4fadee288bf4a306b8"
dependencies = [
"crypto-common 0.2.2",
"inout 0.2.2",
@@ -277,9 +277,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
[[package]]
name = "ar_archive_writer"
version = "0.5.1"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b"
checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348"
dependencies = [
"object 0.37.3",
]
@@ -887,9 +887,9 @@ dependencies = [
[[package]]
name = "aws-runtime"
version = "1.7.4"
version = "1.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ed8e8c52d2dc2390ad9f15647fe663f71e9780b4262c190fbb823a32721566"
checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39"
dependencies = [
"aws-credential-types",
"aws-sigv4",
@@ -915,9 +915,9 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
version = "1.135.0"
version = "1.136.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f97e3e7e7d86fd26fcdc18bc382da5ca9e8b2ff8d54030d187fd0dac8a236d96"
checksum = "fbd03e531a7d981fba45114c5813a7565dd470a1bd3ef1188ca98c98ebdfc668"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -951,9 +951,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
version = "1.101.0"
version = "1.102.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b647baea49ff551960b904f905681e9b4765a6c4ea08631e89dc52d8bd3f5896"
checksum = "8c82b3ac19f1431854f7ace3a7531674633e286bfdde21976893bfee36fd493b"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -976,9 +976,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
version = "1.103.0"
version = "1.104.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ae401c65ff288aa7873117fe535cd32b7b1bb0bc43751d28901a1d5f20636b9"
checksum = "321000d2b4c5519ee573f73167f612efd7329322d9b26969ad1979f0427f1913"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1001,9 +1001,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
version = "1.106.0"
version = "1.107.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c80de7bb7d03e9ca8c9fd7b489f20f3948d3f3be91a7953591347d238115408"
checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1086,9 +1086,9 @@ dependencies = [
[[package]]
name = "aws-smithy-eventstream"
version = "0.60.20"
version = "0.60.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf09d74e5e32f76b8762da505a3cd59303e367a664ca67295387baa8c1d7548"
checksum = "78d8391e65fcea47c586a22e1a41f173b38615b112b2c6b7a44e80cec3e6b706"
dependencies = [
"aws-smithy-types",
"bytes",
@@ -1509,9 +1509,9 @@ dependencies = [
[[package]]
name = "block-buffer"
version = "0.12.0"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be"
checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
dependencies = [
"hybrid-array",
"zeroize",
@@ -1560,9 +1560,9 @@ dependencies = [
[[package]]
name = "bon"
version = "3.9.1"
version = "3.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe"
checksum = "b2f04f6fef12d70d42a77b1433c9e0f065238479a6cefc4f5bab105e9873a3c3"
dependencies = [
"bon-macros",
"rustversion",
@@ -1570,9 +1570,9 @@ dependencies = [
[[package]]
name = "bon-macros"
version = "3.9.1"
version = "3.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c"
checksum = "7d0bd4c2f75335ad98052a37efb54f428b492f64340257143b3429c8a508fa7b"
dependencies = [
"darling 0.23.0",
"ident_case",
@@ -1661,9 +1661,9 @@ dependencies = [
[[package]]
name = "bytesize"
version = "2.3.1"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3"
checksum = "49e78e506b9d7633710dab98996f22f95f3d0f488e8f1aa162830556ed9fc14d"
[[package]]
name = "bytestring"
@@ -1742,9 +1742,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.63"
version = "1.2.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -1856,7 +1856,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c"
dependencies = [
"block-buffer 0.12.0",
"block-buffer 0.12.1",
"crypto-common 0.2.2",
"inout 0.2.2",
"zeroize",
@@ -2348,12 +2348,11 @@ dependencies = [
[[package]]
name = "crypto-primes"
version = "0.7.0"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21f41f23de7d24cdbda7f0c4d9c0351f99a4ceb258ef30e5c1927af8987ffe5a"
checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f"
dependencies = [
"crypto-bigint 0.7.3",
"libm",
"rand_core 0.10.1",
]
@@ -3380,7 +3379,6 @@ version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
dependencies = [
"powerfmt",
"serde_core",
]
@@ -3551,7 +3549,7 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
dependencies = [
"block-buffer 0.12.0",
"block-buffer 0.12.1",
"const-oid 0.10.2",
"crypto-common 0.2.2",
"ctutils",
@@ -3576,7 +3574,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3912,7 +3910,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4294,11 +4292,10 @@ dependencies = [
[[package]]
name = "getset"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912"
checksum = "6cf442baaabe4213ce7d1239afc26c039180b6456da2cededa316ae2c8a77a77"
dependencies = [
"proc-macro-error2",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -5345,7 +5342,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -5512,13 +5509,12 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.99"
version = "0.3.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31"
dependencies = [
"cfg-if",
"futures-util",
"once_cell",
"wasm-bindgen",
]
@@ -5978,6 +5974,29 @@ dependencies = [
"zerocopy-derive",
]
[[package]]
name = "manyhow"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587"
dependencies = [
"manyhow-macros",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "manyhow-macros"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495"
dependencies = [
"proc-macro-utils",
"proc-macro2",
"quote",
]
[[package]]
name = "mappings"
version = "0.7.2"
@@ -6040,9 +6059,9 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"
[[package]]
name = "memchr"
version = "2.8.1"
version = "2.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
[[package]]
name = "memmap2"
@@ -6104,9 +6123,9 @@ dependencies = [
[[package]]
name = "metrique"
version = "0.1.25"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2638fb6c2325d3e35c7b0d1ce49cf8ce74b89b7439d4dc5ce1c9fff0f7fa18a1"
checksum = "ffdfd0c9954a916c5e1db230cc7593569858afec7744cebfd0fbef45de7b07c7"
dependencies = [
"itoa",
"jiff",
@@ -6124,9 +6143,9 @@ dependencies = [
[[package]]
name = "metrique-core"
version = "0.1.20"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16b99ea3484ab9afe6337413bacc842d516e65623a476699f9873803028e6a9e"
checksum = "f56ef332b47a4d2dc14209bf9ce4334682dd2fec43a4b84e5ed063bf2f4ea29a"
dependencies = [
"itertools 0.14.0",
"metrique-writer-core",
@@ -6134,9 +6153,9 @@ dependencies = [
[[package]]
name = "metrique-macro"
version = "0.1.16"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a02c5cb6e55c1ec5eb3ef2d10d137805547c486ad54b587d42d0a49713e6ef9"
checksum = "403c61cd847b11680ae6cf1feb384cbe06e412e4dc221f1e16ccf48038d190bb"
dependencies = [
"Inflector",
"darling 0.23.0",
@@ -6147,9 +6166,9 @@ dependencies = [
[[package]]
name = "metrique-service-metrics"
version = "0.1.21"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb71714973a9ba53d609a577311aae4d8beb7df37869b5a92cb777a3f4c905b8"
checksum = "93045b7f5c8b998946b715de07523b1c6d81932512c5c27590bf73328aaf3832"
dependencies = [
"metrique-writer",
]
@@ -6162,9 +6181,9 @@ checksum = "d607939211e4eaaa8cd35394fa5e57faffb7390d0ac513b39992edcaf3cc526c"
[[package]]
name = "metrique-writer"
version = "0.1.22"
version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04f770fe8a45a7c2f285c7ce81b9364e0e5604cbb7daf693fef1f624bf8f0bf8"
checksum = "ae9e23184d81616fcdfcdc5768e14c2bca765096094d08e7895259fe658378e4"
dependencies = [
"ahash 0.8.12",
"crossbeam-queue",
@@ -6183,9 +6202,9 @@ dependencies = [
[[package]]
name = "metrique-writer-core"
version = "0.1.16"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a0b67a72a8849987122ca8ceacaf81d47eaba36cd673767fc3742a086dd04ac"
checksum = "23421d85733c0182613b32f32094080e07cd5e98be5893f1cd3baf7eed16c4d0"
dependencies = [
"derive-where",
"itertools 0.14.0",
@@ -6337,15 +6356,15 @@ checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b"
[[package]]
name = "mysql-common-derive"
version = "0.32.1"
version = "0.32.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66f62cad7623a9cb6f8f64037f0c4f69c8db8e82914334a83c9788201c2c1bfa"
checksum = "a4db8a44120571277accfaa3f3d91e7d3989d601d817c2fc01a9391b86135666"
dependencies = [
"darling 0.20.11",
"darling 0.23.0",
"heck",
"manyhow",
"num-bigint",
"proc-macro-crate",
"proc-macro-error2",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -6571,7 +6590,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -6743,7 +6762,7 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [
"base64 0.21.7",
"base64 0.22.1",
"chrono",
"getrandom 0.2.17",
"http 1.4.2",
@@ -7798,9 +7817,9 @@ dependencies = [
[[package]]
name = "primefield"
version = "0.14.0-rc.10"
version = "0.14.0-rc.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f845ec3240cd5ed5e1e31cf3ff633a5bf47c698dc4092ba9e767415b3d393406"
checksum = "b1d7e42f46a29abc16fb621a3466ee453358ebaae48a9e515f287e0af052ed8f"
dependencies = [
"crypto-bigint 0.7.3",
"crypto-common 0.2.2",
@@ -7838,25 +7857,14 @@ dependencies = [
]
[[package]]
name = "proc-macro-error-attr2"
version = "2.0.0"
name = "proc-macro-utils"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5"
checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071"
dependencies = [
"proc-macro2",
"quote",
]
[[package]]
name = "proc-macro-error2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
dependencies = [
"proc-macro-error-attr2",
"proc-macro2",
"quote",
"syn 2.0.117",
"smallvec",
]
[[package]]
@@ -8480,9 +8488,9 @@ dependencies = [
[[package]]
name = "redis"
version = "1.2.2"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a12e6b5f4d8ef33944e833e2b1859ad478deab6e431d7337b30ee2efe21f7543"
checksum = "f9fd510128eda94d1d49b9f81487744d5c451422431cce41238fe2853d29f4cc"
dependencies = [
"arc-swap",
"arcstr",
@@ -10216,7 +10224,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.12.1",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -10290,7 +10298,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -11177,9 +11185,9 @@ dependencies = [
[[package]]
name = "starshard"
version = "2.2.0"
version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "472e5a677707be0fbe7bd851ca25ed7e7757e83dc07a05447cdb6fbdd27db5e2"
checksum = "4dafd0cbefb050fa7a7f95ac229f7978ed4c99866a1a5b62e34a154b52d73d77"
dependencies = [
"async-trait",
"hashbrown 0.17.1",
@@ -11441,10 +11449,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.3.4",
"getrandom 0.4.2",
"once_cell",
"rustix 1.1.4",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -11582,12 +11590,11 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.47"
version = "0.3.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469"
dependencies = [
"deranged",
"itoa",
"js-sys",
"libc",
"num-conv",
@@ -11600,15 +11607,15 @@ dependencies = [
[[package]]
name = "time-core"
version = "0.1.8"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
[[package]]
name = "time-macros"
version = "0.2.27"
version = "0.2.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d"
dependencies = [
"num-conv",
"time-core",
@@ -12372,9 +12379,9 @@ dependencies = [
[[package]]
name = "wasip2"
version = "1.0.3+wasi-0.2.9"
version = "1.0.4+wasi-0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
dependencies = [
"wit-bindgen 0.57.1",
]
@@ -12399,9 +12406,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
version = "0.2.122"
version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a"
dependencies = [
"cfg-if",
"once_cell",
@@ -12412,9 +12419,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.72"
version = "0.4.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"
checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -12422,9 +12429,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.122"
version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -12432,9 +12439,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.122"
version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd"
dependencies = [
"bumpalo",
"proc-macro2",
@@ -12445,9 +12452,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.122"
version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f"
dependencies = [
"unicode-ident",
]
@@ -12501,9 +12508,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.99"
version = "0.3.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -12608,7 +12615,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -13067,18 +13074,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.50"
version = "0.8.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.50"
version = "0.8.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
dependencies = [
"proc-macro2",
"quote",
@@ -13108,18 +13115,18 @@ dependencies = [
[[package]]
name = "zeroize"
version = "1.8.2"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
dependencies = [
"zeroize_derive",
]
[[package]]
name = "zeroize_derive"
version = "1.4.3"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e"
checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328"
dependencies = [
"proc-macro2",
"quote",
Cargo.toml
+6 -6
View File
@@ -159,7 +159,7 @@ tower-http = { version = "0.6.11", features = ["cors"] }
# Serialization and Data Formats
bytes = { version = "1.11.1", features = ["serde"] }
bytesize = "2.3.1"
bytesize = "2.4.0"
byteorder = "1.5.0"
flatbuffers = "25.12.19"
form_urlencoded = "1.2.2"
@@ -188,13 +188,13 @@ rustls-pki-types = "1.14.1"
sha1 = "0.11.0"
sha2 = "0.11.0"
subtle = "2.6"
zeroize = { version = "1.8.2", features = ["derive"] }
zeroize = { version = "1.9.0", features = ["derive"] }
# Time and Date
chrono = { version = "0.4.45", features = ["serde"] }
humantime = "2.3.0"
jiff = { version = "0.2.28", features = ["serde"] }
time = { version = "=0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
time = { version = "0.3.49", features = ["std", "parsing", "formatting", "macros", "serde"] }
# Database
deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] }
@@ -209,7 +209,7 @@ atoi = "3.0.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.8.18" }
aws-credential-types = { version = "1.2.14" }
aws-sdk-s3 = { version = "1.135.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-sdk-s3 = { version = "1.136.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-smithy-http-client = { version = "1.1.13", default-features = false, features = ["default-client", "rustls-aws-lc"] }
aws-smithy-types = { version = "1.5.0" }
base64 = "0.22.1"
@@ -264,7 +264,7 @@ reed-solomon-erasure = { version = "6.0", default-features = false, features = [
reed-solomon-simd = "3.1.0"
regex = { version = "1.12.4" }
rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket"] }
redis = { version = "1.2.2", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
redis = { version = "1.2.3", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
rustix = { version = "1.1.4", features = ["fs"] }
rust-embed = { version = "8.11.0" }
rustc-hash = { version = "2.1.2" }
@@ -276,7 +276,7 @@ smallvec = { version = "1.15.2", features = ["serde"] }
smartstring = "1.0.1"
snafu = "0.9.1"
snap = "1.1.1"
starshard = { version = "2.2.0", features = ["rayon", "async", "serde"] }
starshard = { version = "2.2.1", features = ["rayon", "async", "serde"] }
strum = { version = "0.28.0", features = ["derive"] }
sysinfo = "0.39.3"
temp-env = "0.3.6"
crates/audit/src/observability.rs
+3 -3
View File
@@ -160,7 +160,7 @@ impl AuditMetrics {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
state = "config_reloaded",
"Audit observability state updated"
"audit observability state"
);
}
@@ -173,7 +173,7 @@ impl AuditMetrics {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
state = "system_started",
"Audit observability state updated"
"audit observability state"
);
}
@@ -249,7 +249,7 @@ impl AuditMetrics {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
state = "metrics_reset",
"Audit observability state updated"
"audit observability state"
);
}
crates/audit/src/pipeline.rs
+13 -13
View File
@@ -266,7 +266,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id,
state = "enabled",
"Changed audit target state"
"audit target state"
);
Ok(())
} else {
@@ -283,7 +283,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id,
state = "disabled",
"Changed audit target state"
"audit target state"
);
Ok(())
} else {
@@ -300,7 +300,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id,
state = "removed",
"Changed audit target state"
"audit target state"
);
Ok(())
} else {
@@ -323,7 +323,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id,
state = "upserted",
"Changed audit target state"
"audit target state"
);
Ok(())
}
@@ -349,7 +349,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(),
replay_key = %key,
"Delivered queued audit event"
"audit replay delivery"
);
observability::record_target_success();
}
@@ -361,7 +361,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(),
reason = "not_connected",
"Retrying queued audit event delivery"
"audit replay delivery"
);
}
rustfs_targets::TargetError::Timeout(_) => {
@@ -371,7 +371,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(),
reason = "timeout",
"Retrying queued audit event delivery"
"audit replay delivery"
);
}
_ => {}
@@ -383,7 +383,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(),
reason = %reason,
"Dropped queued audit payload"
"audit replay delivery"
);
observability::record_target_failure();
}
@@ -395,7 +395,7 @@ impl AuditRuntimeFacade {
target_id = %target.id(),
error = %error,
reason = "permanent_failure",
"Queued audit payload failed permanently"
"audit replay delivery"
);
target.record_final_failure();
observability::record_target_failure();
@@ -408,7 +408,7 @@ impl AuditRuntimeFacade {
target_id = %target.id(),
replay_key = %key,
reason = "retry_exhausted",
"Dropped queued audit payload after retry exhaustion"
"audit replay delivery"
);
target.record_final_failure();
observability::record_target_failure();
@@ -422,7 +422,7 @@ impl AuditRuntimeFacade {
replay_key = %key,
error = %error,
reason = "unreadable_entry",
"Skipped unreadable audit store entry"
"audit replay delivery"
);
}
}
@@ -436,7 +436,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id,
replay_enabled = true,
"Audit replay stream started"
"audit replay stream"
);
} else {
debug!(
@@ -446,7 +446,7 @@ impl AuditRuntimeFacade {
target_id = %target_id,
replay_enabled = false,
reason = "no_store_configured",
"Audit replay stream skipped"
"audit replay stream"
);
}
}),
crates/audit/src/registry.rs
+3 -3
View File
@@ -198,7 +198,7 @@ impl AuditRegistry {
target_type = %target_type,
target_id = %target_id,
registry_key = %key,
"Created audit target registry key"
"audit target registry state"
);
key.to_string()
}
@@ -221,7 +221,7 @@ impl AuditRegistry {
target_type = %target_type,
target_id = %target_id,
state = "enabled",
"Audit target registry state changed"
"audit target registry state"
);
Ok(())
} else {
@@ -250,7 +250,7 @@ impl AuditRegistry {
target_type = %target_type,
target_id = %target_id,
state = "disabled",
"Audit target registry state changed"
"audit target registry state"
);
Ok(())
} else {
crates/audit/src/system.rs
+8 -8
View File
@@ -120,7 +120,7 @@ impl AuditSystem {
subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "targets_created",
target_count = targets.len(),
"Created audit targets"
"audit system state"
);
let activation = self.runtime_facade().activate_targets_with_replay(targets).await;
@@ -159,7 +159,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "starting",
"Starting audit system"
"audit system state"
);
// Record system start
@@ -268,7 +268,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "stopping",
"Stopping audit system"
"audit system state"
);
// Stop all stream tasks first
@@ -441,7 +441,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "reloading",
"Reloading audit configuration"
"audit config reload"
);
observability::record_config_reload();
@@ -465,7 +465,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "reloaded",
"Reloaded audit configuration"
"audit config reload"
);
Ok(())
}
@@ -513,7 +513,7 @@ fn info_audit_state(state: &str, reason: Option<&str>, target_count: Option<usiz
state,
reason = reason.unwrap_or_default(),
target_count = target_count.unwrap_or_default(),
"Changed audit system state"
"audit system state"
);
}
@@ -526,7 +526,7 @@ fn debug_audit_state(state: &str, reason: Option<&str>, error: Option<&str>, tar
reason = reason.unwrap_or_default(),
error = error.unwrap_or_default(),
target_count,
"Observed audit system state"
"audit system state"
);
}
@@ -537,7 +537,7 @@ fn warn_audit_state(state: &str, reason: Option<&str>) {
subsystem = LOG_SUBSYSTEM_SYSTEM,
state,
reason = reason.unwrap_or_default(),
"Audit system state transition skipped"
"audit system state"
);
}
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+9 -9
View File
@@ -77,7 +77,7 @@ use tokio::sync::mpsc::{Receiver, Sender};
use tokio::sync::{RwLock, mpsc};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
use tracing::{debug, error, warn};
use uuid::Uuid;
use xxhash_rust::xxh64;
@@ -762,13 +762,13 @@ impl TransitionState {
"Transition compensation backfill failed"
);
} else {
info!(
debug!(
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %bucket,
state = "completed",
"Completed transition compensation backfill"
"Transition compensation completed"
);
}
@@ -958,7 +958,7 @@ impl TransitionState {
pub async fn init(api: Arc<ECStore>) {
let (configured, absolute_max, n) = resolve_transition_worker_count();
info!(
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
@@ -968,7 +968,7 @@ impl TransitionState {
transition_queue_capacity = GLOBAL_TransitionState.transition_queue_capacity,
transition_queue_send_timeout_ms = GLOBAL_TransitionState.transition_queue_send_timeout.as_millis() as u64,
state = "configured",
"Lifecycle worker state resolved"
"Lifecycle worker configuration resolved"
);
//let mut transition_state = GLOBAL_TransitionState.write().await;
@@ -1158,7 +1158,7 @@ impl TransitionState {
GLOBAL_TransitionState.num_workers.store(current_workers, Ordering::SeqCst);
GLOBAL_TransitionState.record_scanner_transition_state();
info!(
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
@@ -1169,7 +1169,7 @@ impl TransitionState {
current_transition_workers = current_workers,
pruned_finished_transition_workers = pruned_finished_workers,
state = "resized",
"Lifecycle worker state updated"
"Lifecycle worker pool resized"
);
}
}
@@ -2008,14 +2008,14 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
Arc::new(move || {
let tier = tier.clone();
Box::pin(async move {
info!(
debug!(
event = EVENT_LIFECYCLE_TIER_AUDIT,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
tier = %tier,
bytes = bytes,
state = "transition_completed",
"Lifecycle tier transition audit completed"
"Lifecycle tier transition recorded"
);
})
})
crates/ecstore/src/cache_value/metacache_set.rs
+87 -11
View File
@@ -29,7 +29,11 @@ use tokio::io::AsyncRead;
use tokio::spawn;
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, warn};
use tracing::{error, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
const EVENT_METACACHE_LISTING: &str = "metacache_listing";
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
pub type PartialFn =
@@ -114,6 +118,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
return Err(DiskError::ErasureReadQuorum);
}
let log_bucket = opts.bucket.clone();
let log_path = opts.path.clone();
let mut jobs: Vec<tokio::task::JoinHandle<std::result::Result<(), DiskError>>> = Vec::new();
let mut readers = Vec::with_capacity(opts.disks.len());
let fds = opts.fallback_disks.iter().flatten().cloned().collect::<VecDeque<_>>();
@@ -180,7 +187,17 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
match disk.walk_dir(wakl_opts, &mut wr).await {
Ok(_res) => {}
Err(err) => {
info!("walk dir err {:?}", &err);
warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "walk_dir_failed",
error = ?err,
"Metacache walk_dir failed"
);
last_err = Some(err);
need_fallback = true;
}
@@ -205,7 +222,16 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
}
let Some(disk) = disk_op else {
warn!("list_path_raw: fallback disk is none");
warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_disk_missing",
"Metacache fallback disk missing"
);
let err = last_err.unwrap_or(DiskError::DiskNotFound);
record_producer_error(&producer_errs_clone, disk_idx, &err);
return Err(err);
@@ -234,7 +260,17 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
last_err = None;
}
Err(err) => {
error!("walk dir2 err {:?}", &err);
error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_walk_dir_failed",
error = ?err,
"Metacache fallback walk_dir failed"
);
last_err = Some(err);
}
}
@@ -355,11 +391,15 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
)
.increment(1);
warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
drive = %endpoint,
bucket = %opts.bucket,
path = %opts.path,
timeout_ms = peek_timeout.as_millis(),
"list_path_raw reader peek timed out; excluding drive from current merge"
state = "peek_timed_out",
"Metacache reader peek timed out"
);
let (detached_rd, write_half) = tokio::io::duplex(1);
drop(write_half);
@@ -439,8 +479,14 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
});
error!(
"list_path_raw: has_err > 0 && has_err > opts.disks.len() - opts.min_disks break, err: {:?}",
&combined_err.join(", ")
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts.bucket,
path = %opts.path,
state = "quorum_failed",
error = %combined_err.join(", "),
"Metacache listing quorum failed"
);
return Err(DiskError::other(combined_err.join(", ")));
}
@@ -493,7 +539,16 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
});
if let Err(err) = revjob.await.map_err(std::io::Error::other)? {
error!("list_path_raw: revjob err {:?}", err);
error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %log_bucket,
path = %log_path,
state = "merge_job_failed",
error = ?err,
"Metacache merge job failed"
);
cancel_rx.cancel();
for job in jobs {
job.abort();
@@ -521,9 +576,23 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
Ok(Ok(())) => {}
Ok(Err(err)) => {
if matches!(err, DiskError::FileNotFound | DiskError::VolumeNotFound) {
warn!("list_path_raw producer missing path {:?}", err);
warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
state = "producer_missing_path",
error = ?err,
"Metacache producer missing path"
);
} else {
error!("list_path_raw producer err {:?}", err);
error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
state = "producer_failed",
error = ?err,
"Metacache producer failed"
);
}
job_errs.push(err);
}
@@ -531,7 +600,14 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
if err.is_cancelled() {
continue;
}
error!("list_path_raw join err {:?}", err);
error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
state = "producer_join_failed",
error = ?err,
"Metacache producer join failed"
);
job_errs.push(err.into());
}
}
crates/ecstore/src/disk/local.rs
+28 -4
View File
@@ -472,7 +472,15 @@ fn resolve_local_disk_root(ep_path: &str) -> Result<PathBuf> {
impl LocalDisk {
pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
debug!("Creating local disk");
debug!(
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
endpoint = %ep,
state = "create_started",
cleanup,
"Local disk creation started"
);
let endpoint_path = ep.get_file_path();
let root = resolve_local_disk_root(&endpoint_path).inspect_err(|err| {
log_startup_disk_error("resolve_local_disk_root", Path::new(&endpoint_path), err);
@@ -501,13 +509,21 @@ impl LocalDisk {
root = ?root,
state = "failed",
error = ?err,
"Disk local startup cleanup failed"
"Local disk startup cleanup failed"
);
}
// Use optimized path resolution instead of absolutize_virtually
let format_path = root.join(RUSTFS_META_BUCKET).join(super::FORMAT_CONFIG_FILE);
debug!("format_path: {:?}", format_path);
debug!(
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
root = ?root,
format_path = ?format_path,
state = "format_path_resolved",
"Local disk format path resolved"
);
let (format_data, format_meta) = read_file_exists(&format_path).await.inspect_err(|err| {
log_startup_disk_error("read_format_json", &format_path, err);
})?;
@@ -639,7 +655,15 @@ impl LocalDisk {
let root = disk.root.clone();
tokio::spawn(Self::cleanup_deleted_objects_loop(root, exit_rx));
debug!("LocalDisk created: {:?}", disk);
debug!(
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
endpoint = %disk.endpoint,
root = ?disk.root,
state = "created",
"Local disk created"
);
Ok(disk)
}
crates/ecstore/src/pools.rs
+323 -43
View File
@@ -71,6 +71,12 @@ use time::{Duration, OffsetDateTime};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_POOLS: &str = "pools";
const EVENT_DECOMMISSION_STATE: &str = "decommission_state";
const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket";
const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry";
pub const POOL_META_NAME: &str = "pool.bin";
pub const POOL_META_FORMAT: u16 = 1;
pub const POOL_META_VERSION: u16 = 1;
@@ -1280,7 +1286,15 @@ impl ECStore {
take_decommission_canceler(cancelers.as_mut_slice(), idx)
};
if !cancel_decommission_canceler(canceler) {
warn!("decommission_cancel: no active canceler found for pool {}", idx);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "cancel_skipped",
reason = "no_active_canceler",
"Decommission cancel skipped"
);
}
if should_reload_pool_meta && let Some(notification_sys) = get_global_notification_sys() {
@@ -1338,7 +1352,15 @@ impl ECStore {
let store = store.clone();
tokio::spawn(async move {
if let Err(err) = store.do_decommission_in_routine(canceler, idx).await {
error!("decommission: routine failed for idx {}: {err}", idx);
error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "routine_failed",
error = %err,
"Decommission routine failed"
);
}
});
}
@@ -1350,7 +1372,14 @@ impl ECStore {
pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> {
let indices = dedup_indices(&indices);
warn!("decommission: {:?}", indices);
info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_indices = ?indices,
state = "requested",
"Decommission requested"
);
validate_start_decommission_request(&indices, self.single_pool())?;
ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?;
@@ -1363,8 +1392,13 @@ impl ECStore {
for idx in indices {
if let Err(cancel_err) = self.decommission_cancel(idx).await {
error!(
"decommission: failed to rollback decommission state for idx {} after spawn error: {:?}",
idx, cancel_err
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "rollback_failed",
error = ?cancel_err,
"Decommission rollback failed after spawn error"
);
if rollback_err.is_none() {
rollback_err = Some(Error::other(format!("decommission rollback failed for idx {idx}: {cancel_err}")));
@@ -1390,10 +1424,28 @@ impl ECStore {
lock_retention: Option<DefaultRetention>,
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
) -> Result<()> {
warn!("decommission_entry: {} {}", &bucket, &entry.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "started",
"Decommission entry started"
);
wk.give().await;
if entry.is_dir() {
warn!("decommission_entry: skip dir {}", &entry.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "skipped_directory",
"Decommission entry skipped directory"
);
return Ok(());
}
@@ -1426,7 +1478,16 @@ impl ECStore {
if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) {
//
decommissioned += 1;
info!("decommission_pool: DELETE marked object with no other non-current versions will be skipped");
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "skipped_delete_marker",
"Decommission delete marker skipped"
);
continue;
}
@@ -1447,8 +1508,16 @@ impl ECStore {
{
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) {
warn!(
"decommission_pool: ignore delete-marker copy for {}/{} version {:?}: {:?}",
&bucket, &version.name, &version_id, &err
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
version_id = ?version_id,
state = "ignored_delete_marker_copy",
error = ?err,
"Decommission delete marker copy ignored"
);
ignore = true;
cleanup_ignored = true;
@@ -1463,7 +1532,16 @@ impl ECStore {
if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
decommissioned += 1;
}
info!("decommission_pool: ignore {}", &version.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "ignored",
"Decommission entry ignored"
);
continue;
}
@@ -1483,9 +1561,17 @@ impl ECStore {
decommissioned += 1;
}
info!(
"decommission_pool: DecomCopyDeleteMarker {} {} {:?} {:?}",
&bucket, &version.name, &version_id, error
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
version_id = ?version_id,
result = ?error,
state = "delete_marker_copied",
"Decommission delete marker copied"
);
continue;
}
@@ -1571,8 +1657,15 @@ impl ECStore {
}
warn!(
"decommission_pool: decommission_object done {}/{} {}",
&bucket_name, &object_name, &version.name
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket_name,
object = %object_name,
version = %version.name,
state = "object_migrated",
"Decommission object migrated"
);
failure = false;
@@ -1583,7 +1676,16 @@ impl ECStore {
if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
decommissioned += 1;
}
info!("decommission_pool: ignore {}", &version.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "ignored",
"Decommission entry ignored"
);
continue;
}
@@ -1624,12 +1726,17 @@ impl ECStore {
resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())?
} else if decommissioned != fivs.versions.len() || expired > 0 {
warn!(
"decommission_pool: source object retained for {}/{} because only {}/{} versions were decommissioned and {} expired by lifecycle",
&bucket,
&entry.name,
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
decommissioned,
fivs.versions.len(),
expired
total_versions = fivs.versions.len(),
expired,
state = "source_retained",
"Decommission source object retained"
);
}
@@ -1667,7 +1774,16 @@ impl ECStore {
}
}
warn!("decommission_pool: decommission_entry done {} {}", &bucket, &entry.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "completed",
"Decommission entry completed"
);
Ok(())
}
@@ -1705,7 +1821,16 @@ impl ECStore {
for (set_idx, set) in pool.disk_set.iter().enumerate() {
wk.clone().take().await;
warn!("decommission_pool: decommission_pool {} {}", set_idx, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_idx,
bucket = %bi.name,
state = "listing_worker_started",
"Decommission listing worker started"
);
let decommission_entry: ListCallback = Arc::new({
let this = Arc::clone(self);
@@ -1753,23 +1878,69 @@ impl ECStore {
let worker = tokio::spawn(async move {
loop {
if rx_clone.is_cancelled() {
warn!("decommission_pool: cancel {}", set_id);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_worker_cancelled",
"Decommission listing worker cancelled"
);
break;
}
warn!("decommission_pool: list_objects_to_decommission {} {}", set_id, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_started",
"Decommission listing started"
);
match set
.list_objects_to_decommission(rx_clone.clone(), bi.clone(), decommission_entry.clone())
.await
{
Ok(_) => {
warn!("decommission_pool: list_objects_to_decommission {} done", set_id);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_completed",
"Decommission listing completed"
);
break;
}
Err(err) => {
error!("decommission_pool: list_objects_to_decommission {} err {:?}", set_id, &err);
error!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_failed",
error = ?err,
"Decommission listing failed"
);
if is_err_bucket_not_found(&err) {
warn!("decommission_pool: list_objects_to_decommission {} volume not found", set_id);
warn!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_bucket_missing",
"Decommission listing bucket missing"
);
break;
}
@@ -1783,7 +1954,15 @@ impl ECStore {
listing_workers.push((set_id, worker));
}
warn!("decommission_pool: decommission_pool wait {} {}", idx, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "waiting_for_workers",
"Decommission waiting for workers"
);
let mut listing_worker_error = None;
for (set_id, worker) in listing_workers {
@@ -1807,11 +1986,28 @@ impl ECStore {
}
if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
warn!("decommission_pool: canceled after wait {} {}", idx, &bi.name);
warn!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "cancelled_after_wait",
error = %err,
"Decommission bucket cancelled after wait"
);
return Err(err);
}
warn!("decommission_pool: decommission_pool done {} {}", idx, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "completed",
"Decommission bucket completed"
);
Ok(())
}
@@ -1821,7 +2017,14 @@ impl ECStore {
defer!(|| async {
let mut cancelers = self.decommission_cancelers.write().await;
if take_decommission_canceler(cancelers.as_mut_slice(), idx).is_none() {
warn!("decommission: canceler already cleared for pool {}", idx);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "canceler_already_cleared",
"Decommission canceler already cleared"
);
}
});
@@ -1830,7 +2033,14 @@ impl ECStore {
let (final_state, canceled, cmd_line) = {
let pool_meta = self.pool_meta.read().await;
let Some(pool) = pool_meta.pools.get(idx) else {
error!("decommission: pool metadata missing for idx {}", idx);
error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "pool_metadata_missing",
"Decommission pool metadata missing"
);
return Err(Error::other(format!(
"failed to resolve decommission final state: pool metadata missing for idx {idx}"
)));
@@ -1849,29 +2059,75 @@ impl ECStore {
};
if let Err(err) = result {
error!("decom err {:?}", &err);
error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "background_failed",
error = ?err,
"Decommission background routine failed"
);
if is_err_operation_canceled(&err) || should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
warn!("decommission: canceled for pool {}, preserving canceled state", cmd_line);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "cancelled_preserved",
"Decommission cancelled; preserving canceled state"
);
return Ok(());
}
resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?;
warn!("decommission: decommission_failed {}", idx);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "marked_failed",
"Decommission marked failed"
);
return Ok(());
}
warn!("decommission: decommission_in_background complete {}", idx);
debug!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "background_complete",
"Decommission background routine completed"
);
if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
warn!("decommission: canceled for pool {}, skipping terminal state overwrite", cmd_line);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "terminal_state_preserved",
"Decommission terminal state preserved after cancellation"
);
return Ok(());
}
match final_state {
DecommissionFinalState::Complete => {
warn!("Decommissioning complete for pool {}, verifying for any pending objects", cmd_line);
debug!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "verifying_completion",
"Decommission completion verification started"
);
if let Err(err) = self.check_after_decommission(idx).await {
resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?;
return Err(Error::other(format!(
@@ -1879,16 +2135,40 @@ impl ECStore {
)));
}
warn!("Decommissioning complete for pool {}, marking completed state", cmd_line);
info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "marking_completed",
"Decommission marking completed state"
);
resolve_decommission_terminal_mark_result(self.complete_decommission(idx).await, "completed", &cmd_line)?;
}
DecommissionFinalState::Failed => {
warn!("Decommissioning finished with failed items for pool {}, marking failed state", cmd_line);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "marking_failed",
"Decommission marking failed state"
);
resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?;
}
}
warn!("Decommissioning complete for pool {}", cmd_line);
info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "completed",
"Decommission completed"
);
Ok(())
}
crates/ecstore/src/rebalance.rs
+176 -30
View File
@@ -858,7 +858,14 @@ impl ECStore {
"init_rebalance_meta",
)?;
info!("init_rebalance_meta: rebalance meta saved");
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "metadata_initialized",
bucket_count = bucktes.len(),
"Rebalance metadata initialized"
);
let id = meta.id.clone();
@@ -898,9 +905,16 @@ impl ECStore {
#[tracing::instrument(skip(self))]
pub async fn next_rebal_bucket(&self, pool_index: usize) -> Result<Option<String>> {
info!("next_rebal_bucket: pool_index: {}", pool_index);
let rebalance_meta = self.rebalance_meta.read().await;
info!("next_rebal_bucket: rebalance_meta: {:?}", rebalance_meta);
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
has_meta = rebalance_meta.is_some(),
state = "next_bucket_lookup",
"Rebalance next bucket lookup"
);
resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index)
}
@@ -931,20 +945,38 @@ impl ECStore {
let rebalance_meta = self.rebalance_meta.read().await;
if let Some(meta) = rebalance_meta.as_ref() {
meta.pool_stats.iter().enumerate().for_each(|(i, v)| {
info!(
"is_rebalance_started: pool_index: {}, participating: {:?}, status: {:?}",
i, v.participating, v.info.status
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = i,
participating = v.participating,
status = ?v.info.status,
state = "status_inspected",
"Rebalance status inspected"
);
});
let started = is_rebalance_conflicting_with_decommission(meta);
if started {
info!("is_rebalance_started: rebalance started");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "running",
"Rebalance is running"
);
return true;
}
}
info!("is_rebalance_started: rebalance not started");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "not_running",
"Rebalance is not running"
);
false
}
@@ -1108,15 +1140,23 @@ impl ECStore {
workers_started += 1;
tokio::spawn(async move {
if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx).await {
error!("Rebalance failed for pool {}: {}", pool_idx, err);
error!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = pool_idx,
state = "pool_failed",
error = %err,
"Rebalance pool failed"
);
} else {
info!(
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = pool_idx,
state = "completed",
"Completed rebalance pool"
"Rebalance pool completed"
);
}
});
@@ -1139,7 +1179,8 @@ impl ECStore {
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "started",
"Started rebalance"
worker_count = workers_started,
"Rebalance started"
);
Ok(())
}
@@ -1240,7 +1281,7 @@ impl ECStore {
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "pool_started",
"Started rebalance worker"
"Rebalance worker started"
);
let mut final_result: Result<()> = Ok(());
let mut deferred_buckets = HashSet::new();
@@ -1267,7 +1308,15 @@ impl ECStore {
let next_bucket = match self.next_rebal_bucket(pool_index).await {
Ok(bucket) => bucket,
Err(err) => {
error!("next_rebal_bucket failed for pool {}: {:?}", pool_index, err);
error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "next_bucket_failed",
error = ?err,
"Rebalance next bucket lookup failed"
);
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1294,7 +1343,16 @@ impl ECStore {
) {
Ok(outcome) => outcome,
Err(err) => {
error!("Error rebalancing bucket {}: {:?}", bucket, err);
error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "bucket_failed",
error = ?err,
"Rebalance bucket failed"
);
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1308,7 +1366,16 @@ impl ECStore {
let err = Error::other(format!(
"rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
));
error!("Error rebalancing bucket {}: {:?}", bucket, err);
error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "bucket_deferred_repeatedly",
error = ?err,
"Rebalance bucket failed after repeated deferral"
);
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1327,7 +1394,16 @@ impl ECStore {
"Deferred rebalance bucket after transient object failures"
);
if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await {
error!("defer_rebalance_bucket failed for pool {}: {:?}", pool_index, err);
error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "defer_failed",
error = ?err,
"Rebalance bucket defer failed"
);
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1347,7 +1423,15 @@ impl ECStore {
"Completed rebalance bucket"
);
if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await {
error!("bucket_rebalance_done failed for pool {}: {:?}", pool_index, err);
error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "bucket_done_mark_failed",
error = ?err,
"Rebalance bucket completion mark failed"
);
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1374,7 +1458,7 @@ impl ECStore {
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "pool_done",
"Finished rebalance worker"
"Rebalance worker finished"
);
if final_result.is_ok()
@@ -1388,7 +1472,14 @@ impl ECStore {
{
final_result = Err(err);
}
info!("Pool {} rebalancing is done2", pool_index);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "pool_result_returned",
"Rebalance worker result returned"
);
final_result
}
@@ -2591,7 +2682,15 @@ impl ECStore {
ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
// Placeholder for actual bucket rebalance logic
info!("Rebalancing bucket {} in pool {}", bucket, pool_index);
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "entry_scan_started",
"Rebalance bucket entry scan started"
);
// TODO: other config
// if bucket != RUSTFS_META_BUCKET{
@@ -2765,9 +2864,14 @@ impl ECStore {
let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?;
info!(
"save_rebalance_stats: save rebalance meta, pool_idx: {}, opt: {:?}, meta: {:?}",
pool_idx, opt, meta_to_save
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = pool_idx,
save_opt = ?opt,
state = "metadata_save_requested",
"Rebalance metadata save requested"
);
let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}");
resolve_rebalance_meta_save_result(
@@ -2838,12 +2942,27 @@ impl SetDisks {
bucket: String,
cb: ListCallback,
) -> Result<()> {
info!("list_objects_to_rebalance: start list_objects_to_rebalance");
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
bucket = %bucket,
state = "started",
"Rebalance listing started"
);
// Placeholder for actual object listing logic
let (disks, _) = self.get_online_disks_with_healing(false).await;
ensure_rebalance_listing_disks_available(!disks.is_empty(), &bucket)?;
info!("list_objects_to_rebalance: get online disks with healing");
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
bucket = %bucket,
disk_count = disks.len(),
state = "disks_resolved",
"Rebalance listing disks resolved"
);
let listing_quorum = self.set_drive_count.div_ceil(2);
let resolver = MetadataResolutionParams {
@@ -2863,7 +2982,14 @@ impl SetDisks {
min_disks: listing_quorum,
skip_walkdir_total_timeout: true,
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
info!("list_objects_to_rebalance: agreed: {:?}", &entry.name);
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
entry = %entry.name,
state = "agreed_entry",
"Rebalance listing agreed entry"
);
Box::pin(cb1(entry))
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
@@ -2873,11 +2999,24 @@ impl SetDisks {
match entries.resolve(resolver) {
Some(entry) => {
info!("list_objects_to_rebalance: list_objects_to_decommission get {}", &entry.name);
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
entry = %entry.name,
state = "resolved_partial_entry",
"Rebalance listing resolved partial entry"
);
Box::pin(async move { cb(entry).await })
}
None => {
info!("list_objects_to_rebalance: list_objects_to_decommission get none");
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "partial_entry_missing",
"Rebalance listing partial entry missing"
);
Box::pin(async {})
}
}
@@ -2887,7 +3026,14 @@ impl SetDisks {
)
.await?;
info!("list_objects_to_rebalance: list_objects_to_rebalance done");
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
bucket = %bucket,
state = "completed",
"Rebalance listing completed"
);
Ok(())
}
}
crates/ecstore/src/rpc/remote_disk.rs
+363 -37
View File
@@ -65,7 +65,7 @@ use tokio::{
};
use tokio_util::sync::CancellationToken;
use tonic::{Request, service::interceptor::InterceptedService, transport::Channel};
use tracing::{debug, info, warn};
use tracing::{debug, warn};
use uuid::Uuid;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -76,6 +76,10 @@ enum FailureHealthAction {
const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2;
const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20);
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64>
where
@@ -275,7 +279,16 @@ impl RemoteDisk {
if initial_probe_ok {
health.record_operation_success(&endpoint, "connectivity_probe_success");
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!("Remote disk health check failed for {}: marking as faulty", addr);
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "initial_probe_failed",
result = "mark_faulty",
"Remote disk initial health probe failed"
);
// Start recovery monitoring
let health_clone = Arc::clone(&health);
@@ -291,7 +304,15 @@ impl RemoteDisk {
loop {
tokio::select! {
_ = cancel_token.cancelled() => {
debug!("Health monitoring cancelled for remote disk: {}", addr);
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "monitor_cancelled",
"Remote disk health monitor cancelled"
);
return;
}
_ = interval.tick() => {
@@ -320,7 +341,16 @@ impl RemoteDisk {
if Self::perform_connectivity_check(&addr).await.is_ok() {
health.record_operation_success(&endpoint, "connectivity_probe_success");
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!("Remote disk health check failed for {}: marking as faulty", addr);
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "probe_failed",
result = "mark_faulty",
"Remote disk health probe failed"
);
// Start recovery monitoring
let health_clone = Arc::clone(&health);
@@ -354,9 +384,25 @@ impl RemoteDisk {
_ = interval.tick() => {
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
info!("Remote disk recovery probe succeeded: {}", addr);
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "recovery_probe_succeeded",
"Remote disk recovery probe succeeded"
);
if became_online {
info!("Remote disk recovered: {}", addr);
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "recovered",
"Remote disk recovered"
);
return;
}
} else {
@@ -469,7 +515,16 @@ impl RemoteDisk {
{
// Check if disk is faulty
if self.health.is_faulty() {
warn!("remote disk {} health is faulty, returning error", self.to_string());
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
op,
state = "faulty_short_circuit",
"Remote disk operation short-circuited by faulty state"
);
return Err(DiskError::FaultyDisk);
}
@@ -519,10 +574,14 @@ impl RemoteDisk {
self.mark_faulty_and_evict("operation_timeout").await;
}
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
op,
timeout_ms = timeout_duration.as_millis(),
state = "timeout",
"Remote disk operation timed out"
);
Err(DiskError::Timeout)
@@ -547,11 +606,15 @@ impl RemoteDisk {
)
.increment(1);
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
op,
timeout_ms = timeout_duration.as_millis(),
"Remote disk operation returned a network-like error"
state = "network_like_error",
"Remote disk operation returned network-like error"
);
if failure_health_action == FailureHealthAction::MarkFailure {
self.mark_faulty_and_evict("operation_network_error").await;
@@ -574,13 +637,26 @@ impl RemoteDisk {
.increment(1);
if transitioned_to_offline {
warn!(
"Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}",
self.endpoint, self.addr, reason
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
reason,
state = "marked_faulty",
"Remote disk marked faulty"
);
} else {
warn!(
"Remote disk marked suspect after timeout: endpoint={}, addr={}, reason={}, state={:?}",
self.endpoint, self.addr, reason, state
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
reason,
runtime_state = ?state,
state = "marked_suspect",
"Remote disk marked suspect"
);
}
counter!(
@@ -589,11 +665,15 @@ impl RemoteDisk {
"reason" => reason.to_string()
)
.increment(1);
info!(
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
reason,
"Evicting cached remote disk connection after fault transition"
state = "evict_cached_connection",
"Remote disk cached connection evicted"
);
evict_failed_connection(&self.addr).await;
}
@@ -705,7 +785,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn make_volume(&self, volume: &str) -> Result<()> {
info!("make_volume");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
op = "make_volume",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -733,7 +822,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
info!("make_volumes");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume_count = volumes.len(),
op = "make_volumes",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -761,7 +859,15 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
info!("list_volumes");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
op = "list_volumes",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -794,7 +900,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
info!("stat_volume");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
op = "stat_volume",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -824,7 +939,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn delete_volume(&self, volume: &str) -> Result<()> {
info!("delete_volume {}/{}", self.endpoint.to_string(), volume);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
op = "delete_volume",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -909,7 +1033,17 @@ impl DiskAPI for RemoteDisk {
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()> {
info!("delete_version");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "delete_version",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -946,7 +1080,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> {
info!("delete_versions");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
version_count = versions.len(),
op = "delete_versions",
state = "started",
"Remote disk RPC started"
);
if self.health.is_faulty() {
return vec![Some(DiskError::FaultyDisk); versions.len()];
@@ -1041,7 +1185,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
info!("delete_paths");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path_count = paths.len(),
op = "delete_paths",
state = "started",
"Remote disk RPC started"
);
let paths = paths.to_owned();
self.execute_with_timeout(
@@ -1071,7 +1225,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
info!("write_metadata {}/{}", volume, path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "write_metadata",
state = "started",
"Remote disk RPC started"
);
let file_info = serde_json::to_string(&fi)?;
let file_info_bin = encode_msgpack(&fi)?;
@@ -1134,7 +1298,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
info!("update_metadata");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "update_metadata",
state = "started",
"Remote disk RPC started"
);
let file_info = serde_json::to_string(&fi)?;
let opts_str = serde_json::to_string(&opts)?;
let file_info_bin = encode_msgpack(&fi)?;
@@ -1180,7 +1354,18 @@ impl DiskAPI for RemoteDisk {
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
info!("read_version");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
version_id,
op = "read_version",
state = "started",
"Remote disk RPC started"
);
let opts_str = serde_json::to_string(opts)?;
let opts_bin = encode_msgpack(opts)?;
@@ -1217,7 +1402,18 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
info!("read_xl {}/{}/{}", self.endpoint.to_string(), volume, path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
read_data,
op = "read_xl",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1257,7 +1453,19 @@ impl DiskAPI for RemoteDisk {
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
info!("rename_data {}/{}/{}/{}", self.addr, self.endpoint.to_string(), dst_volume, dst_path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
src_volume,
src_path,
dst_volume,
dst_path,
op = "rename_data",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout_for_op(
"rename_data",
@@ -1325,7 +1533,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self, wr))]
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
info!("walk_dir {}", self.endpoint.to_string());
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
bucket = %opts.bucket,
base_dir = %opts.base_dir,
op = "walk_dir",
state = "started",
"Remote disk RPC started"
);
let disk = self.disk_ref().await;
let body = serde_json::to_vec(&opts)?;
@@ -1443,7 +1661,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
info!("append_file {}/{}", volume, path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "append_file",
state = "started",
"Remote disk RPC started"
);
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
@@ -1487,7 +1715,19 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
info!("rename_file");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
src_volume,
src_path,
dst_volume,
dst_path,
op = "rename_file",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1518,7 +1758,19 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> {
info!("rename_part {}/{}", src_volume, src_path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
src_volume,
src_path,
dst_volume,
dst_path,
op = "rename_part",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1550,7 +1802,19 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
recursive = opt.recursive,
immediate = opt.immediate,
op = "delete",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1581,7 +1845,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("verify_file");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "verify_file",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1614,6 +1888,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result<Vec<ObjectPartInfo>> {
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
bucket,
path_count = paths.len(),
op = "read_parts",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
let mut client = self
@@ -1642,7 +1927,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("check_parts");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "check_parts",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1675,7 +1970,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
bucket = %req.bucket,
prefix = %req.prefix,
op = "read_multiple",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1721,7 +2026,18 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> {
info!("write_all");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
bytes = data.len(),
op = "write_all",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1779,7 +2095,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes> {
info!("read_all {}/{}", volume, path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "read_all",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
crates/ecstore/src/set_disk.rs
+110 -10
View File
@@ -130,6 +130,12 @@ use tracing::error;
use tracing::{debug, info, warn};
use uuid::Uuid;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_SET_DISK: &str = "set_disk";
const EVENT_SET_DISK_MULTIPART: &str = "set_disk_multipart";
const EVENT_SET_DISK_WRITE: &str = "set_disk_write";
const EVENT_SET_DISK_HEAL: &str = "set_disk_heal";
use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
@@ -948,7 +954,16 @@ impl ObjectIO for SetDisks {
)
.await
{
error!("get_object_with_fileinfo {bucket}/{object} err {:?}", e);
error!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
state = "read_pipeline_failed",
error = ?e,
"Set disk object read pipeline failed"
);
};
});
@@ -1065,7 +1080,15 @@ impl ObjectIO for SetDisks {
{
Ok(writer) => (Some(writer), None),
Err(err) => {
warn!("create_bitrot_writer disk {}, err {:?}, skipping operation", disk.to_string(), err);
warn!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
disk = ?disk,
state = "bitrot_writer_skipped",
error = ?err,
"Set disk bitrot writer skipped"
);
(None, Some(err))
}
}
@@ -1085,7 +1108,18 @@ impl ObjectIO for SetDisks {
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
if nil_count < write_quorum {
error!("not enough disks to write: {:?}", errors);
error!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
write_quorum,
available_writers = nil_count,
state = "write_quorum_unavailable",
error = ?errors,
"Set disk write quorum unavailable"
);
if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(to_object_err(write_err.into(), vec![bucket, object]));
}
@@ -1136,7 +1170,17 @@ impl ObjectIO for SetDisks {
// }
if (w_size as i64) < data.size() {
warn!("put_object write size < data.size(), w_size={}, data.size={}", w_size, data.size());
warn!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
written_size = w_size,
expected_size = data.size(),
state = "short_write",
"Set disk write produced fewer bytes than expected"
);
return Err(Error::other(format!(
"put_object write size < data.size(), w_size={}, data.size={}",
w_size,
@@ -2976,7 +3020,15 @@ impl MultipartOperations for SetDisks {
{
Ok(writer) => writer,
Err(err) => {
warn!("create_bitrot_writer disk {}, err {:?}, skipping operation", disk.to_string(), err);
warn!(
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
disk = ?disk,
state = "bitrot_writer_skipped",
error = ?err,
"Set disk multipart bitrot writer skipped"
);
errors.push(Some(err));
writers.push(None);
continue;
@@ -3021,7 +3073,18 @@ impl MultipartOperations for SetDisks {
let _ = mem::replace(&mut data.stream, reader);
if (w_size as i64) < data.size() {
warn!("put_object_part write size < data.size(), w_size={}, data.size={}", w_size, data.size());
warn!(
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
part_number = part_id,
written_size = w_size,
expected_size = data.size(),
state = "short_write",
"Set disk multipart write produced fewer bytes than expected"
);
return Err(Error::other(format!(
"put_object_part write size < data.size(), w_size={}, data.size={}",
w_size,
@@ -3657,7 +3720,17 @@ impl MultipartOperations for SetDisks {
);
return Err(Error::InvalidPart(p.part_num, "".to_owned(), p.etag.clone().unwrap_or_default()));
};
info!(target:"rustfs_ecstore::set_disk", part_number = p.part_num, part_size = ext_part.size, part_actual_size = ext_part.actual_size, "Completing multipart part");
debug!(
target:"rustfs_ecstore::set_disk",
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
part_number = p.part_num,
part_size = ext_part.size,
part_actual_size = ext_part.actual_size,
state = "part_validated",
"Set disk multipart part validated"
);
// Normalize ETags by removing quotes before comparison (PR #592 compatibility)
let client_etag = p.etag.as_ref().map(|e| rustfs_utils::path::trim_etag(e));
@@ -3988,7 +4061,16 @@ impl HealOperations for SetDisks {
let disks = disks.clone();
let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false).await?;
if DiskError::is_all_not_found(&errs) {
debug!(bucket, object, version_id, "heal_object skipped missing object");
debug!(
event = EVENT_SET_DISK_HEAL,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
version_id,
state = "missing_object_skipped",
"Set disk heal skipped missing object"
);
let err = if !version_id.is_empty() {
Error::FileVersionNotFound
} else {
@@ -4356,7 +4438,16 @@ async fn disks_with_all_parts(
verify_resp = v;
}
Err(err) => {
info!("verify_file failed: {err:?}, object_name={}, index: {index}", object_name);
debug!(
event = EVENT_SET_DISK_HEAL,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
object = %object_name,
disk_index = index,
state = "verify_failed",
error = ?err,
"Set disk verify_file failed"
);
verify_err = Some(err);
}
}
@@ -4366,7 +4457,16 @@ async fn disks_with_all_parts(
verify_resp = v;
}
Err(err) => {
info!("check_parts failed: {err:?}, object_name={}, index: {index}", object_name);
debug!(
event = EVENT_SET_DISK_HEAL,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
object = %object_name,
disk_index = index,
state = "check_parts_failed",
error = ?err,
"Set disk check_parts failed"
);
verify_err = Some(err);
}
}
crates/heal/src/heal/channel.rs
+17 -17
View File
@@ -69,7 +69,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "started",
"Heal channel processor state updated"
"Heal channel started"
);
loop {
@@ -85,7 +85,7 @@ impl HealChannelProcessor {
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "process_failed",
error = %e,
"Heal channel request processing failed"
"Heal channel processing failed"
);
}
}
@@ -96,7 +96,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "receiver_closed",
"Heal channel processor state updated"
"Heal channel receiver closed"
);
break;
}
@@ -105,7 +105,7 @@ impl HealChannelProcessor {
response = self.response_receiver.recv() => {
if let Some(response) = response {
// Handle response if needed
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_RESPONSE,
component = LOG_COMPONENT_HEAL,
@@ -113,7 +113,7 @@ impl HealChannelProcessor {
request_id = %response.request_id,
success = response.success,
state = "received_local",
"Heal channel response observed"
"Heal response observed"
);
}
}
@@ -126,7 +126,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "stopped",
"Heal channel processor state updated"
"Heal channel stopped"
);
Ok(())
}
@@ -154,7 +154,7 @@ impl HealChannelProcessor {
request: HealChannelRequest,
response_tx: oneshot::Sender<std::result::Result<HealAdmissionResult, String>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -163,7 +163,7 @@ impl HealChannelProcessor {
bucket = %request.bucket,
object_prefix = %request.object_prefix.as_deref().unwrap_or(""),
state = "start_received",
"Heal channel start request received"
"Heal start received"
);
// Convert channel request to heal request
@@ -185,7 +185,7 @@ impl HealChannelProcessor {
// Submit to heal manager
match self.heal_manager.submit_heal_request(heal_request).await {
Ok(admission) => {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -193,7 +193,7 @@ impl HealChannelProcessor {
request_id = %request.id,
admission = admission.result_label(),
state = "admission_decided",
"Heal channel admission decision completed"
"Heal admission decided"
);
let _ = response_tx.send(Ok(admission));
@@ -225,7 +225,7 @@ impl HealChannelProcessor {
request_id = %request.id,
state = "submit_failed",
error = %error_text,
"Heal channel start request failed"
"Heal start submission failed"
);
let _ = response_tx.send(Err(error_text.clone()));
@@ -251,7 +251,7 @@ impl HealChannelProcessor {
client_token: String,
response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -259,7 +259,7 @@ impl HealChannelProcessor {
request_id = %client_token,
heal_path = %heal_path,
state = "query_received",
"Heal channel query request received"
"Heal query received"
);
let (summary, detail, items) = match self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await {
@@ -332,7 +332,7 @@ impl HealChannelProcessor {
client_token: String,
response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -340,7 +340,7 @@ impl HealChannelProcessor {
request_id = %client_token,
heal_path = %heal_path,
state = "cancel_received",
"Heal channel cancel request received"
"Heal cancel received"
);
let request_id = if client_token.is_empty() {
@@ -451,7 +451,7 @@ impl HealChannelProcessor {
request_id = %response.request_id,
state = "enqueue_local_failed",
error = %e,
"Heal channel response enqueue failed"
"Heal response local enqueue failed"
);
}
// Always attempt to broadcast, even if local send failed
@@ -464,7 +464,7 @@ impl HealChannelProcessor {
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "broadcast_failed",
error = %e,
"Heal channel response broadcast failed"
"Heal response broadcast failed"
);
}
}
crates/heal/src/heal/erasure_healer.rs
+27 -27
View File
@@ -27,7 +27,7 @@ use std::sync::{
atomic::{AtomicUsize, Ordering},
};
use tokio::sync::{RwLock, Semaphore};
use tracing::{error, info, warn};
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_ERASURE_HEALER: &str = "erasure_healer";
@@ -99,7 +99,7 @@ impl ErasureSetHealer {
/// execute erasure set heal with resume
#[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))]
pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -162,7 +162,7 @@ impl ErasureSetHealer {
Ok(manager) => {
let state = manager.get_state().await;
if state.set_disk_id == set_disk_id && ResumeUtils::can_resume_task(&self.disk, &task_id).await {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -170,7 +170,7 @@ impl ErasureSetHealer {
task_id,
set_disk_id,
state = "resume_found",
"Erasure set resume state selected"
"Erasure set resume selected"
);
return Ok(task_id);
}
@@ -193,7 +193,7 @@ impl ErasureSetHealer {
// create new task id
let task_id = format!("{}_{}", set_disk_id, ResumeUtils::generate_task_id());
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -215,7 +215,7 @@ impl ErasureSetHealer {
) -> Result<(ResumeManager, CheckpointManager)> {
// check if resume state exists
if ResumeManager::has_resume_state(&self.disk, task_id).await {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -235,7 +235,7 @@ impl ErasureSetHealer {
Ok((resume_manager, checkpoint_manager))
} else {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -243,7 +243,7 @@ impl ErasureSetHealer {
task_id,
set_disk_id,
state = "creating_new",
"Erasure set resume state creating"
"Erasure set resume created"
);
let resume_manager = ResumeManager::new(
@@ -273,7 +273,7 @@ impl ErasureSetHealer {
let state = resume_manager.get_state().await;
let checkpoint = checkpoint_manager.get_checkpoint().await;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -282,7 +282,7 @@ impl ErasureSetHealer {
current_bucket_index = checkpoint.current_bucket_index,
current_object_index = checkpoint.current_object_index,
state = "resuming",
"Erasure set heal resumed from checkpoint"
"Erasure set resumed"
);
// 2. initialize progress
@@ -349,7 +349,7 @@ impl ErasureSetHealer {
match bucket_result {
Ok(_) => {
resume_manager.complete_bucket(bucket).await?;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
@@ -357,7 +357,7 @@ impl ErasureSetHealer {
set_disk_id,
bucket,
state = "completed",
"Erasure set bucket heal completed"
"Erasure set bucket completed"
);
}
Err(e) => {
@@ -383,14 +383,14 @@ impl ErasureSetHealer {
// 5. mark task completed
resume_manager.mark_completed().await?;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
set_disk_id,
state = "completed",
"Erasure set heal completed"
"Erasure set completed"
);
Ok(())
}
@@ -411,7 +411,7 @@ impl ErasureSetHealer {
resume_manager: &ResumeManager,
checkpoint_manager: &CheckpointManager,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
@@ -421,7 +421,7 @@ impl ErasureSetHealer {
bucket_index,
current_object_index = *current_object_index,
state = "started",
"Erasure set bucket heal started"
"Erasure set bucket started"
);
// 1. get bucket info
@@ -584,7 +584,7 @@ impl ErasureSetHealer {
Ok(true) => {
*successful_objects += 1;
checkpoint_manager.add_processed_object(object.clone()).await?;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -593,13 +593,13 @@ impl ErasureSetHealer {
bucket,
object = %object,
state = "healed",
"Erasure set object heal completed"
"Erasure set object healed"
);
}
Ok(false) => {
checkpoint_manager.add_processed_object(object.clone()).await?;
*successful_objects += 1;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -608,7 +608,7 @@ impl ErasureSetHealer {
bucket,
object = %object,
state = "missing_treated_as_ok",
"Erasure set object heal skipped because object disappeared"
"Erasure set missing object treated as ok"
);
}
Err(Error::TaskCancelled) => {
@@ -743,14 +743,14 @@ impl ErasureSetHealer {
bucket: &str,
progress: &Arc<RwLock<HealProgress>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
bucket,
state = "started",
"Erasure set bucket heal started"
"Erasure set bucket started"
);
// 1. get bucket info
@@ -847,7 +847,7 @@ impl ErasureSetHealer {
p.set_current_object(Some(format!("completed bucket: {bucket}")));
}
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
@@ -857,7 +857,7 @@ impl ErasureSetHealer {
total_failed,
total_scanned,
state = "completed",
"Erasure set bucket heal completed"
"Erasure set bucket completed"
);
Ok(())
@@ -890,7 +890,7 @@ impl ErasureSetHealer {
match storage.heal_object(&bucket, &object, None, &heal_opts).await {
Ok((_result, None)) => {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -898,7 +898,7 @@ impl ErasureSetHealer {
bucket,
object = %object,
state = "healed",
"Erasure set object heal completed"
"Erasure set object healed"
);
Ok(())
}
@@ -956,7 +956,7 @@ impl ErasureSetHealer {
success_count,
total,
state = "summary",
"Erasure set heal summary recorded"
"Erasure set summary recorded"
);
if failure_count > 0 {
crates/heal/src/heal/manager.rs
+56 -36
View File
@@ -558,7 +558,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "already_running",
"Heal manager state unchanged"
"Heal manager already running"
);
return Ok(());
}
@@ -571,7 +571,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "starting",
"Heal manager state updated"
"Heal manager starting"
);
// start scheduler
@@ -586,7 +586,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "running",
"Heal manager state updated"
"Heal manager started"
);
Ok(())
}
@@ -599,7 +599,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "stopping",
"Heal manager state updated"
"Heal manager stopping"
);
// cancel all tasks
@@ -617,7 +617,7 @@ impl HealManager {
state = "task_cancel_failed",
task_id = %task.id,
error = %e,
"Heal manager failed to cancel active task"
"Heal active task cancellation failed"
);
}
}
@@ -636,7 +636,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "stopped",
"Heal manager state updated"
"Heal manager stopped"
);
Ok(())
}
@@ -659,7 +659,7 @@ impl HealManager {
match admission {
HealAdmissionResult::Merged => {
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL,
@@ -667,7 +667,7 @@ impl HealManager {
request_id = %request.id,
priority = ?request.priority,
result = "merged_duplicate",
"Heal queue admission decided"
"Heal queue request merged"
);
}
HealAdmissionResult::Dropped(reason) => {
@@ -680,7 +680,7 @@ impl HealManager {
priority = ?request.priority,
reason = reason.as_str(),
result = "dropped_duplicate",
"Heal queue admission decided"
"Heal queue request dropped"
);
}
HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
@@ -707,7 +707,7 @@ impl HealManager {
queue_len,
queue_capacity,
result = "accepted_by_displacement",
"Heal queue admission decided"
"Heal queue request accepted by displacement"
);
drop(queue);
if config.event_driven_scheduler_enable {
@@ -726,7 +726,7 @@ impl HealManager {
queue_len,
queue_capacity,
result = "full_no_displacement_candidate",
"Heal queue admission decided"
"Heal queue request rejected without displacement"
);
return Ok(HealAdmissionResult::Full);
}
@@ -745,7 +745,7 @@ impl HealManager {
queue_capacity,
reason = reason.as_str(),
result = "dropped_full",
"Heal queue admission decided"
"Heal queue request dropped"
);
}
HealAdmissionResult::Full => {
@@ -759,7 +759,7 @@ impl HealManager {
queue_len,
queue_capacity,
result = "rejected_full",
"Heal queue admission decided"
"Heal queue request rejected"
);
}
HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {}
@@ -779,7 +779,7 @@ impl HealManager {
queue_capacity,
queue_usage_pct = (queue_len * 100) / queue_capacity,
result = "queue_pressure_high",
"Heal queue pressure increased"
"Heal queue pressure high"
);
}
@@ -793,7 +793,7 @@ impl HealManager {
// Log queue statistics periodically (when adding high/urgent priority items)
if matches!(priority, HealPriority::High | HealPriority::Urgent) {
let stats = queue.get_priority_stats();
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL,
@@ -812,7 +812,7 @@ impl HealManager {
drop(queue);
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL,
@@ -821,7 +821,7 @@ impl HealManager {
priority = ?priority,
queue_len = queue_len + 1,
result = "accepted",
"Heal queue admission decided"
"Heal queue request accepted"
);
if config.event_driven_scheduler_enable {
self.notify.notify_one();
@@ -1119,7 +1119,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "shutdown",
"Heal scheduler state updated"
"Heal scheduler stopped"
);
break;
}
@@ -1158,13 +1158,17 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "started",
interval = ?duration,
"Heal auto disk scanner state updated"
"Heal auto disk scanner started"
);
tokio::spawn(async move {
let mut interval = interval(duration);
loop {
let mut candidate_count = 0usize;
let mut skipped_duplicate_count = 0usize;
let mut skipped_invalid_count = 0usize;
let mut enqueued_count = 0usize;
tokio::select! {
_ = cancel_token.cancelled() => {
info!(
@@ -1173,7 +1177,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "shutdown",
"Heal auto disk scanner state updated"
"Heal auto disk scanner stopped"
);
break;
}
@@ -1185,14 +1189,15 @@ impl HealManager {
// detect unformatted disk via get_disk_id()
match disk.get_disk_id().await {
Err(DiskError::UnformattedDisk) => {
info!(
candidate_count += 1;
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_DISK,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
endpoint = %disk.endpoint(),
disk_state = "unformatted",
"Heal auto disk scanner detected candidate disk"
"Heal auto-scan candidate detected"
);
endpoints.push(disk.endpoint());
}
@@ -1205,7 +1210,7 @@ impl HealManager {
endpoint = %disk.endpoint(),
disk_state = "check_failed",
error = ?e,
"Heal auto disk scanner failed to inspect disk"
"Heal auto-scan disk inspection failed"
);
}
Ok(_) => {
@@ -1222,7 +1227,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "idle",
"Heal auto disk scanner found no candidate disks"
"Heal auto disk scanner idle"
);
continue;
}
@@ -1238,7 +1243,7 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "bucket_list_failed",
error = %e,
"Heal auto disk scanner failed to list buckets"
"Heal auto-scan bucket listing failed"
);
continue;
}
@@ -1256,8 +1261,9 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
endpoint = %ep,
result = "skipped_invalid_set_disk_id",
"Heal auto disk scanner skipped enqueue"
"Heal auto-scan enqueue skipped"
);
skipped_invalid_count += 1;
continue;
};
// skip if already queued or healing
@@ -1283,7 +1289,8 @@ impl HealManager {
}
if skip {
info!(
skipped_duplicate_count += 1;
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
component = LOG_COMPONENT_HEAL,
@@ -1291,7 +1298,7 @@ impl HealManager {
endpoint = %ep,
set_disk_id,
result = "skipped_duplicate",
"Heal auto disk scanner skipped enqueue"
"Heal auto-scan duplicate skipped"
);
continue;
}
@@ -1312,7 +1319,8 @@ impl HealManager {
if config.event_driven_scheduler_enable {
notify.notify_one();
}
info!(
enqueued_count += 1;
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
component = LOG_COMPONENT_HEAL,
@@ -1321,10 +1329,22 @@ impl HealManager {
set_disk_id,
bucket_count = buckets.len(),
result = "enqueued",
"Heal auto disk scanner enqueued task"
"Heal auto-scan task enqueued"
);
}
}
info!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "cycle_completed",
candidate_count,
enqueued_count,
skipped_duplicate_count,
skipped_invalid_count,
"Heal auto-scan cycle completed"
);
}
}
}
@@ -1405,7 +1425,7 @@ impl HealManager {
// start heal task
tokio::spawn(async move {
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_SCHEDULER_STATE,
component = LOG_COMPONENT_HEAL,
@@ -1415,12 +1435,12 @@ impl HealManager {
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
state = "task_started",
"Heal scheduler started task"
"Heal scheduler task started"
);
let result = task.execute().await;
match result {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_SCHEDULER_STATE,
component = LOG_COMPONENT_HEAL,
@@ -1429,7 +1449,7 @@ impl HealManager {
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
state = "task_completed",
"Heal scheduler finished task"
"Heal scheduler task completed"
);
}
Err(e) => {
@@ -1443,7 +1463,7 @@ impl HealManager {
set = %task_set_label_for_spawn,
state = "task_failed",
error = %e,
"Heal scheduler observed task failure"
"Heal scheduler task failed"
);
}
}
@@ -1500,7 +1520,7 @@ impl HealManager {
queue_len = remaining,
active_tasks = active_heals_guard.len(),
state = "backlog_high",
"Heal queue backlog summary recorded"
"Heal queue backlog high"
);
}
}
crates/heal/src/heal/resume.rs
+5 -5
View File
@@ -20,7 +20,7 @@ use std::path::Path;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use tracing::{debug, warn};
use uuid::Uuid;
const LOG_COMPONENT_HEAL: &str = "heal";
@@ -299,7 +299,7 @@ impl ResumeManager {
let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await;
}
info!(
debug!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -524,7 +524,7 @@ impl CheckpointManager {
let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await;
}
info!(
debug!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_CHECKPOINT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -649,7 +649,7 @@ impl ResumeUtils {
let age_hours = (current_time - state.last_update) / 3600;
if age_hours > max_age_hours {
info!(
debug!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -657,7 +657,7 @@ impl ResumeUtils {
task_id,
age_hours,
state = "expired_cleanup_started",
"Heal resume state cleanup started"
"Heal resume cleanup started"
);
if let Err(e) = resume_manager.cleanup().await {
warn!(
crates/heal/src/heal/storage.rs
+33 -33
View File
@@ -24,7 +24,7 @@ use rustfs_ecstore::{
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::{BucketInfo, DiskSetSelector, StorageAdminApi};
use std::sync::Arc;
use tracing::{debug, error, info, warn};
use tracing::{debug, error, warn};
const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_STORAGE: &str = "storage";
@@ -206,7 +206,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "not_found",
"Heal storage request finished"
"Heal storage object metadata missing"
);
Ok(None)
} else {
@@ -333,7 +333,7 @@ impl HealStorageAPI for ECStoreHealStorage {
.await
{
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_IO,
component = LOG_COMPONENT_HEAL,
@@ -342,7 +342,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "ok",
"Heal storage request finished"
"Heal storage object write completed"
);
Ok(())
}
@@ -378,7 +378,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.delete_object(bucket, object, Default::default()).await {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_IO,
component = LOG_COMPONENT_HEAL,
@@ -387,7 +387,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "ok",
"Heal storage request finished"
"Heal storage object delete completed"
);
Ok(())
}
@@ -447,7 +447,7 @@ impl HealStorageAPI for ECStoreHealStorage {
let mut stream = reader.stream;
match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
component = LOG_COMPONENT_HEAL,
@@ -455,7 +455,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
state = "ok",
"Heal storage object verification finished"
"Heal storage object verified"
);
Ok(true)
}
@@ -544,7 +544,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// After healing, try to read the object data
match self.get_object_data(bucket, object).await? {
Some(data) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -554,7 +554,7 @@ impl HealStorageAPI for ECStoreHealStorage {
object,
bytes = data.len(),
state = "ok",
"Heal storage repair finished"
"Heal storage EC decode rebuild completed"
);
Ok(data)
}
@@ -608,7 +608,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// TODO: implement disk status check using ecstore
// For now, return Ok status
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -617,7 +617,7 @@ impl HealStorageAPI for ECStoreHealStorage {
endpoint = ?endpoint,
result = "ok",
disk_status = "ok",
"Heal storage admin operation finished"
"Heal storage disk status resolved"
);
Ok(DiskStatus::Ok)
}
@@ -640,7 +640,7 @@ impl HealStorageAPI for ECStoreHealStorage {
if error.is_some() {
return Err(Error::other(format!("Format failed: {error:?}")));
}
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -648,7 +648,7 @@ impl HealStorageAPI for ECStoreHealStorage {
operation = "format_disk",
endpoint = ?endpoint,
result = "ok",
"Heal storage admin operation finished"
"Heal storage disk format completed"
);
Ok(())
}
@@ -726,7 +726,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.heal_bucket(bucket, &heal_opts).await {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -734,7 +734,7 @@ impl HealStorageAPI for ECStoreHealStorage {
operation = "heal_bucket_metadata",
bucket,
result = "ok",
"Heal storage repair finished"
"Heal storage bucket metadata repaired"
);
Ok(())
}
@@ -816,7 +816,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "not_found",
"Heal storage request finished"
"Heal storage object absence confirmed"
);
Ok(false)
} else if is_transient_object_exists_error(&e) {
@@ -923,7 +923,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_object(bucket, object, version_id_str, opts).await {
Ok((result, ecstore_error)) => {
let error = ecstore_error.map(Error::other);
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -935,7 +935,7 @@ impl HealStorageAPI for ECStoreHealStorage {
drives_after = result.after.drives.len(),
has_error = error.is_some(),
result = "ok",
"Heal storage repair finished"
"Heal storage object repair completed"
);
Ok((result, error))
}
@@ -974,7 +974,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_bucket(bucket, opts).await {
Ok(result) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -983,7 +983,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
drives_after = result.after.drives.len(),
result = "ok",
"Heal storage repair finished"
"Heal storage bucket repair completed"
);
Ok(result)
}
@@ -1019,7 +1019,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_format(dry_run).await {
Ok((result, ecstore_error)) => {
let error = ecstore_error.map(Error::other);
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -1028,7 +1028,7 @@ impl HealStorageAPI for ECStoreHealStorage {
drives_after = result.after.drives.len(),
has_error = error.is_some(),
result = "ok",
"Heal storage repair finished"
"Heal storage format repair completed"
);
Ok((result, error))
}
@@ -1103,7 +1103,7 @@ impl HealStorageAPI for ECStoreHealStorage {
}
}
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -1113,7 +1113,7 @@ impl HealStorageAPI for ECStoreHealStorage {
prefix,
object_count = all_objects.len(),
result = "ok",
"Heal storage admin operation finished"
"Heal storage object listing completed"
);
Ok(all_objects)
}
@@ -1175,12 +1175,12 @@ impl HealStorageAPI for ECStoreHealStorage {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_STORAGE,
operation = "list_objects_for_heal_page",
bucket,
prefix,
object_count = page_count,
is_truncated = list_info.is_truncated,
state = "page_loaded",
"Heal storage admin operation finished"
bucket,
prefix,
object_count = page_count,
is_truncated = list_info.is_truncated,
state = "page_loaded",
"Heal storage object listing page loaded"
);
Ok((page_objects, list_info.next_continuation_token, list_info.is_truncated))
@@ -1210,7 +1210,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// Find the first available disk
if let Some(disk_store) = disks.into_iter().flatten().next() {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -1219,7 +1219,7 @@ impl HealStorageAPI for ECStoreHealStorage {
set_disk_id,
result = "ok",
disk = ?disk_store,
"Heal storage admin operation finished"
"Heal storage resume disk resolved"
);
return Ok(disk_store);
}
crates/heal/src/heal/task.rs
+51 -51
View File
@@ -445,7 +445,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(),
state = "started",
queue_delay = ?queue_delay,
"Heal task state updated"
"Heal task started"
);
let result = match &self.heal_type {
@@ -484,7 +484,7 @@ impl HealTask {
task_id = %self.id,
heal_type = self.heal_type.log_kind(),
state = "completed",
"Heal task state updated"
"Heal task completed"
);
}
Err(Error::TaskCancelled) => {
@@ -498,7 +498,7 @@ impl HealTask {
task_id = %self.id,
heal_type = self.heal_type.log_kind(),
state = "cancelled",
"Heal task state updated"
"Heal task cancelled"
);
}
Err(Error::TaskTimeout) => {
@@ -512,7 +512,7 @@ impl HealTask {
task_id = %self.id,
heal_type = self.heal_type.log_kind(),
state = "timed_out",
"Heal task state updated"
"Heal task timed out"
);
}
Err(e) => {
@@ -527,7 +527,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(),
state = "failed",
error = %e,
"Heal task state updated"
"Heal task failed"
);
}
}
@@ -539,7 +539,7 @@ impl HealTask {
self.cancel_token.cancel();
let mut status = self.status.write().await;
*status = HealTaskStatus::Cancelled;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_TASK_STATE,
component = LOG_COMPONENT_HEAL,
@@ -548,7 +548,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(),
state = "cancelled",
source = "manual",
"Heal task state updated"
"Heal task cancellation requested"
);
Ok(())
}
@@ -572,7 +572,7 @@ impl HealTask {
// specific heal implementation method
#[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))]
async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -582,7 +582,7 @@ impl HealTask {
object,
version_id = ?version_id,
stage = "start",
"Heal object workflow started"
"Heal object started"
);
// update progress
@@ -625,7 +625,7 @@ impl HealTask {
"Heal target object is missing"
);
if self.options.recreate_missing {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -634,7 +634,7 @@ impl HealTask {
bucket,
object,
stage = "recreate_missing",
"Heal object recovery started"
"Heal object recreate requested"
);
return self.recreate_missing_object(bucket, object, version_id).await;
} else {
@@ -690,7 +690,7 @@ impl HealTask {
// Check if this is a "File not found" error during delete operations
let error_msg = format!("{e}");
if error_msg.contains("File not found") || error_msg.contains("not found") {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -699,7 +699,7 @@ impl HealTask {
bucket,
object,
result = "treated_as_deleted",
"Heal object finished after target disappeared during repair"
"Heal missing object treated as deleted"
);
{
let mut progress = self.progress.write().await;
@@ -723,7 +723,7 @@ impl HealTask {
// If heal failed and remove_corrupted is enabled, delete the corrupted object
if self.options.remove_corrupted {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -737,7 +737,7 @@ impl HealTask {
);
if !self.options.dry_run {
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -747,7 +747,7 @@ impl HealTask {
object,
action = "delete_corrupted_object",
result = "deleted",
"Heal object cleanup completed"
"Heal corrupted object deleted"
);
}
}
@@ -775,7 +775,7 @@ impl HealTask {
"Heal object stage entered"
);
let object_size = result.object_size as u64;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -786,7 +786,7 @@ impl HealTask {
object_size = object_size,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal object completed"
"Heal object repaired"
);
{
@@ -806,7 +806,7 @@ impl HealTask {
// Check if this is a "File not found" error during delete operations
let error_msg = format!("{e}");
if error_msg.contains("File not found") || error_msg.contains("not found") {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -815,7 +815,7 @@ impl HealTask {
bucket,
object,
result = "treated_as_deleted",
"Heal object finished after target disappeared during repair"
"Heal missing object treated as deleted"
);
{
let mut progress = self.progress.write().await;
@@ -839,7 +839,7 @@ impl HealTask {
// If heal failed and remove_corrupted is enabled, delete the corrupted object
if self.options.remove_corrupted {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -853,7 +853,7 @@ impl HealTask {
);
if !self.options.dry_run {
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -863,7 +863,7 @@ impl HealTask {
object,
action = "delete_corrupted_object",
result = "deleted",
"Heal object cleanup completed"
"Heal corrupted object deleted"
);
}
}
@@ -882,7 +882,7 @@ impl HealTask {
/// Recreate missing object (for EC decode scenarios)
async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -892,7 +892,7 @@ impl HealTask {
object,
version_id = ?version_id,
stage = "recreate_missing",
"Heal object recovery started"
"Heal object recreate started"
);
// Use ecstore's heal_object with recreate option
@@ -932,7 +932,7 @@ impl HealTask {
}
let object_size = result.object_size as u64;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -942,7 +942,7 @@ impl HealTask {
object,
object_size,
result = "recreated",
"Heal object recovery finished"
"Heal object recreated"
);
{
@@ -975,7 +975,7 @@ impl HealTask {
}
async fn heal_bucket(&self, bucket: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -984,7 +984,7 @@ impl HealTask {
bucket,
stage = "start",
recursive = self.options.recursive,
"Heal bucket workflow started"
"Heal bucket started"
);
// update progress
@@ -1060,7 +1060,7 @@ impl HealTask {
match heal_result {
Ok(result) => {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1269,7 +1269,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1287,7 +1287,7 @@ impl HealTask {
}
async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_METADATA_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1296,7 +1296,7 @@ impl HealTask {
bucket,
object,
stage = "start",
"Heal metadata workflow started"
"Heal metadata started"
);
// update progress
@@ -1400,7 +1400,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_METADATA_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1410,7 +1410,7 @@ impl HealTask {
object,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal metadata completed"
"Heal metadata repaired"
);
{
@@ -1447,7 +1447,7 @@ impl HealTask {
}
async fn heal_mrf(&self, meta_path: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_MRF_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1455,7 +1455,7 @@ impl HealTask {
task_id = %self.id,
meta_path,
stage = "start",
"Heal MRF workflow started"
"Heal MRF started"
);
// update progress
@@ -1530,7 +1530,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_MRF_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1541,7 +1541,7 @@ impl HealTask {
object = %object,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal MRF completed"
"Heal MRF repaired"
);
{
@@ -1579,7 +1579,7 @@ impl HealTask {
}
async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_EC_DECODE_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1589,7 +1589,7 @@ impl HealTask {
object,
version_id = ?version_id,
stage = "start",
"Heal EC decode workflow started"
"Heal EC decode started"
);
// update progress
@@ -1694,7 +1694,7 @@ impl HealTask {
}
let object_size = result.object_size as u64;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_EC_DECODE_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1705,7 +1705,7 @@ impl HealTask {
object_size,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal EC decode completed"
"Heal EC decode repaired"
);
{
@@ -1742,7 +1742,7 @@ impl HealTask {
}
async fn heal_erasure_set(&self, buckets: Vec<String>, set_disk_id: String) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1751,7 +1751,7 @@ impl HealTask {
set_disk_id,
bucket_count = buckets.len(),
stage = "start",
"Heal erasure set workflow started"
"Heal erasure set started"
);
// update progress
@@ -1762,7 +1762,7 @@ impl HealTask {
}
let buckets = if buckets.is_empty() {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1770,7 +1770,7 @@ impl HealTask {
task_id = %self.id,
set_disk_id,
stage = "list_buckets",
"Heal erasure set resolved bucket list from storage"
"Heal erasure set bucket list resolved"
);
let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
bucket_infos.into_iter().map(|info| info.name).collect()
@@ -1814,7 +1814,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1823,7 +1823,7 @@ impl HealTask {
set_disk_id,
drives_healed = result.after.drives.len(),
result = "format_ok",
"Heal erasure set format repair completed"
"Heal erasure set format repaired"
);
}
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
@@ -1970,7 +1970,7 @@ impl HealTask {
match result {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1979,7 +1979,7 @@ impl HealTask {
set_disk_id,
bucket_count = buckets.len(),
result = "ok",
"Heal erasure set completed"
"Heal erasure set repaired"
);
Ok(())
}
crates/iam/src/lib.rs
+79 -11
View File
@@ -19,7 +19,13 @@ use rustfs_ecstore::store::ECStore;
use std::sync::{Arc, OnceLock};
use store::object::ObjectStore;
use sys::IamSys;
use tracing::{error, info, instrument, warn};
use tracing::{debug, error, info, instrument, warn};
const LOG_COMPONENT_IAM: &str = "iam";
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
const LOG_SUBSYSTEM_OIDC: &str = "oidc";
const EVENT_IAM_STATE: &str = "iam_state";
const EVENT_OIDC_STATE: &str = "oidc_state";
pub mod cache;
pub mod error;
@@ -37,11 +43,23 @@ static OIDC_SYS: OnceLock<Arc<OidcSys>> = OnceLock::new();
#[instrument(skip(ecstore))]
pub async fn init_iam_sys(ecstore: Arc<ECStore>) -> Result<()> {
if IAM_SYS.get().is_some() {
info!("IAM system already initialized, skipping.");
info!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "already_initialized",
"IAM runtime already initialized"
);
return Ok(());
}
info!("Starting IAM system initialization sequence...");
info!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "starting",
"IAM runtime starting"
);
// 1. Create the persistent storage adapter
let storage_adapter = ObjectStore::new(ecstore);
@@ -55,11 +73,23 @@ pub async fn init_iam_sys(ecstore: Arc<ECStore>) -> Result<()> {
// 4. Securely set the global singleton
if IAM_SYS.set(iam_instance).is_err() {
error!("Critical: Race condition detected during IAM initialization!");
error!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "singleton_set_failed",
"IAM runtime singleton set failed"
);
return Err(Error::IamSysAlreadyInitialized);
}
info!("IAM system initialization completed successfully.");
info!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "ready",
"IAM runtime ready"
);
Ok(())
}
@@ -84,29 +114,67 @@ pub fn get_global_iam_sys() -> Option<Arc<IamSys<ObjectStore>>> {
/// Initialize the global OIDC system. Non-fatal if no OIDC providers are configured.
pub async fn init_oidc_sys() -> Result<()> {
if OIDC_SYS.get().is_some() {
info!("OIDC system already initialized, skipping.");
debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "already_initialized",
"OIDC runtime already initialized"
);
return Ok(());
}
info!("Starting OIDC system initialization...");
debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "starting",
"OIDC runtime starting"
);
let oidc_sys = match OidcSys::new().await {
Ok(sys) => {
if sys.has_providers() {
info!("OIDC system initialized with {} provider(s)", sys.list_providers().len());
debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
provider_count = sys.list_providers().len(),
state = "ready",
"OIDC runtime ready"
);
} else {
info!("No OIDC providers configured");
debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "empty",
"OIDC runtime has no providers"
);
}
sys
}
Err(e) => {
warn!("OIDC initialization failed (non-fatal): {}", e);
warn!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "init_failed_non_fatal",
error = %e,
"OIDC runtime initialization failed"
);
OidcSys::empty().map_err(Error::StringError)?
}
};
if OIDC_SYS.set(Arc::new(oidc_sys)).is_err() {
warn!("Race condition during OIDC initialization (non-fatal)");
warn!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "singleton_set_race",
"OIDC runtime singleton set raced"
);
}
Ok(())
crates/iam/src/manager.rs
+14 -10
View File
@@ -54,7 +54,7 @@ use tokio::{
},
};
use tracing::warn;
use tracing::{error, info};
use tracing::{debug, error};
const IAM_FORMAT_FILE: &str = "format.json";
const IAM_FORMAT_VERSION_1: i32 = 1;
@@ -153,10 +153,14 @@ where
if let Err(e) = self.clone().load().await {
if attempt == MAX_RETRIES - 1 {
self.state.store(IamState::Error as u8, Ordering::SeqCst);
warn!("IAM failed to load initial data after {} attempts: {:?}", MAX_RETRIES, e);
warn!(
attempts = MAX_RETRIES,
error = ?e,
"IAM initial load failed"
);
load_error = Some(e);
} else {
warn!("IAM load failed, retrying... attempt {}", attempt + 1);
warn!(attempt = attempt + 1, max_attempts = MAX_RETRIES, "IAM load retry scheduled");
tokio::time::sleep(INITIAL_LOAD_RETRY_DELAY).await;
}
} else {
@@ -169,7 +173,7 @@ where
}
self.state.store(IamState::Ready as u8, Ordering::SeqCst);
info!("IAM System successfully initialized and marked as READY");
debug!(state = "ready", "IAM manager ready");
// Background ticker for synchronization
// Check if environment variable is set
@@ -185,20 +189,20 @@ where
loop {
select! {
_ = ticker.tick() => {
info!("iam load ticker");
debug!(source = "ticker", "IAM reload tick");
if let Err(err) =s.clone().load().await{
warn!("iam load err {:?}", err);
warn!(source = "ticker", error = ?err, "IAM reload failed");
}
},
i = receiver.recv() => {
info!("iam load receiver");
debug!(source = "receiver", "IAM reload signal received");
match i {
Some(t) => {
let last = s.last_timestamp.load(Ordering::Relaxed);
if last <= t {
info!("iam load receiver load");
debug!(source = "receiver", "IAM reload accepted");
if let Err(err) =s.clone().load().await{
warn!("iam load err {:?}", err);
warn!(source = "receiver", error = ?err, "IAM reload failed");
}
ticker.reset();
}
@@ -1294,7 +1298,7 @@ where
let cache = self.cache.snapshot();
let users = Arc::clone(&cache.users);
if let Some(x) = users.get(access_key) {
warn!("user already exists: {:?}", x);
warn!(error = ?x, "IAM user already exists");
if x.credentials.is_temp() {
return Err(Error::IAMActionNotAllowed);
}
crates/iam/src/oidc.rs
+5 -5
View File
@@ -40,7 +40,7 @@ use std::pin::Pin;
use std::sync::{LazyLock, Mutex, MutexGuard, RwLock};
use std::time::{Duration as StdDuration, Instant};
use tokio::time::sleep;
use tracing::{error, info, warn};
use tracing::{debug, error, warn};
use url::Url;
const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
@@ -76,7 +76,7 @@ fn lock_oidc_plugin_authn_metrics<'a, T>(mutex: &'a Mutex<T>, metric: &'static s
match mutex.lock() {
Ok(guard) => guard,
Err(err) => {
warn!("recovering poisoned OIDC plugin authn metrics lock: {}", metric);
warn!(metric, "Recovering poisoned OIDC authn metrics lock");
err.into_inner()
}
}
@@ -417,18 +417,18 @@ impl OidcSys {
for sourced_config in parsed_configs {
let config = sourced_config.config;
if !config.enabled {
info!("OIDC provider '{}' is disabled, skipping", config.id);
debug!(provider = %config.id, "OIDC provider disabled");
continue;
}
match Self::discover_provider(&config, &http_client).await {
Ok(state) => {
info!("OIDC provider '{}' discovered successfully", config.id);
debug!(provider = %config.id, "OIDC provider discovered");
provider_states.insert(config.id.clone(), state);
configs.insert(config.id.clone(), config);
}
Err(e) => {
error!("Failed to discover OIDC provider '{}': {}", config.id, e);
error!(provider = %config.id, error = %e, "OIDC provider discovery failed");
}
}
}
crates/iam/src/store/object.rs
+31 -33
View File
@@ -41,7 +41,7 @@ use std::time::{Duration, Instant};
use std::{collections::HashMap, sync::Arc};
use tokio::sync::mpsc::{self, Sender};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
use tracing::{debug, error, warn};
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam"));
pub static IAM_CONFIG_USERS_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam/users/"));
@@ -307,11 +307,11 @@ impl ObjectStore {
}
Err(StorageError::PreconditionFailed) => {
Self::complete_lazy_rewrite(path.as_str(), false);
debug!("iam lazy rewrite skipped due to stale etag, path: {}", path);
debug!(path = %path, state = "stale_etag", "IAM lazy rewrite skipped");
}
Err(err) => {
Self::complete_lazy_rewrite(path.as_str(), false);
warn!("iam lazy rewrite failed, path: {}, err: {}", path, err);
warn!(path = %path, error = %err, state = "rewrite_failed", "IAM lazy rewrite failed");
}
}
});
@@ -377,7 +377,7 @@ impl ObjectStore {
bucket = Self::BUCKET_NAME,
prefix = %path,
error = %err,
"system path walk failed"
"IAM config walk failed"
);
let _ = sender_on_error
.send(StringOrErr {
@@ -643,7 +643,7 @@ impl Store for ObjectStore {
let outcome = match Self::decrypt_data_with_source(&data) {
Ok(v) => v,
Err(err) => {
warn!("config decrypt failed, keeping file: {}, path: {}", err, path_ref);
warn!(path = %path_ref, error = %err, "IAM config decrypt failed; keeping file");
// keep the config file when decrypt failed - do not delete
return Err(Error::ConfigNotFound);
}
@@ -692,7 +692,7 @@ impl Store for ObjectStore {
tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await;
}
Err(e) => {
error!("Final failure saving IAM config to {}: {:?}", path_ref, e);
error!(path = %path_ref, error = ?e, "IAM config save failed");
return Err(e.into());
}
}
@@ -717,7 +717,7 @@ impl Store for ObjectStore {
debug!("Saving IAM identity to path: {}", path);
self.save_iam_config(user_identity, path).await.map_err(|e| {
error!("ObjectStore save failure for {}: {:?}", name, e);
error!(name, error = ?e, "IAM identity save failed");
e
})
}
@@ -739,10 +739,10 @@ impl Store for ObjectStore {
.await
.map_err(|err| {
if is_err_config_not_found(&err) {
warn!("load_user_identity failed: no such user, name: {name}, user_type: {user_type:?}");
warn!(name, user_type = ?user_type, "IAM user identity missing");
Error::NoSuchUser(name.to_owned())
} else {
warn!("load_user_identity failed: {err:?}, name: {name}, user_type: {user_type:?}");
warn!(name, user_type = ?user_type, error = ?err, "IAM user identity load failed");
err
}
})?;
@@ -750,9 +750,7 @@ impl Store for ObjectStore {
if u.credentials.is_expired() {
let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
warn!(
"load_user_identity failed: user is expired, delete the user and mapped policy, name: {name}, user_type: {user_type:?}"
);
warn!(name, user_type = ?user_type, "IAM user identity expired and was removed");
return Err(Error::NoSuchUser(name.to_owned()));
}
@@ -776,7 +774,7 @@ impl Store for ObjectStore {
let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
}
warn!("extract_jwt_claims failed: {err:?}, name: {name}, user_type: {user_type:?}");
warn!(name, user_type = ?user_type, error = ?err, "IAM JWT claim extraction failed");
return Err(Error::NoSuchUser(name.to_owned()));
}
}
@@ -804,7 +802,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await {
if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err);
warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel();
return Err(err);
@@ -866,7 +864,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await {
if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err);
warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel();
return Err(err);
@@ -934,7 +932,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await {
if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err);
warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel();
return Err(err);
@@ -1009,7 +1007,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await {
if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err);
warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel();
return Err(err);
@@ -1044,7 +1042,7 @@ impl Store for ObjectStore {
let policy_name = rustfs_utils::path::dir(&policies_list[idx]);
info!("load policy: {}", policy_name);
debug!(policy = %policy_name, "IAM policy loaded");
policy_docs_cache.insert(policy_name, p);
}
@@ -1059,7 +1057,7 @@ impl Store for ObjectStore {
}
let policy_name = rustfs_utils::path::dir(&policies_list[idx]);
info!("load policy: {}", policy_name);
debug!(policy = %policy_name, "IAM policy loaded");
policy_docs_cache.insert(policy_name, p);
}
@@ -1083,7 +1081,7 @@ impl Store for ObjectStore {
}
let name = rustfs_utils::path::dir(&item_name_list[idx]);
info!("load reg user: {}", name);
debug!(user = %name, "IAM regular user loaded");
user_items_cache.insert(name, p);
}
break;
@@ -1097,7 +1095,7 @@ impl Store for ObjectStore {
}
let name = rustfs_utils::path::dir(&item_name_list[idx]);
info!("load reg user: {}", name);
debug!(user = %name, "IAM regular user loaded");
user_items_cache.insert(name, p);
}
@@ -1112,7 +1110,7 @@ impl Store for ObjectStore {
for item in item_name_list.iter() {
let name = rustfs_utils::path::dir(item);
info!("load group: {}", name);
debug!(group = %name, "IAM group loaded");
if let Err(err) = self.load_group(&name, &mut items_cache).await {
return Err(Error::other(format!("load group failed: {err}")));
};
@@ -1140,7 +1138,7 @@ impl Store for ObjectStore {
}
let name = item_name_list[idx].trim_end_matches(".json").to_owned();
info!("load user policy: {}", name);
debug!(user = %name, "IAM user policy loaded");
items_cache.insert(name, p);
}
break;
@@ -1156,7 +1154,7 @@ impl Store for ObjectStore {
}
let name = item_name_list[idx].trim_end_matches(".json").to_owned();
info!("load user policy: {}", name);
debug!(user = %name, "IAM user policy loaded");
items_cache.insert(name, p);
}
@@ -1174,7 +1172,7 @@ impl Store for ObjectStore {
for item in item_name_list.iter() {
let name = item.trim_end_matches(".json");
info!("load group policy: {}", name);
debug!(group = %name, "IAM group policy loaded");
if let Err(err) = self.load_mapped_policy(name, UserType::Reg, true, &mut items_cache).await
&& !is_err_no_such_policy(&err)
{
@@ -1193,7 +1191,7 @@ impl Store for ObjectStore {
for item in item_name_list.iter() {
let name = rustfs_utils::path::dir(item);
info!("load svc user: {}", name);
debug!(user = %name, "IAM service user loaded");
if let Err(err) = self.load_user(&name, UserType::Svc, &mut items_cache).await
&& !is_err_no_such_user(&err)
{
@@ -1204,7 +1202,7 @@ impl Store for ObjectStore {
for (_, v) in items_cache.iter() {
let parent = v.credentials.parent_user.clone();
if !user_items_cache.contains_key(&parent) {
info!("load sts user policy: {}", parent);
debug!(user = %parent, "IAM STS parent policy loaded");
if let Err(err) = self
.load_mapped_policy(&parent, UserType::Sts, false, &mut sts_policies_cache)
.await
@@ -1223,12 +1221,12 @@ impl Store for ObjectStore {
// sts users
if let Some(item_name_list) = listed_config_items.get(STS_LIST_KEY) {
for item in item_name_list.iter() {
info!("load sts user path: {}", item);
debug!(path = %item, "IAM STS user path discovered");
let name = rustfs_utils::path::dir(item);
info!("load sts user: {}", name);
debug!(user = %name, "IAM STS user loaded");
if let Err(err) = self.load_user(&name, UserType::Sts, &mut sts_items_cache).await {
info!("load sts user failed: {}", err);
debug!(user = %name, error = %err, "IAM STS user load failed");
};
}
}
@@ -1237,12 +1235,12 @@ impl Store for ObjectStore {
if let Some(item_name_list) = listed_config_items.get(POLICY_DB_STS_USERS_LIST_KEY) {
for item in item_name_list.iter() {
let name = item.trim_end_matches(".json");
info!("load sts user policy: {}", name);
debug!(user = %name, "IAM STS user policy loaded");
if let Err(err) = self
.load_mapped_policy(name, UserType::Sts, false, &mut sts_policies_cache)
.await
{
info!("load sts user policy failed: {}", err);
debug!(user = %name, error = %err, "IAM STS user policy load failed");
};
}
}
@@ -1264,7 +1262,7 @@ impl Store for ObjectStore {
cache.replace_sts_policies(sts_policies_cache);
cache.build_user_group_memberships();
} else {
warn!("skip IAM full reload cache commit because one or more IAM caches changed during reload");
warn!("IAM full reload cache commit skipped due to concurrent cache changes");
}
});
crates/kms/src/backends/local.rs
+12 -12
View File
@@ -34,7 +34,7 @@ use std::path::PathBuf;
use std::time::Duration;
use tokio::fs;
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use tracing::{debug, warn};
/// Local KMS client that stores keys in local files
pub struct LocalKmsClient {
@@ -74,7 +74,7 @@ impl LocalKmsClient {
// Create key directory if it doesn't exist
if !config.key_dir.exists() {
fs::create_dir_all(&config.key_dir).await?;
info!("Created KMS key directory: {:?}", config.key_dir);
debug!(path = ?config.key_dir, "KMS key directory created");
}
// Initialize master cipher if master key is provided
@@ -219,7 +219,7 @@ impl LocalKmsClient {
fs::rename(&temp_path, &key_path).await?;
info!("Saved master key {} to {:?}", master_key.key_id, key_path);
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
Ok(())
}
@@ -278,7 +278,7 @@ impl KmsClient for LocalKmsClient {
let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
info!("Generated data key for master key: {}", request.master_key_id);
debug!(key_id = %request.master_key_id, "Local KMS data key generated");
Ok(data_key)
}
@@ -334,7 +334,7 @@ impl KmsClient for LocalKmsClient {
.decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce)
.await?;
info!("Successfully decrypted data");
debug!("Local KMS data decrypted");
Ok(plaintext)
}
@@ -367,7 +367,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone());
info!("Created master key: {}", key_id);
debug!(key_id, "Local KMS master key created");
Ok(master_key)
}
@@ -453,7 +453,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
info!("Enabled key: {}", key_id);
debug!(key_id, "Local KMS key enabled");
Ok(())
}
@@ -470,7 +470,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
info!("Disabled key: {}", key_id);
debug!(key_id, "Local KMS key disabled");
Ok(())
}
@@ -492,7 +492,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
warn!("Scheduled key deletion: {}", key_id);
debug!(key_id, "Local KMS key deletion scheduled");
Ok(())
}
@@ -509,7 +509,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
info!("Canceled deletion for key: {}", key_id);
debug!(key_id, "Local KMS key deletion canceled");
Ok(())
}
@@ -528,7 +528,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone());
info!("Rotated key: {}", key_id);
debug!(key_id, "Local KMS key rotated");
Ok(master_key)
}
@@ -721,7 +721,7 @@ impl KmsBackend for LocalKmsBackend {
let mut cache = self.client.key_cache.write().await;
cache.remove(key_id);
info!("Immediately deleted key: {}", key_id);
debug!(key_id, "Local KMS key deleted immediately");
// Return success response for immediate deletion
let key_metadata = KeyMetadata {
crates/kms/src/backends/vault.rs
+13 -13
View File
@@ -98,7 +98,7 @@ impl VaultKmsClient {
let client =
VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}")))?;
info!("Successfully connected to Vault at {}", config.address);
info!(address = %config.address, "Vault KMS backend connected");
Ok(Self {
client,
@@ -136,7 +136,7 @@ impl VaultKmsClient {
// If encrypted_key_material is empty, generate and store it (fix for old keys)
if key_data.encrypted_key_material.is_empty() {
warn!("Key {} has empty encrypted_key_material, generating and storing new key material", key_id);
warn!(key_id, "Vault KMS key material missing; regenerating");
let key_material = generate_key_material(&key_data.algorithm)?;
key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
// Store the updated key data back to Vault
@@ -147,7 +147,7 @@ impl VaultKmsClient {
let key_material = match self.decrypt_key_material(&key_data.encrypted_key_material).await {
Ok(km) => km,
Err(e) => {
warn!("Failed to decrypt key material for key {}: {}, generating new key material", key_id, e);
warn!(key_id, error = %e, "Vault KMS key material decrypt failed; regenerating");
let new_key_material = generate_key_material(&key_data.algorithm)?;
key_data.encrypted_key_material = self.encrypt_key_material(&new_key_material).await?;
// Store the updated key data back to Vault
@@ -210,7 +210,7 @@ impl VaultKmsClient {
// If encrypted_key_material is empty, generate it (this handles the case where
// an old key was created without proper key material)
if existing_key_data.encrypted_key_material.is_empty() {
warn!("Key {} has empty encrypted_key_material, generating new key material", key_id);
warn!(key_id, "Vault KMS key metadata missing encrypted key material");
let key_material = generate_key_material(&existing_key_data.algorithm)?;
existing_key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
}
@@ -316,7 +316,7 @@ impl KmsClient for VaultKmsClient {
let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
info!("Generated data key for master key: {}", request.master_key_id);
debug!(key_id = %request.master_key_id, "Vault KMS data key generated");
Ok(data_key)
}
@@ -373,7 +373,7 @@ impl KmsClient for VaultKmsClient {
.decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce)
.await?;
info!("Successfully decrypted data");
debug!("Vault KMS data decrypted");
Ok(plaintext)
}
@@ -418,7 +418,7 @@ impl KmsClient for VaultKmsClient {
created_by: None,
};
info!("Successfully created master key: {}", key_id);
debug!(key_id, "Vault KMS master key created");
Ok(master_key)
}
@@ -486,7 +486,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Active;
self.store_key_data(key_id, &key_data).await?;
info!("Enabled key: {}", key_id);
debug!(key_id, "Vault KMS key enabled");
Ok(())
}
@@ -497,7 +497,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Disabled;
self.store_key_data(key_id, &key_data).await?;
info!("Disabled key: {}", key_id);
debug!(key_id, "Vault KMS key disabled");
Ok(())
}
@@ -513,7 +513,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::PendingDeletion;
self.store_key_data(key_id, &key_data).await?;
info!("Scheduled key deletion: {}", key_id);
debug!(key_id, "Vault KMS key deletion scheduled");
Ok(())
}
@@ -524,7 +524,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Active;
self.store_key_data(key_id, &key_data).await?;
info!("Canceled key deletion: {}", key_id);
debug!(key_id, "Vault KMS key deletion canceled");
Ok(())
}
@@ -553,7 +553,7 @@ impl KmsClient for VaultKmsClient {
created_by: None,
};
info!("Successfully rotated key: {}", key_id);
debug!(key_id, "Vault KMS key rotated");
Ok(master_key)
}
@@ -573,7 +573,7 @@ impl KmsClient for VaultKmsClient {
debug!("Vault health check passed - 404 error is expected when no keys exist yet");
Ok(())
} else {
warn!("Vault health check failed: {}", e);
warn!(error = %e, "Vault KMS health check failed");
Err(e)
}
}
crates/kms/src/service.rs
+19 -7
View File
@@ -24,7 +24,7 @@ use rand::random;
use std::collections::HashMap;
use std::io::Cursor;
use tokio::io::{AsyncRead, AsyncReadExt};
use tracing::{debug, info};
use tracing::debug;
use zeroize::Zeroize;
/// Data key for object encryption
@@ -209,7 +209,7 @@ impl ObjectEncryptionService {
// Generate a unique random nonce for this data key
// This ensures each object/part gets a unique base nonce for streaming encryption
let nonce: [u8; 12] = random();
tracing::info!("Generated random nonce for data key: {:02x?}", nonce);
tracing::debug!("Generated random nonce for data key");
let data_key = DataKey {
plaintext_key: data_key_response
@@ -302,7 +302,7 @@ impl ObjectEncryptionService {
key_id: actual_key_id.to_string(),
};
if let Err(KmsError::KeyNotFound { .. }) = self.kms_manager.describe_key(describe_req).await {
info!("Auto-creating SSE-S3 key: {}", actual_key_id);
debug!(key_id = %actual_key_id, "Auto-creating SSE-S3 key");
let create_req = CreateKeyRequest {
key_name: Some(actual_key_id.to_string()),
key_usage: KeyUsage::EncryptDecrypt,
@@ -364,7 +364,13 @@ impl ObjectEncryptionService {
encrypted_data_key: data_key.ciphertext_blob,
};
info!("Successfully encrypted object {}/{} ({} bytes)", bucket, object_key, original_size);
debug!(
bucket,
object = object_key,
original_size,
algorithm = %algorithm.as_str(),
"Object encrypted"
);
Ok(EncryptionResult { ciphertext, metadata })
}
@@ -429,7 +435,13 @@ impl ObjectEncryptionService {
// Decrypt the data
let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?;
info!("Successfully decrypted object {}/{} ({} bytes)", bucket, object_key, plaintext.len());
debug!(
bucket,
object = object_key,
plaintext_len = plaintext.len(),
algorithm = %metadata.algorithm,
"Object decrypted"
);
Ok(Box::new(Cursor::new(plaintext)))
}
@@ -507,7 +519,7 @@ impl ObjectEncryptionService {
encrypted_data_key: Vec::new(), // Empty for SSE-C
};
info!(
debug!(
"Successfully encrypted object {}/{} with SSE-C ({} bytes)",
bucket, object_key, original_size
);
@@ -568,7 +580,7 @@ impl ObjectEncryptionService {
// Decrypt the data
let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?;
info!(
debug!(
"Successfully decrypted SSE-C object {}/{} ({} bytes)",
bucket,
object_key,
crates/kms/src/service_manager.rs
+61 -11
View File
@@ -25,7 +25,11 @@ use std::sync::{
atomic::{AtomicU64, Ordering},
};
use tokio::sync::{Mutex, RwLock};
use tracing::{error, info, warn};
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_KMS: &str = "kms";
const LOG_SUBSYSTEM_SERVICE: &str = "service";
const EVENT_KMS_SERVICE_STATE: &str = "kms_service_state";
/// KMS service status
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
@@ -106,7 +110,13 @@ impl KmsServiceManager {
*status = KmsServiceStatus::Configured;
}
info!("KMS configuration updated successfully");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "configured",
"KMS service configured"
);
Ok(())
}
@@ -132,7 +142,14 @@ impl KmsServiceManager {
}
};
info!("Starting KMS service with backend: {:?}", config.backend);
info!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
backend = ?config.backend,
state = "starting",
"KMS service starting"
);
match self.create_service_version(&config).await {
Ok(service_version) => {
@@ -146,7 +163,13 @@ impl KmsServiceManager {
*status = KmsServiceStatus::Running;
}
info!("KMS service started successfully");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "running",
"KMS service running"
);
Ok(())
}
Err(e) => {
@@ -170,7 +193,13 @@ impl KmsServiceManager {
/// Internal stop implementation (called within lifecycle mutex)
async fn stop_internal(&self) -> Result<()> {
info!("Stopping KMS service");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "stopping",
"KMS service stopping"
);
// Atomically clear current service version (lock-free, instant)
// Note: Existing Arc references will keep the service alive until operations complete
@@ -184,7 +213,13 @@ impl KmsServiceManager {
}
}
info!("KMS service stopped successfully (existing operations may continue)");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "configured",
"KMS service stopped"
);
Ok(())
}
@@ -201,7 +236,13 @@ impl KmsServiceManager {
pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> {
let _guard = self.lifecycle_mutex.lock().await;
info!("Reconfiguring KMS service (zero-downtime)");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "reconfiguring",
"KMS service reconfiguring"
);
new_config.validate()?;
// Configure with new config
@@ -230,13 +271,22 @@ impl KmsServiceManager {
if let Some(old_ver) = old_version {
info!(
"KMS service reconfigured successfully: version {} -> {} (old service will be cleaned up when operations complete)",
old_ver, new_service_version.version
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
old_version = old_ver,
new_version = new_service_version.version,
state = "running",
"KMS service reconfigured"
);
} else {
info!(
"KMS service reconfigured successfully: version {} (service started)",
new_service_version.version
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
new_version = new_service_version.version,
state = "running",
"KMS service started from reconfigure"
);
}
Ok(())
crates/notify/src/bucket_config_manager.rs
+4 -4
View File
@@ -69,7 +69,7 @@ impl NotifyBucketConfigManager {
bucket = %bucket,
region = %cfg.region,
available_arn_count = arn_list.len(),
"Loaded available notify target ARNs for bucket config validation"
"notify bucket config validation"
);
if let Err(e) = cfg.validate(&cfg.region, &arn_list) {
@@ -81,7 +81,7 @@ impl NotifyBucketConfigManager {
region = %cfg.region,
error = %e,
result = "validation_failed",
"Bucket notification config validation failed"
"notify bucket config validation"
);
if !matches!(e, ParseConfigError::ArnNotFound(_)) {
return Err(NotificationError::BucketNotification(e.to_string()));
@@ -94,7 +94,7 @@ impl NotifyBucketConfigManager {
region = %cfg.region,
error = %e,
result = "missing_target_arn",
"Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules"
"notify bucket config validation"
);
}
@@ -107,7 +107,7 @@ impl NotifyBucketConfigManager {
bucket = %bucket,
region = %cfg.region,
rule_count = cfg.get_rules_map().inner().len(),
"Loaded bucket notification config"
"notify bucket config state"
);
Ok(())
}
crates/notify/src/config_manager.rs
+15 -15
View File
@@ -91,7 +91,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "initializing",
"Initializing notification system"
"notify runtime lifecycle"
);
let config = {
@@ -111,7 +111,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "targets_created",
target_count = targets.len(),
"Created notification targets"
"notify runtime lifecycle"
);
if targets.is_empty() {
debug!(
@@ -121,7 +121,7 @@ impl NotifyConfigManager {
state = "idle",
reason = "no_targets_configured",
hint = %notify_configuration_hint(),
"Notification runtime has no configured targets"
"notify runtime lifecycle"
);
}
@@ -132,7 +132,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "initialized",
"Initialized notification system"
"notify runtime lifecycle"
);
Ok(())
}
@@ -156,13 +156,13 @@ impl NotifyConfigManager {
if let Some(targets_of_type) = config.0.get_mut(&ttype) {
if targets_of_type.remove(&tname).is_some() {
info!(
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "remove_target",
target_id = %target_id,
result = "removed",
"Removed notification target from configuration"
"notify config update"
);
changed = true;
}
@@ -178,7 +178,7 @@ impl NotifyConfigManager {
action = "remove_target",
target_id = %target_id,
result = "not_found",
"Notification target not found in configuration"
"notify config update"
);
}
changed
@@ -246,7 +246,7 @@ impl NotifyConfigManager {
target_type = %target_type,
target_name = %target_name,
result = "not_found",
"Notification target configuration not found"
"notify config update"
);
}
debug!(
@@ -264,7 +264,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "reloading",
"Reloading notification configuration"
"notify runtime lifecycle"
);
self.update_config(new_config.clone()).await;
@@ -281,7 +281,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "targets_created",
target_count = targets.len(),
"Created notification targets from reloaded configuration"
"notify runtime lifecycle"
);
if targets.is_empty() {
debug!(
@@ -291,7 +291,7 @@ impl NotifyConfigManager {
state = "idle",
reason = "no_targets_configured",
hint = %notify_configuration_hint(),
"Notification runtime has no configured targets after reload"
"notify runtime lifecycle"
);
}
@@ -302,7 +302,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "reloaded",
"Reloaded notification configuration"
"notify runtime lifecycle"
);
Ok(())
}
@@ -333,7 +333,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "reload_if_changed",
result = "unchanged",
"Notification configuration unchanged; skipping reload"
"notify config update"
);
return Ok(());
}
@@ -348,7 +348,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "reload_if_changed",
result = "updated",
"Notification configuration updated; reloading runtime"
"notify config update"
);
self.reload_config(new_config).await
}
crates/notify/src/global.rs
+10 -1
View File
@@ -23,6 +23,9 @@ use std::sync::{Arc, OnceLock};
use tracing::error;
static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new();
const LOG_COMPONENT_NOTIFY: &str = "notify";
const LOG_SUBSYSTEM_GLOBAL: &str = "global";
const EVENT_NOTIFY_GLOBAL_STATE: &str = "notify_global_state";
/// Initialize the global notification system with the given configuration.
/// This function should only be called once throughout the application life cycle.
@@ -102,7 +105,13 @@ pub mod notifier_global {
// If the notification system itself cannot be retrieved, it will be returned directly
Some(sys) => sys,
None => {
error!("Notification system is not initialized.");
error!(
event = EVENT_NOTIFY_GLOBAL_STATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_GLOBAL,
state = "uninitialized",
"notify global state"
);
return;
}
};
crates/notify/src/integration.rs
+3 -3
View File
@@ -372,7 +372,7 @@ impl Drop for NotificationSystem {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_INTEGRATION,
state = "dropping",
"Notification system instance is being dropped"
"notify system integration state"
);
let snapshot = self.snapshot_metrics();
@@ -394,7 +394,7 @@ impl Drop for NotificationSystem {
metric_name = name,
metric_value = value,
metric_kind = if is_gauge { "gauge" } else { "counter" },
"Notification shutdown metric snapshot"
"notify system integration state"
);
}
@@ -406,7 +406,7 @@ impl Drop for NotificationSystem {
subsystem = LOG_SUBSYSTEM_INTEGRATION,
status_key = %key,
status_value = %value,
"Notification system status snapshot"
"notify system integration state"
);
}
}
crates/notify/src/notifier.rs
+6 -6
View File
@@ -96,7 +96,7 @@ impl EventNotifier {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
state = "targets_cleared",
"Removed all notify targets"
"notify runtime lifecycle"
);
}
@@ -136,7 +136,7 @@ impl EventNotifier {
bucket = %bucket_name,
object = %object_key,
target_count = target_ids_len,
"Dispatching notify event"
"notify dispatch"
);
for target_id in target_ids {
// `get` now returns Option<Arc<dyn Target + Send + Sync>>
@@ -166,7 +166,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_for_task.id(),
deferred = is_deferred,
"Prepared notify target dispatch"
"notify dispatch"
);
// Use cloned data in closures to avoid borrowing conflicts
// Create an EntityTarget from the event
@@ -217,7 +217,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_name_for_task,
deferred = is_deferred,
"Completed notify target dispatch"
"notify dispatch"
);
}
});
@@ -256,7 +256,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH,
bucket = %bucket_name,
target_count = target_ids_len,
"Finished notify dispatch fan-out"
"notify dispatch"
);
}
@@ -284,7 +284,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH,
state = "targets_initialized",
target_count = target_list_guard.len(),
"Initialized notify runtime targets"
"notify runtime lifecycle"
);
Ok(())
}
crates/notify/src/rule_engine.rs
+2 -2
View File
@@ -69,7 +69,7 @@ impl NotifyRuleEngine {
bucket = %bucket,
state = "updated",
event_count,
"Updated bucket notification rules"
"notify bucket rules state"
);
}
@@ -85,7 +85,7 @@ impl NotifyRuleEngine {
subsystem = LOG_SUBSYSTEM_RULE_ENGINE,
bucket = %bucket,
state = "removed",
"Removed bucket notification rules"
"notify bucket rules state"
);
}
}
crates/notify/src/runtime_facade.rs
+5 -5
View File
@@ -66,7 +66,7 @@ impl NotifyRuntimeFacade {
subsystem = LOG_SUBSYSTEM_RUNTIME,
target_id = %target_id,
state = "replay_started",
"Started notify replay worker"
"notify runtime lifecycle"
);
} else {
debug!(
@@ -76,7 +76,7 @@ impl NotifyRuntimeFacade {
target_id = %target_id,
state = "replay_skipped",
reason = "no_store_configured",
"Skipped notify replay worker startup"
"notify runtime lifecycle"
);
}
}),
@@ -122,7 +122,7 @@ impl NotifyRuntimeFacade {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "stopping",
"Stopping notification runtime"
"notify runtime lifecycle"
);
let active_targets = self.replay_workers.read().await.len();
@@ -132,7 +132,7 @@ impl NotifyRuntimeFacade {
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "replay_stopping",
active_targets,
"Stopping notify replay workers"
"notify runtime lifecycle"
);
{
@@ -160,7 +160,7 @@ impl NotifyRuntimeFacade {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "stopped",
"Stopped notification runtime"
"notify runtime lifecycle"
);
}
}
crates/obs/src/telemetry/dial9.rs
+2 -2
View File
@@ -203,7 +203,7 @@ impl Dial9SessionGuard {
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_DIAL9,
state = "shutdown_requested",
"dial9 state changed"
"dial9 state"
);
// TelemetryGuard handles flushing automatically when dropped
}
@@ -219,7 +219,7 @@ impl Drop for Dial9SessionGuard {
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_DIAL9,
state = "flushed",
"dial9 state changed"
"dial9 state"
);
}
}
crates/obs/src/telemetry/local.rs
+3 -3
View File
@@ -175,7 +175,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
output_format = "json",
logger_level,
is_production,
"local logging state changed"
"local logging state"
);
OtelGuard {
@@ -293,7 +293,7 @@ fn init_file_logging_internal(
stdout_mirror_enabled = stdout_guard.is_some(),
is_production,
logger_level,
"local logging state changed"
"local logging state"
);
Ok(OtelGuard {
@@ -482,7 +482,7 @@ pub fn spawn_cleanup_task(
zstd_level,
zstd_fallback_to_gzip,
zstd_workers,
"log cleaner state changed"
"log cleaner state"
);
tokio::spawn(async move {
crates/protocols/src/ftps/driver.rs
+5 -5
View File
@@ -655,7 +655,7 @@ where
state = "requested",
username = %masked_username,
path = %path_str,
"ftps object delete state changed"
"FTPS delete requested"
);
let (bucket, key) = self
@@ -749,7 +749,7 @@ where
username = %masked_username,
path = %path_str,
bucket = %bucket,
"ftps directory state changed"
"FTPS directory created"
);
Ok(())
}
@@ -793,7 +793,7 @@ where
state = "removed",
path = %path_str,
bucket = %bucket,
"ftps directory state changed"
"FTPS directory removed"
);
Ok(())
}
@@ -807,7 +807,7 @@ where
subsystem = LOG_SUBSYSTEM_FTPS_DRIVER,
state = "already_removed",
bucket = %bucket,
"ftps directory state changed"
"FTPS directory already removed"
);
Ok(())
} else {
@@ -877,7 +877,7 @@ where
username = %MaskedAccessKey(&user.username),
from = %from_str,
to = %to_str,
"ftps rename state changed"
"FTPS rename unsupported"
);
Err(Error::new(
crates/protocols/src/ftps/server.rs
+24 -24
View File
@@ -107,7 +107,7 @@ where
bind_addr = %self.config.bind_addr,
tls_enabled = self.config.tls_enabled,
ftps_required = self.config.ftps_required,
"ftps server state changed"
"FTPS server starting"
);
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
@@ -121,14 +121,14 @@ where
// Configure passive ports for data connections
if let Some(passive_ports) = &self.config.passive_ports {
let range = self.config.parse_passive_ports()?;
info!(
debug!(
event = EVENT_FTPS_CONFIG_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "passive_ports_configured",
passive_ports = %passive_ports,
passive_port_range = ?range,
"ftps config state changed"
"FTPS passive ports configured"
);
server_builder = server_builder.passive_ports(range);
} else {
@@ -137,19 +137,19 @@ where
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
result = "system_assigned_passive_ports",
"ftps config state changed"
"FTPS passive ports defaulted"
);
}
// Configure external IP address for passive mode
if let Some(ref external_ip) = self.config.external_ip {
info!(
debug!(
event = EVENT_FTPS_CONFIG_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "external_ip_configured",
external_ip = %external_ip,
"ftps config state changed"
"FTPS external IP configured"
);
server_builder = server_builder.passive_host(external_ip.as_str());
}
@@ -157,13 +157,13 @@ where
// Configure both active and passive mode support
use libunftp::options::ActivePassiveMode;
server_builder = server_builder.active_passive_mode(ActivePassiveMode::ActiveAndPassive);
info!(
debug!(
event = EVENT_FTPS_CONFIG_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "active_passive_mode_enabled",
mode = "active_and_passive",
"ftps config state changed"
"FTPS active/passive mode configured"
);
// Configure FTPS / TLS
@@ -175,7 +175,7 @@ where
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "enabled",
cert_dir = %cert_dir,
"ftps tls state changed"
"FTPS TLS enabled"
);
let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
@@ -197,12 +197,12 @@ where
server_builder = server_builder.ftps_manual::<std::path::PathBuf>(Arc::new(server_config));
if self.config.ftps_required {
info!(
debug!(
event = EVENT_FTPS_TLS_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "required",
"ftps tls state changed"
"FTPS TLS required"
);
server_builder = server_builder.ftps_required(FtpsRequired::All, FtpsRequired::All);
}
@@ -218,7 +218,7 @@ where
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "disabled",
mode = "plain_ftp",
"ftps tls state changed"
"FTPS TLS disabled"
);
}
@@ -248,13 +248,13 @@ where
let _ = reload_shutdown_tx.send(true);
match result {
Ok(Ok(())) => {
info!(
debug!(
event = EVENT_FTPS_SERVER_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "stopped",
result = "ok",
"ftps server state changed"
"FTPS server stopped"
);
Ok(())
}
@@ -288,7 +288,7 @@ where
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "shutdown_requested",
"ftps server state changed"
"FTPS shutdown requested"
);
let _ = reload_shutdown_tx.send(true);
// libunftp listen() is not easily cancellable gracefully without dropping the future.
@@ -330,7 +330,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
result = "iam_unavailable",
phase = "user_detail",
error = %e,
"ftps auth state changed"
"FTPS user-detail IAM unavailable"
);
UserDetailError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e)))
})?;
@@ -344,7 +344,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
phase = "user_detail",
username = %masked_username,
error = %e,
"ftps auth state changed"
"FTPS user-detail key check failed"
);
UserDetailError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e)))
})?;
@@ -357,7 +357,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
result = "identity_missing",
phase = "user_detail",
username = %masked_username,
"ftps auth state changed"
"FTPS user-detail identity missing"
);
UserDetailError::UserNotFound {
username: principal.username.clone(),
@@ -406,7 +406,7 @@ impl Authenticator for FtpsAuthenticator {
result = "iam_unavailable",
phase = "authenticate",
error = %e,
"ftps auth state changed"
"FTPS auth IAM unavailable"
);
AuthenticationError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e)))
})?;
@@ -433,7 +433,7 @@ impl Authenticator for FtpsAuthenticator {
phase = "authenticate",
username = %masked_username,
error = %e,
"ftps auth state changed"
"FTPS auth key check failed"
);
AuthenticationError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e)))
})?;
@@ -446,7 +446,7 @@ impl Authenticator for FtpsAuthenticator {
result = "invalid_access_key",
phase = "authenticate",
username = %masked_username,
"ftps auth state changed"
"FTPS auth rejected access key"
);
return Err(AuthenticationError::BadUser);
}
@@ -459,7 +459,7 @@ impl Authenticator for FtpsAuthenticator {
result = "identity_missing",
phase = "authenticate",
username = %masked_username,
"ftps auth state changed"
"FTPS auth identity missing"
);
AuthenticationError::BadUser
})?;
@@ -472,7 +472,7 @@ impl Authenticator for FtpsAuthenticator {
result = "invalid_secret_key",
phase = "authenticate",
username = %masked_username,
"ftps auth state changed"
"FTPS auth rejected secret key"
);
return Err(AuthenticationError::BadPassword);
}
@@ -484,7 +484,7 @@ impl Authenticator for FtpsAuthenticator {
result = "authenticated",
phase = "authenticate",
username = %masked_username,
"ftps auth state changed"
"FTPS auth accepted"
);
Ok(Principal {
username: username.to_string(),
crates/protocols/src/sftp/driver.rs
+2 -2
View File
@@ -188,7 +188,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
peer = %self.session_context.source_ip,
user = %MaskedAccessKey(&self.session_context.principal.user_identity.credentials.access_key),
result = "read_only_rejected",
"sftp driver state changed"
"SFTP write rejected by read-only mode"
);
return Err(SftpError::code(StatusCode::PermissionDenied));
}
@@ -250,7 +250,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
op = op,
timeout_secs = self.backend_op_timeout_secs,
result = "timeout",
"sftp backend state changed"
"SFTP backend operation timed out"
);
Err(SftpError::code(StatusCode::Failure))
}
crates/protocols/src/sftp/server.rs
+8 -8
View File
@@ -255,7 +255,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
state = "enabled",
host_key_dir = %config.host_key_dir.display(),
interval_secs,
"sftp host key reload state changed"
"SFTP host-key reload enabled"
);
tokio::spawn(async move {
@@ -265,13 +265,13 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
loop {
tokio::select! {
_ = shutdown_token.cancelled() => {
info!(
debug!(
event = EVENT_SFTP_HOST_KEY_RELOAD_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
state = "stopped",
host_key_dir = %config.host_key_dir.display(),
"sftp host key reload state changed"
"SFTP host-key reload stopped"
);
break;
}
@@ -280,14 +280,14 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
match holder.reload_from_config(&config).await {
Ok(Some(host_key_count)) => {
info!(
debug!(
event = EVENT_SFTP_HOST_KEY_RELOAD_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
result = "reloaded",
host_key_dir = %config.host_key_dir.display(),
host_key_count,
"sftp host key reload state changed"
"SFTP host keys reloaded"
);
}
Ok(None) => {
@@ -297,7 +297,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
result = "unchanged",
host_key_dir = %config.host_key_dir.display(),
"sftp host key reload state changed"
"SFTP host keys unchanged"
);
}
Err(err) => {
@@ -308,7 +308,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
result = "reload_failed",
host_key_dir = %config.host_key_dir.display(),
err = %err,
"sftp host key reload state changed"
"SFTP host-key reload failed"
);
}
}
@@ -398,7 +398,7 @@ where
state = "listening",
bind_addr = %self.config.bind_addr,
read_only = self.config.read_only,
"sftp server state changed"
"SFTP server listening"
);
let mut sessions: JoinSet<()> = JoinSet::new();
crates/protocols/src/sftp/write.rs
+15 -15
View File
@@ -75,7 +75,7 @@ pub(super) fn write_dispatch_byte_count(phase: &WritePhase, part_size: u64, satu
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "byte_count_overflow",
"sftp write state changed"
"SFTP write byte-count overflow"
);
Err(SftpError::code(StatusCode::Failure))
}
@@ -90,7 +90,7 @@ pub(super) fn write_dispatch_byte_count(phase: &WritePhase, part_size: u64, satu
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "handle_poisoned",
"sftp write state changed"
"SFTP write handle poisoned"
);
Err(SftpError::code(StatusCode::Failure))
}
@@ -115,7 +115,7 @@ pub(super) fn write_dispatch_append_bytes(phase: &mut WritePhase, data: &[u8]) -
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "append_on_failed_handle",
"sftp write state changed"
"SFTP write append rejected on failed handle"
);
Err(SftpError::code(StatusCode::Failure))
}
@@ -368,7 +368,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
key = %sanitise_control_bytes(key),
attempt = attempt,
state = "retrying_put_object",
"sftp write state changed",
"SFTP put_object retry scheduled",
);
}
@@ -414,7 +414,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
bucket = %sanitise_control_bytes(bucket),
key = %sanitise_control_bytes(key),
result = "retry_loop_fell_through",
"sftp write state changed",
"SFTP put_object retry loop fell through",
);
Err(SftpError::code(StatusCode::Failure))
}
@@ -459,7 +459,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
upload_id = %upload_id,
part_number = part_number,
result = "etag_missing",
"sftp multipart state changed"
"SFTP multipart part missing etag"
);
SftpError::code(StatusCode::Failure)
})?;
@@ -543,7 +543,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
bucket = %sanitise_control_bytes(bucket),
key = %sanitise_control_bytes(key),
result = "upload_id_missing",
"sftp multipart state changed"
"SFTP multipart upload missing upload_id"
);
SftpError::code(StatusCode::Failure)
})?;
@@ -708,7 +708,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "buffering_phase_lost",
"sftp multipart state changed"
"SFTP multipart buffering phase lost"
);
return Err(SftpError::code(StatusCode::Failure));
}
@@ -772,7 +772,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
key = %key,
limit = S3_MAX_MULTIPART_PARTS,
result = "parts_limit_exceeded",
"sftp multipart state changed",
"SFTP multipart parts limit exceeded",
);
let upload_id_for_fail = upload_id.clone();
let abort_authorized_for_fail = *abort_authorized;
@@ -791,7 +791,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "flush_without_streaming",
"sftp multipart state changed"
"SFTP multipart flush requested without streaming state"
);
return Err(SftpError::code(StatusCode::Failure));
}
@@ -817,7 +817,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "post_upload_phase_missing",
"sftp multipart state changed"
"SFTP multipart post-upload phase missing"
);
Err(SftpError::code(StatusCode::Failure))
}
@@ -887,7 +887,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
err = ?abort_err,
context = context,
result = "abort_failed",
"sftp abort state changed",
"SFTP multipart abort failed",
);
}
} else {
@@ -901,7 +901,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
access_key = %MaskedAccessKey(self.access_key()),
context = context,
result = "abort_skipped_unauthorized",
"sftp abort state changed",
"SFTP multipart abort skipped by policy",
);
}
}
@@ -959,7 +959,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
upload_id = %upload_id,
limit = S3_MAX_MULTIPART_PARTS,
result = "final_part_limit_exceeded",
"sftp multipart state changed",
"SFTP multipart final-part limit exceeded",
);
self.close_abort_or_skip(bucket, key, &upload_id, abort_authorized, "parts-limit breach")
.await;
@@ -1080,7 +1080,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
upload_id = %mp.upload_id,
part_number = part_number,
result = "etag_missing",
"sftp multipart copy state changed"
"SFTP multipart copy part missing etag"
);
SftpError::code(StatusCode::Failure)
})?;
crates/protocols/src/webdav/driver.rs
+14 -14
View File
@@ -423,7 +423,7 @@ where
bucket = %bucket,
object = %key,
file_size,
"webdav object write state changed"
"WebDAV object flush completed"
);
// Buffer already cleared by std::mem::take above
Ok(())
@@ -438,7 +438,7 @@ where
object = %key,
file_size,
error = %e,
"webdav object write state changed"
"WebDAV object flush failed"
);
Err(FsError::GeneralFailure)
}
@@ -1219,7 +1219,7 @@ where
result = "not_found",
bucket = %bucket,
error = %e,
"webdav bucket metadata state changed"
"WebDAV bucket metadata listed"
);
Err(FsError::NotFound)
}
@@ -1279,7 +1279,7 @@ where
state = "created",
bucket = %bucket,
object = %dir_key,
"webdav directory state changed"
"WebDAV directory marker created"
);
return Ok(());
}
@@ -1292,7 +1292,7 @@ where
bucket = %bucket,
object = %dir_key,
error = %e,
"webdav directory state changed"
"WebDAV directory marker create failed"
);
return Err(FsError::GeneralFailure);
}
@@ -1320,7 +1320,7 @@ where
subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
state = "bucket_created",
bucket = %bucket,
"webdav directory state changed"
"WebDAV bucket created"
);
Ok(())
}
@@ -1332,7 +1332,7 @@ where
state = "bucket_create_failed",
bucket = %bucket,
error = %e,
"webdav directory state changed"
"WebDAV bucket create failed"
);
Err(FsError::GeneralFailure)
}
@@ -1465,7 +1465,7 @@ where
state = "deleted",
bucket = %bucket,
object = %key,
"webdav object delete state changed"
"WebDAV object deleted"
);
Ok(())
}
@@ -1478,7 +1478,7 @@ where
bucket = %bucket,
object = %key,
error = %e,
"webdav object delete state changed"
"WebDAV object delete failed"
);
Err(FsError::GeneralFailure)
}
@@ -1529,7 +1529,7 @@ where
dst_bucket = %dst_bucket,
dst_object = %dst_key,
error = %e,
"webdav rename state changed"
"WebDAV rename source delete failed"
);
FsError::GeneralFailure
})?;
@@ -1543,7 +1543,7 @@ where
src_object = %src_key,
dst_bucket = %dst_bucket,
dst_object = %dst_key,
"webdav rename state changed"
"WebDAV file renamed"
);
return Ok(());
}
@@ -1603,7 +1603,7 @@ where
dst_bucket = %dst_bucket,
dst_prefix = %dst_prefix,
error = %e,
"webdav rename state changed"
"WebDAV rename directory listing failed"
);
FsError::GeneralFailure
})?;
@@ -1652,7 +1652,7 @@ where
src_object = %src_key,
dst_bucket = %dst_bucket,
dst_object = %dst_key,
"webdav rename state changed"
"WebDAV rename source not found"
);
return Err(FsError::NotFound);
}
@@ -1666,7 +1666,7 @@ where
src_object = %src_key,
dst_bucket = %dst_bucket,
dst_object = %dst_key,
"webdav rename state changed"
"WebDAV directory renamed"
);
Ok(())
}
crates/protocols/src/webdav/server.rs
+11 -11
View File
@@ -86,7 +86,7 @@ where
bind_addr = %self.config.bind_addr,
tls_enabled = self.config.tls_enabled,
max_body_size = self.config.max_body_size,
"webdav server state changed"
"WebDAV server starting"
);
let listener = TcpListener::bind(self.config.bind_addr).await?;
@@ -96,7 +96,7 @@ where
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "listening",
bind_addr = %self.config.bind_addr,
"webdav server state changed"
"WebDAV server listening"
);
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
@@ -109,7 +109,7 @@ where
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "enabled",
cert_dir = %cert_dir,
"webdav tls state changed"
"WebDAV TLS enabled"
);
let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
@@ -211,7 +211,7 @@ where
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "shutdown_requested",
"webdav server state changed"
"WebDAV shutdown requested"
);
let _ = reload_shutdown_tx.send(true);
break;
@@ -225,7 +225,7 @@ where
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "stopped",
"webdav server state changed"
"WebDAV server stopped"
);
Ok(())
}
@@ -376,7 +376,7 @@ where
result = "iam_unavailable",
source_ip = %source_ip,
error = %e,
"webdav auth state changed"
"WebDAV auth IAM unavailable"
);
WebDavInitError::Server("Internal authentication service unavailable".to_string())
})?;
@@ -403,7 +403,7 @@ where
source_ip = %source_ip,
access_key = %masked_access_key,
error = %e,
"webdav auth state changed"
"WebDAV auth key check failed"
);
WebDavInitError::Server("Authentication verification failed".to_string())
})?;
@@ -416,7 +416,7 @@ where
result = "invalid_access_key",
source_ip = %source_ip,
access_key = %masked_access_key,
"webdav auth state changed"
"WebDAV auth rejected access key"
);
return Err(WebDavInitError::Server("Invalid credentials".to_string()));
}
@@ -429,7 +429,7 @@ where
result = "identity_missing",
source_ip = %source_ip,
access_key = %masked_access_key,
"webdav auth state changed"
"WebDAV auth identity missing"
);
WebDavInitError::Server("User not found".to_string())
})?;
@@ -442,7 +442,7 @@ where
result = "invalid_secret_key",
source_ip = %source_ip,
access_key = %masked_access_key,
"webdav auth state changed"
"WebDAV auth rejected secret key"
);
return Err(WebDavInitError::Server("Invalid credentials".to_string()));
}
@@ -454,7 +454,7 @@ where
result = "authenticated",
source_ip = %source_ip,
access_key = %masked_access_key,
"webdav auth state changed"
"WebDAV auth accepted"
);
Ok(SessionContext::new(
crates/scanner/src/scanner.rs
+24 -24
View File
@@ -97,7 +97,7 @@ fn resolve_scanner_runtime_config() -> crate::runtime_config::ScannerRuntimeConf
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "resolve_failed",
error = %err,
"Scanner runtime config resolution failed; using last applied config"
"Scanner runtime config fallback applied"
);
current_scanner_runtime_config()
}
@@ -179,7 +179,7 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "startup_inspect_failed",
error = %err,
"Scanner startup cache inspection failed; keeping configured startup delay"
"Scanner startup cache inspection failed"
);
return false;
}
@@ -198,7 +198,7 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "startup_decode_failed",
error = %err,
"Scanner startup cache decode failed; skipping startup delay"
"Scanner startup cache decode failed"
);
true
}
@@ -222,7 +222,7 @@ async fn initial_scanner_startup_usage_state(storeapi: &Arc<ECStore>) -> (bool,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "startup_bucket_inspect_failed",
error = %err,
"Scanner startup bucket inspection failed; keeping configured startup delay"
"Scanner startup bucket inspection failed"
);
false
}
@@ -418,7 +418,7 @@ async fn configure_scanner_defaults(storeapi: &Arc<ECStore>) {
replication_active = features.replication,
feature_inspection_failed = features.inspection_failed,
state = "single_disk_defaults_applied",
"Scanner defaults updated for single-disk deployment"
"Scanner defaults applied"
);
} else {
set_scanner_default_speed(ScannerSpeed::Default);
@@ -562,7 +562,7 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "decode_failed",
error = %e,
"Scanner background heal state decode failed"
"Scanner background heal decode failed"
);
BackgroundHealInfo::default()
}),
@@ -577,7 +577,7 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "read_failed",
error = %e,
"Scanner background heal state read failed"
"Scanner background heal read failed"
);
}
BackgroundHealInfo::default()
@@ -605,7 +605,7 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "encode_failed",
error = %e,
"Scanner background heal state encode failed"
"Scanner background heal encode failed"
);
return;
}
@@ -621,7 +621,7 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "save_failed",
error = %e,
"Scanner background heal state save failed"
"Scanner background heal save failed"
);
}
}
@@ -650,7 +650,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "refresh_failed",
error = %err,
"Scanner runtime config refresh failed; using last applied config"
"Scanner runtime config refresh failed"
);
}
let configured_cycle_interval = scanner_cycle_interval();
@@ -685,7 +685,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
cycle = cycle_info.current,
scan_mode = ?scan_mode,
state = "started",
"Scanner cycle state updated"
"Scanner cycle started"
);
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
if let Some(new_heal_info) = background_heal_info_for_scan_start(
@@ -730,7 +730,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(),
state = "budget_reached",
"Scanner cycle stopped after reaching budget"
"Scanner cycle budget reached"
);
let budget_reason = cycle_budget.reason();
emit_scan_cycle_partial_with_source(
@@ -751,7 +751,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
state = "failed",
duration = ?now.elapsed(),
error = %e,
"Scanner cycle state updated"
"Scanner cycle failed"
);
emit_scan_cycle_complete(false, cycle_start.elapsed());
if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) {
@@ -772,7 +772,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(),
state = "budget_reached",
"Scanner cycle stopped after reaching budget"
"Scanner cycle budget reached"
);
global_metrics().finish_scan_cycle_work(cycle_work_start);
let budget_reason = cycle_budget.reason();
@@ -806,7 +806,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
state = "completed",
duration = ?now.elapsed(),
cycles_total = cycle_info.cycle_completed.len(),
"Scanner cycle state updated"
"Scanner cycle completed"
);
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
@@ -830,14 +830,14 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
"Scanner state persistence failed"
);
} else {
info!(
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "saved",
"Scanner state persisted"
"Scanner state saved"
);
}
}
@@ -854,7 +854,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
subsystem = LOG_SUBSYSTEM_RUNTIME,
lock_name = "leader.lock",
state = "acquired",
"Scanner leader lock state updated"
"Scanner leader lock acquired"
);
guard
}
@@ -867,7 +867,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
lock_name = "leader.lock",
state = "contended",
error = ?e,
"Scanner leader lock state updated"
"Scanner leader lock contended"
);
return Ok(());
}
@@ -881,7 +881,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
lock_name = "leader.lock",
state = "create_failed",
error = %e,
"Scanner leader lock state updated"
"Scanner leader lock creation failed"
);
return Ok(());
}
@@ -977,7 +977,7 @@ pub async fn store_data_usage_in_backend(
&& let (Some(new_ts), Some(existing_ts)) = (data_usage_info.last_update, existing.last_update)
&& new_ts <= existing_ts
{
info!(
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
@@ -986,7 +986,7 @@ pub async fn store_data_usage_in_backend(
incoming_last_update = ?new_ts,
existing_last_update = ?existing_ts,
state = "skip_stale_update",
"Scanner data usage persistence skipped stale update"
"Scanner stale data usage update skipped"
);
continue;
}
@@ -1003,7 +1003,7 @@ pub async fn store_data_usage_in_backend(
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "encode_failed",
error = %e,
"Scanner data usage persistence encode failed"
"Scanner data usage encode failed"
);
continue;
}
@@ -1040,7 +1040,7 @@ pub async fn store_data_usage_in_backend(
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "save_failed",
error = %e,
"Scanner data usage persistence failed"
"Scanner data usage save failed"
);
} else {
rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
crates/scanner/src/scanner_folder.rs
+6 -6
View File
@@ -66,7 +66,7 @@ use time::OffsetDateTime;
use tokio::select;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
use tracing::{debug, error, warn};
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_FOLDER: &str = "folder";
@@ -639,7 +639,7 @@ impl ScannerItem {
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
state = "started",
"Scanner lifecycle action evaluation started"
"Scanner lifecycle evaluation started"
);
let versioning_config = match BucketVersioningSys::get(&self.bucket).await {
@@ -990,7 +990,7 @@ impl ScannerItem {
let age = now - mod_time;
age.whole_seconds().max(0)
});
info!(
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_HEAL_ADMISSION,
component = LOG_COMPONENT_SCANNER,
@@ -1003,7 +1003,7 @@ impl ScannerItem {
original_scan_mode = %HealScanMode::Deep.as_str(),
effective_scan_mode = %scan_mode.as_str(),
state = "downgraded_to_normal",
"Scanner heal admission downgraded deep scan during cooldown"
"Scanner heal deep scan downgraded"
);
}
@@ -1688,14 +1688,14 @@ impl FolderScanner {
if found_objects && is_erasure().await {
// If we found an object in erasure mode, we skip subdirs (only datadirs)...
info!(
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name,
state = "erasure_object_found",
"Scanner folder stopped descending after finding erasure object"
"Scanner folder descent stopped after erasure object"
);
break;
}
crates/scanner/src/scanner_io.rs
+14 -14
View File
@@ -900,7 +900,7 @@ impl ScannerIOCache for SetDisks {
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "scan_started",
"Scanner disk bucket state updated"
"Scanner disk bucket scan started"
);
let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]);
@@ -910,13 +910,13 @@ impl ScannerIOCache for SetDisks {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "cache_load_failed",
error = %e,
"Scanner disk bucket state updated"
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "cache_load_failed",
error = %e,
"Scanner disk bucket cache load failed"
);
}
@@ -942,7 +942,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name,
cache_name = ?cache.info.name,
state = "cache_ready",
"Scanner disk bucket state updated"
"Scanner disk bucket cache ready"
);
let (updates_tx, mut updates_rx) = mpsc::channel::<DataUsageEntry>(1);
@@ -997,7 +997,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name,
state = "cancelled",
error = %e,
"Scanner disk bucket state updated"
"Scanner disk bucket scan cancelled"
);
} else {
error!(
@@ -1008,7 +1008,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name,
state = "scan_failed",
error = %e,
"Scanner disk bucket state updated"
"Scanner disk bucket scan failed"
);
}
@@ -1104,7 +1104,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name,
cache_name = %cache.info.name,
state = "scan_completed",
"Scanner disk bucket state updated"
"Scanner disk bucket scan completed"
);
if let Err(e) = update_fut.await {
@@ -1132,7 +1132,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name,
cache_name = %cache.info.name,
state = "send_root_entry",
"Scanner data usage stream progress updated"
"Scanner root entry publish started"
);
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
@@ -1182,7 +1182,7 @@ impl ScannerIOCache for SetDisks {
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "set_scan_completed",
"Scanner set-level disk scan completed"
"Scanner set scan completed"
);
Ok(())
crates/targets/src/store.rs
+108 -16
View File
@@ -30,6 +30,10 @@ use std::{
use tracing::{debug, warn};
use uuid::Uuid;
const LOG_COMPONENT_TARGETS: &str = "targets";
const LOG_SUBSYSTEM_STORE: &str = "store";
const EVENT_TARGET_STORE_STATE: &str = "target_store_state";
fn resolve_queue_store_compression_from_env_value(value: Option<&str>) -> bool {
value
.and_then(|value| value.parse::<EnableState>().ok().map(|state| state.is_enabled()))
@@ -83,7 +87,14 @@ impl std::fmt::Display for Key {
/// Parses a string into a Key
pub fn parse_key(s: &str) -> Key {
debug!("Parsing key: {}", s);
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "parse_key",
key = %s,
"target store state"
);
let mut name = s.to_string();
let mut extension = String::new();
@@ -111,8 +122,16 @@ pub fn parse_key(s: &str) -> Key {
}
debug!(
"Parsed key - name: {}, extension: {}, item_count: {}, compress: {}",
name, extension, item_count, compress
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "parse_key",
result = "parsed",
key_name = %name,
extension = %extension,
item_count,
compressed = compress,
"target store state"
);
Key {
@@ -276,7 +295,15 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let path = self.file_path(key);
debug!("Reading file for key: {},path: {}", key, path.display());
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "read_file",
key = %key,
path = %path.display(),
"target store state"
);
let data = std::fs::read(&path).map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
StoreError::NotFound
@@ -342,7 +369,14 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
std::fs::write(&path, data).map_err(StoreError::Io)?;
}
let modified = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64;
debug!("Wrote event to store: {}", key);
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "write_file",
key = %key,
"target store state"
);
Ok(modified)
}
@@ -409,7 +443,15 @@ where
}
}
debug!("Opened store at: {:?}", self.directory);
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
state = "opened",
store_dir = ?self.directory,
entry_count = entries_map.len(),
"target store state"
);
Ok(())
}
@@ -471,7 +513,14 @@ where
}
fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error> {
debug!("Reading items from store for key: {}", key);
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "read_batch",
key = %key,
"target store state"
);
let data = self.get_raw(key)?;
if data.is_empty() {
return Err(StoreError::Deserialization("Cannot deserialize empty data".to_string()));
@@ -500,10 +549,15 @@ where
if items.len() < key.item_count && !items.is_empty() {
// Partial read
warn!(
"Expected {} items for key {}, but only found {}. Possible data corruption or incorrect item_count.",
key.item_count,
key,
items.len()
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "read_batch",
key = %key,
expected_items = key.item_count,
actual_items = items.len(),
reason = "partial_batch_read",
"target store state"
);
// Depending on strictness, this could be an error.
} else if items.is_empty() {
@@ -538,7 +592,15 @@ where
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// File already gone — still clean up the entries map to avoid stale keys.
warn!("File not found for key {} during del, cleaning up entries map.", key);
warn!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "delete",
key = %key,
result = "file_missing",
"target store state"
);
}
Err(e) => return Err(StoreError::Io(e)),
}
@@ -550,9 +612,25 @@ where
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
if entries.remove(&key.to_key_string()).is_none() {
debug!("Key {} not found in entries map during del, might have been already removed.", key);
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "delete",
key = %key,
result = "entry_missing",
"target store state"
);
}
debug!("Deleted event from store: {}", key.to_string());
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "delete",
key = %key,
result = "deleted",
"target store state"
);
Ok(())
}
@@ -580,7 +658,14 @@ where
let entries = match self.entries.read() {
Ok(entries) => entries,
Err(_) => {
debug!("Failed to acquire read lock on entries for listing");
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "list",
result = "lock_unavailable",
"target store state"
);
return Vec::new();
}
};
@@ -597,7 +682,14 @@ where
match self.entries.read() {
Ok(entries) => entries.len(),
Err(_) => {
debug!("Failed to acquire read lock on entries for len");
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "len",
result = "lock_unavailable",
"target store state"
);
0
}
}
crates/targets/src/target/mqtt.rs
+464 -56
View File
@@ -60,6 +60,10 @@ const DEFAULT_MQTT_TLS_PORT: u16 = 8883;
const DEFAULT_MQTT_WSS_PORT: u16 = 443;
const MAX_MQTT_PACKET_SIZE_BYTES: u32 = 100 * 1024 * 1024;
const DEFAULT_MQTT_WS_PATH_ALLOWLIST: &[&str] = &["/", "/mqtt"];
const LOG_COMPONENT_TARGETS: &str = "targets";
const LOG_SUBSYSTEM_MQTT: &str = "mqtt";
const EVENT_MQTT_TARGET_STATE: &str = "mqtt_target_state";
const EVENT_MQTT_DELIVERY_STATE: &str = "mqtt_delivery_state";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MQTTTlsPolicy {
@@ -574,7 +578,14 @@ where
Some(MAX_MQTT_PACKET_SIZE_BYTES),
)?;
info!(target_id = %target_id, "MQTT target created");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "created",
"mqtt target state"
);
Ok(MQTTTarget::<E> {
id: target_id,
args,
@@ -593,7 +604,14 @@ where
#[instrument(skip(self), fields(target_id = %self.id))]
async fn init(&self) -> Result<(), TargetError> {
if self.connected.load(Ordering::SeqCst) {
debug!(target_id = %self.id, "Already connected.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "already_connected",
"mqtt target state"
);
return Ok(());
}
@@ -607,7 +625,14 @@ where
let _ = bg_task_manager
.init_cell
.get_or_try_init(|| async {
debug!(target_id = %target_id_clone, "Initializing MQTT background task.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "background_task_initializing",
"mqtt target state"
);
// Use the latest MqttOptions (may have been updated by TLS reload coordinator).
let mqtt_options: MqttOptions = (**pending_mqtt_options.load()).clone();
@@ -615,30 +640,67 @@ where
let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build();
if let Err(e) = new_client.subscribe(&args_clone.topic, args_clone.qos).await {
error!(target_id = %target_id_clone, error = %e, "Failed to subscribe to MQTT topic during init");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "subscribe_failed",
error = %e,
"mqtt target state"
);
return Err(TargetError::Network(format!("MQTT subscribe failed: {e}")));
}
let mut rx_guard = bg_task_manager.initial_cancel_rx.lock().await;
let cancel_rx = rx_guard.take().ok_or_else(|| {
error!(target_id = %target_id_clone, "MQTT cancel receiver already taken for task.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "cancel_receiver_unavailable",
"mqtt target state"
);
TargetError::Configuration("MQTT cancel receiver already taken for task".to_string())
})?;
drop(rx_guard);
*client_arc.lock().await = Some(new_client.clone());
info!(target_id = %target_id_clone, "Spawning MQTT event loop task.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "event_loop_spawning",
"mqtt target state"
);
let task_handle =
tokio::spawn(run_mqtt_event_loop(eventloop, connected_arc.clone(), target_id_clone.clone(), cancel_rx));
Ok(task_handle)
})
.await
.map_err(|e: TargetError| {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT background task");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "background_task_init_failed",
error = %e,
"mqtt target state"
);
e
})?;
debug!(target_id = %self.id, "MQTT background task initialized successfully.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "background_task_initialized",
"mqtt target state"
);
match tokio::time::timeout(DEFAULT_CONNECTION_TIMEOUT, async {
while !self.connected.load(Ordering::SeqCst) {
@@ -646,23 +708,51 @@ where
&& handle.is_finished()
&& !self.connected.load(Ordering::SeqCst)
{
error!(target_id = %self.id, "MQTT background task exited prematurely before connection was established.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "background_task_exited_before_connect",
"mqtt target state"
);
return Err(TargetError::Network("MQTT background task exited prematurely".to_string()));
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
debug!(target_id = %self.id, "MQTT target connected successfully.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "connected",
"mqtt target state"
);
Ok(())
})
.await
{
Ok(Ok(_)) => {
info!(target_id = %self.id, "MQTT target initialized and connected.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "ready",
"mqtt target state"
);
Ok(())
}
Ok(Err(e)) => Err(e),
Err(_) => {
error!(target_id = %self.id, "Timeout waiting for MQTT connection after task spawn.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "connect_timeout",
"mqtt target state"
);
Err(TargetError::Network("Timeout waiting for MQTT connection".to_string()))
}
}
@@ -680,12 +770,16 @@ where
.ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
debug!(
target = %self.id,
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
bucket = %meta.bucket_name,
object = %meta.object_name,
event = %meta.event_name,
payload_len = body.len(),
"Sending MQTT payload"
state = "publishing",
"mqtt delivery state"
);
client
@@ -693,7 +787,16 @@ where
.await
.map_err(|e| {
if e.to_string().contains("Connection") || e.to_string().contains("Timeout") {
warn!(target_id = %self.id, error = %e, "Publish failed due to connection issue, marking as not connected.");
warn!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "publish_failed",
reason = "connectivity_error",
error = %e,
"mqtt delivery state"
);
let err = TargetError::NotConnected;
mark_target_disconnected_on_connectivity_error(&self.connected, &err);
err
@@ -702,7 +805,15 @@ where
}
})?;
debug!(target_id = %self.id, topic = %self.args.topic, "Event published to MQTT topic");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
topic = %self.args.topic,
state = "published",
"mqtt delivery state"
);
self.delivery_counters.record_success();
Ok(())
}
@@ -781,14 +892,28 @@ async fn run_mqtt_event_loop(
target_id: TargetID,
mut cancel_rx: mpsc::Receiver<()>,
) {
info!(target_id = %target_id, "MQTT event loop task started.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "event_loop_started",
"mqtt target state"
);
let mut initial_connection_established = false;
loop {
tokio::select! {
biased;
_ = cancel_rx.recv() => {
info!(target_id = %target_id, "MQTT event loop task received cancellation signal. Shutting down.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "cancellation_received",
"mqtt target state"
);
break;
}
polled_event_result = async {
@@ -796,7 +921,14 @@ async fn run_mqtt_event_loop(
match tokio::time::timeout(EVENT_LOOP_POLL_TIMEOUT, eventloop.poll()).await {
Ok(result) => Some(result),
Err(_) => {
debug!(target_id = %target_id, "MQTT poll timed out (EVENT_LOOP_POLL_TIMEOUT) while not connected or status pending.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "poll_timeout",
"mqtt target state"
);
connected_status.store(false, Ordering::SeqCst);
None
}
@@ -810,15 +942,38 @@ async fn run_mqtt_event_loop(
trace!(target_id = %target_id, event = ?notification, "Received MQTT event");
match notification {
rumqttc::Event::Incoming(Incoming::ConnAck(_conn_ack)) => {
info!(target_id = %target_id, "MQTT connected (ConnAck).");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "connack_received",
"mqtt target state"
);
connected_status.store(true, Ordering::SeqCst);
initial_connection_established = true;
}
rumqttc::Event::Incoming(Incoming::Publish(publish)) => {
debug!(target_id = %target_id, topic = ?publish.topic, payload_len = publish.payload.len(), "Received message on subscribed topic.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "publish_received",
topic = ?publish.topic,
payload_len = publish.payload.len(),
"mqtt target state"
);
}
rumqttc::Event::Incoming(Incoming::Disconnect(_)) => {
info!(target_id = %target_id, "Received Disconnect packet from broker. MQTT connection lost.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "broker_disconnected",
"mqtt target state"
);
connected_status.store(false, Ordering::SeqCst);
}
rumqttc::Event::Incoming(Incoming::PingResp(_)) => {
@@ -832,7 +987,14 @@ async fn run_mqtt_event_loop(
}
// Process other incoming packet types as needed (PubRec, PubRel, PubComp, UnsubAck)
rumqttc::Event::Outgoing(Outgoing::Disconnect) => {
info!(target_id = %target_id, "MQTT outgoing disconnect initiated by client.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "client_disconnect_requested",
"mqtt target state"
);
connected_status.store(false, Ordering::SeqCst);
}
rumqttc::Event::Outgoing(Outgoing::PingReq) => {
@@ -848,7 +1010,15 @@ async fn run_mqtt_event_loop(
}
Some(Err(e)) => {
connected_status.store(false, Ordering::SeqCst);
error!(target_id = %target_id, error = %e, "Error from MQTT event loop poll");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "poll_failed",
error = %e,
"mqtt target state"
);
if matches!(e,
ConnectionError::Io(_) |
@@ -856,12 +1026,28 @@ async fn run_mqtt_event_loop(
ConnectionError::ConnectionRefused(_) |
ConnectionError::Tls(_)
) {
warn!(target_id = %target_id, error = %e, "MQTT connection error. Relying on rumqttc for reconnection if applicable.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "reconnect_pending",
error = %e,
"mqtt target state"
);
}
// Here you can decide whether to break loops based on the error type.
// For example, for some unrecoverable errors.
if is_fatal_mqtt_error(&e) {
error!(target_id = %target_id, error = %e, "Fatal MQTT error, terminating event loop.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "fatal_error",
error = %e,
"mqtt target state"
);
break;
}
// rumqttc's eventloop.poll() may return Err and terminate after some errors,
@@ -871,7 +1057,14 @@ async fn run_mqtt_event_loop(
tokio::time::sleep(Duration::from_secs(1)).await;
}
None => {
warn!(target_id = %target_id, "Timeout during initial poll or pending state, will retry.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "poll_retry_scheduled",
"mqtt target state"
);
continue;
}
}
@@ -879,7 +1072,14 @@ async fn run_mqtt_event_loop(
}
}
connected_status.store(false, Ordering::SeqCst);
info!(target_id = %target_id, "MQTT event loop task finished.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "event_loop_finished",
"mqtt target state"
);
}
/// Check whether the given MQTT connection error should be considered a fatal error,
@@ -938,26 +1138,61 @@ where
#[instrument(skip(self), fields(target_id = %self.id))]
async fn is_active(&self) -> Result<bool, TargetError> {
debug!(target_id = %self.id, "Checking if MQTT target is active.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "activity_check",
"mqtt target state"
);
if self.client.lock().await.is_none() && !self.connected.load(Ordering::SeqCst) {
// Check if the background task is running and has not panicked
if let Some(handle) = self.bg_task_manager.init_cell.get()
&& handle.is_finished()
{
error!(target_id = %self.id, "MQTT background task has finished, possibly due to an error. Target is not active.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "inactive_background_task_finished",
"mqtt target state"
);
return Err(TargetError::Network("MQTT background task terminated".to_string()));
}
debug!(target_id = %self.id, "MQTT client not yet initialized or task not running/connected.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "inactive_client_unavailable",
"mqtt target state"
);
return Err(TargetError::Configuration(
"MQTT client not available or not initialized/connected".to_string(),
));
}
if self.connected.load(Ordering::SeqCst) {
debug!(target_id = %self.id, "MQTT target is active (connected flag is true).");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "active",
"mqtt target state"
);
Ok(true)
} else {
debug!(target_id = %self.id, "MQTT target is not connected (connected flag is false).");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "inactive_not_connected",
"mqtt target state"
);
Err(TargetError::NotConnected)
}
}
@@ -973,14 +1208,36 @@ where
};
if let Some(store) = &self.store {
debug!(target_id = %self.id, "Event saved to store start");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_enqueue_started",
"mqtt delivery state"
);
match persist_queued_payload_to_store(store.as_ref(), &queued) {
Ok(_) => {
debug!(target_id = %self.id, "Event saved to store for MQTT target successfully.");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_enqueued",
"mqtt delivery state"
);
Ok(())
}
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to save event to store");
error!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_enqueue_failed",
error = %e,
"mqtt delivery state"
);
self.delivery_counters.record_final_failure();
Err(e)
}
@@ -991,18 +1248,47 @@ where
}
if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Attempting to send directly but not connected; trying to init.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "direct_send_requires_init",
"mqtt target state"
);
// Call the struct's init method, not the trait's default
match MQTTTarget::<E>::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
Ok(_) => debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed",
"mqtt target state"
),
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_failed",
error = %e,
"mqtt target state"
);
self.delivery_counters.record_final_failure();
return Err(TargetError::NotConnected);
}
}
if !self.connected.load(Ordering::SeqCst) {
error!(target_id = %self.id, "Cannot save (send directly) as target is not active after init attempt.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed_not_connected",
"mqtt target state"
);
self.delivery_counters.record_final_failure();
return Err(TargetError::NotConnected);
}
@@ -1017,50 +1303,143 @@ where
#[instrument(skip(self, body, meta), fields(target_id = %self.id))]
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
debug!(target_id = %self.id, ?key, "Attempting to send queued payload from store.");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_started",
"mqtt delivery state"
);
if !self.is_enabled() {
return Err(TargetError::Disabled);
}
if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Not connected; trying to init before sending from store.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_replay_requires_init",
"mqtt target state"
);
match MQTTTarget::<E>::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
Ok(_) => debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed",
"mqtt target state"
),
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_failed",
error = %e,
"mqtt target state"
);
return Err(TargetError::NotConnected);
}
}
if !self.connected.load(Ordering::SeqCst) {
error!(target_id = %self.id, "Cannot send from store as target is not active after init attempt.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed_not_connected",
"mqtt target state"
);
return Err(TargetError::NotConnected);
}
}
debug!(target_id = %self.id, ?key, "Sending event from store.");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_publishing",
"mqtt delivery state"
);
if let Err(e) = self.send_body(body, &meta).await {
if matches!(e, TargetError::NotConnected) {
warn!(target_id = %self.id, "Failed to send event from store: Not connected. Event remains in store.");
warn!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_deferred",
reason = "not_connected",
"mqtt delivery state"
);
return Err(TargetError::NotConnected);
}
error!(target_id = %self.id, error = %e, "Failed to send event from store with an unexpected error.");
error!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_failed",
error = %e,
"mqtt delivery state"
);
return Err(e);
}
debug!(target_id = %self.id, ?key, "Event sent from store successfully.");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_published",
"mqtt delivery state"
);
Ok(())
}
async fn close(&self) -> Result<(), TargetError> {
info!(target_id = %self.id, "Attempting to close MQTT target.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "closing",
"mqtt target state"
);
if let Err(e) = self.bg_task_manager.cancel_tx.send(()).await {
warn!(target_id = %self.id, error = %e, "Failed to send cancel signal to MQTT background task. It might have already exited.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "cancel_signal_failed",
error = %e,
"mqtt target state"
);
}
// Wait for the task to finish if it was initialized
if let Some(_task_handle) = self.bg_task_manager.init_cell.get() {
debug!(target_id = %self.id, "Waiting for MQTT background task to complete...");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "waiting_for_event_loop",
"mqtt target state"
);
// It's tricky to await here if close is called from a sync context or Drop
// For async close, this is fine. Consider a timeout.
// let _ = tokio::time::timeout(Duration::from_secs(5), task_handle.await).await;
@@ -1069,9 +1448,24 @@ where
}
if let Some(client_instance) = self.client.lock().await.take() {
info!(target_id = %self.id, "Disconnecting MQTT client.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "disconnecting_client",
"mqtt target state"
);
if let Err(e) = client_instance.disconnect().await {
warn!(target_id = %self.id, error = %e, "Error during MQTT client disconnect.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "disconnect_failed",
error = %e,
"mqtt target state"
);
}
}
@@ -1083,7 +1477,14 @@ where
}
self.connected.store(false, Ordering::SeqCst);
info!(target_id = %self.id, "MQTT target close method finished.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "closed",
"mqtt target state"
);
Ok(())
}
@@ -1097,7 +1498,14 @@ where
async fn init(&self) -> Result<(), TargetError> {
if !self.is_enabled() {
debug!(target_id = %self.id, "Target is disabled, skipping init.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "disabled",
"mqtt target state"
);
return Ok(());
}
// Call the internal init logic
crates/targets/src/target/webhook.rs
+120 -22
View File
@@ -45,6 +45,11 @@ use std::{
use tokio::sync::mpsc;
use tracing::{debug, error, info, instrument, warn};
const LOG_COMPONENT_TARGETS: &str = "targets";
const LOG_SUBSYSTEM_WEBHOOK: &str = "webhook";
const EVENT_WEBHOOK_TARGET_STATE: &str = "webhook_target_state";
const EVENT_WEBHOOK_DELIVERY_STATE: &str = "webhook_delivery_state";
/// Arguments for configuring a Webhook target
#[derive(Clone)]
pub struct WebhookArgs {
@@ -190,7 +195,14 @@ where
// Create a cancel channel
let (cancel_sender, _) = mpsc::channel(1);
info!(target_id = %target_id.id, "Webhook target created");
info!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %target_id.id,
state = "created",
"webhook target state"
);
Ok(WebhookTarget::<E> {
id: target_id,
args,
@@ -220,8 +232,13 @@ where
// DANGEROUS: For testing only, skip all certificate verification
client_builder = client_builder.danger_accept_invalid_certs(true);
warn!(
"Webhook target '{}' is configured to skip TLS verification. This is insecure and should not be used in production.",
args.endpoint
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
endpoint = %args.endpoint,
state = "tls_verification_skipped",
fallback = "danger_accept_invalid_certs",
"webhook target state"
);
} else if !args.client_ca.is_empty() {
// Use user-provided custom CA certificate
@@ -301,10 +318,14 @@ where
match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await {
Ok(Ok(resp)) => {
debug!(
target = %self.id,
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
status = %resp.status(),
health_check_url = %health_check_url,
"Webhook health check request succeeded"
state = "reachability_probe_succeeded",
"webhook target state"
);
Ok(true)
}
@@ -337,7 +358,15 @@ where
// behavior matches real delivery while avoiding path-specific false negatives.
match self.probe_reachability().await {
Ok(true) => {
debug!("Webhook target {} reachability probe succeeded via {:?}", self.id, self.health_check_url);
debug!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
health_check_url = ?self.health_check_url,
state = "reachable",
"webhook target state"
);
}
Ok(false) => {
return Err(TargetError::NotConnected);
@@ -348,7 +377,14 @@ where
}
self.initialized.store(true, Ordering::SeqCst);
info!("Webhook target {} initialized", self.id);
info!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "initialized",
"webhook target state"
);
Ok(())
}
@@ -357,14 +393,17 @@ where
}
async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
info!("Webhook sending queued payload to target: {}", self.id);
debug!(
target = %self.id,
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
bucket = %meta.bucket_name,
object = %meta.object_name,
event = %meta.event_name,
payload_event = %meta.event_name,
payload_len = body.len(),
"Sending webhook payload"
state = "sending",
"webhook delivery state"
);
// When a TLS reload adapter is attached, it drives client rebuilds in
@@ -406,7 +445,15 @@ where
let status = resp.status();
if status.is_success() {
debug!("Event sent to webhook target: {}", self.id);
debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
status = %status,
state = "sent",
"webhook delivery state"
);
self.delivery_counters.record_success();
Ok(())
} else if status == StatusCode::FORBIDDEN {
@@ -454,13 +501,28 @@ where
self.delivery_counters.record_final_failure();
return Err(e);
}
debug!("Event saved to store for target: {}", self.id);
debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "store_enqueued",
"webhook delivery state"
);
Ok(())
} else {
match self.init().await {
Ok(_) => (),
Err(e) => {
error!("Failed to initialize Webhook target {}: {}", self.id.id, e);
error!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id.id,
state = "init_failed",
error = %e,
"webhook target state"
);
self.delivery_counters.record_final_failure();
return Err(TargetError::NotConnected);
}
@@ -474,13 +536,27 @@ where
}
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
debug!("Sending queued payload from store for target: {}, key: {}", self.id, key);
debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
key = %key,
state = "store_replay_started",
"webhook delivery state"
);
match self.init().await {
Ok(_) => {
debug!("Event sent to store for target: {}", self.name());
}
Ok(_) => {}
Err(e) => {
error!("Failed to initialize Webhook target {}: {}", self.id.id, e);
error!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id.id,
state = "init_failed",
error = %e,
"webhook target state"
);
return Err(TargetError::NotConnected);
}
}
@@ -492,7 +568,15 @@ where
return Err(e);
}
debug!("Event sent from store and deleted for target: {}", self.id);
debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
key = %key,
state = "store_replay_sent",
"webhook delivery state"
);
Ok(())
}
@@ -500,7 +584,14 @@ where
// Send cancel signal to background tasks
let _ = self.cancel_sender.try_send(());
// Adapter cleanup is done by the coordinator; no local state to reset.
info!("Webhook target closed: {}", self.id);
info!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "closed",
"webhook target state"
);
Ok(())
}
@@ -515,7 +606,14 @@ where
async fn init(&self) -> Result<(), TargetError> {
if !self.is_enabled() {
debug!("Webhook target {} is disabled, skipping initialization", self.id);
debug!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "disabled",
"webhook target state"
);
return Ok(());
}
self.init_inner().await
rustfs/src/admin/handlers/bucket_meta.rs
+195 -41
View File
@@ -65,6 +65,10 @@ use time::OffsetDateTime;
use tracing::warn;
use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_BUCKET_META: &str = "bucket_meta";
const EVENT_ADMIN_BUCKET_META_STATE: &str = "admin_bucket_meta_state";
#[derive(Debug, Default, serde::Deserialize)]
pub struct ExportBucketMetadataQuery {
pub bucket: String,
@@ -106,7 +110,7 @@ impl Operation for ExportBucketMetadata {
let query = {
if let Some(query) = req.uri.query() {
let input: ExportBucketMetadataQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
ExportBucketMetadataQuery::default()
@@ -114,7 +118,7 @@ impl Operation for ExportBucketMetadata {
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -131,19 +135,19 @@ impl Operation for ExportBucketMetadata {
.await?;
let Some(store) = new_object_layer_fn() else {
return Err(s3_error!(InvalidRequest, "object store not init"));
return Err(s3_error!(InternalError, "object store is not initialized"));
};
let buckets = if query.bucket.is_empty() {
store
.list_bucket(&BucketOptions::default())
.await
.map_err(|e| s3_error!(InternalError, "list buckets failed: {e}"))?
.map_err(|e| s3_error!(InternalError, "failed to list buckets: {e}"))?
} else {
let bucket = store
.get_bucket_info(&query.bucket, &BucketOptions::default())
.await
.map_err(|e| s3_error!(InternalError, "get bucket failed: {e}"))?;
.map_err(|e| s3_error!(InternalError, "failed to load bucket: {e}"))?;
vec![bucket]
};
@@ -173,17 +177,17 @@ impl Operation for ExportBucketMetadata {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
}
};
let config_json =
serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
let config_json = serde_json::to_vec(&config)
.map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
.map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
zip_writer
.write_all(&config_json)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
.map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
}
BUCKET_NOTIFICATION_CONFIG => {
let config: s3s::dto::NotificationConfiguration =
@@ -215,18 +219,18 @@ impl Operation for ExportBucketMetadata {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
}
};
let config_xml =
serialize(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
serialize(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
.map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
zip_writer
.write_all(&config_xml)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
.map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
}
BUCKET_TAGGING_CONFIG => {
let config: Tagging = match metadata_sys::get_tagging_config(&bucket.name).await {
@@ -235,18 +239,18 @@ impl Operation for ExportBucketMetadata {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
}
};
let config_xml =
serialize(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
serialize(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
.map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
zip_writer
.write_all(&config_xml)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
.map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
}
BUCKET_QUOTA_CONFIG_FILE => {
let config: BucketQuota = match metadata_sys::get_quota_config(&bucket.name).await {
@@ -376,7 +380,7 @@ impl Operation for ExportBucketMetadata {
let zip_bytes = zip_writer
.finish()
.map_err(|e| s3_error!(InternalError, "finish zip failed: {e}"))?;
.map_err(|e| s3_error!(InternalError, "failed to finalize export archive: {e}"))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/zip".parse().unwrap());
header.insert(CONTENT_DISPOSITION, "attachment; filename=bucket-meta.zip".parse().unwrap());
@@ -399,7 +403,7 @@ impl Operation for ImportBucketMetadata {
let _query = {
if let Some(query) = req.uri.query() {
let input: ImportBucketMetadataQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
ImportBucketMetadataQuery::default()
@@ -407,7 +411,7 @@ impl Operation for ImportBucketMetadata {
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -427,24 +431,33 @@ impl Operation for ImportBucketMetadata {
let body = match input.store_all_limited(MAX_BUCKET_METADATA_IMPORT_SIZE).await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "body_read_failed",
error = ?e,
"admin bucket meta state"
);
return Err(s3_error!(InvalidRequest, "bucket metadata import body too large or failed to read"));
}
};
let mut zip_reader = ZipArchive::new(Cursor::new(body)).map_err(|e| s3_error!(InternalError, "get body failed: {e}"))?;
let mut zip_reader =
ZipArchive::new(Cursor::new(body)).map_err(|e| s3_error!(InternalError, "failed to read import archive: {e}"))?;
// First pass: read all file contents into memory
let mut file_contents = Vec::new();
for i in 0..zip_reader.len() {
let mut file = zip_reader
.by_index(i)
.map_err(|e| s3_error!(InternalError, "get file failed: {e}"))?;
.map_err(|e| s3_error!(InternalError, "failed to read archive entry: {e}"))?;
let file_path = file.name().to_string();
let mut content = Vec::new();
file.read_to_end(&mut content)
.map_err(|e| s3_error!(InternalError, "read file failed: {e}"))?;
.map_err(|e| s3_error!(InternalError, "failed to read archive entry content: {e}"))?;
file_contents.push((file_path, content));
}
@@ -455,7 +468,15 @@ impl Operation for ImportBucketMetadata {
let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>();
if file_path_split.len() < 2 {
warn!("file path is invalid: {}", file_path);
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "invalid_file_path",
file_path = %file_path,
"admin bucket meta state"
);
continue;
}
@@ -474,17 +495,33 @@ impl Operation for ImportBucketMetadata {
}
Err(e) => {
if e == StorageError::ConfigNotFound {
warn!("bucket metadata not found: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "bucket_metadata_missing",
error = %e,
"admin bucket meta state"
);
continue;
}
warn!("get bucket metadata failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "bucket_metadata_load_failed",
error = %e,
"admin bucket meta state"
);
continue;
}
};
}
let Some(store) = new_object_layer_fn() else {
return Err(s3_error!(InvalidRequest, "object store not init"));
return Err(s3_error!(InternalError, "object store is not initialized"));
};
let update_at = OffsetDateTime::now_utc();
@@ -494,7 +531,15 @@ impl Operation for ImportBucketMetadata {
let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>();
if file_path_split.len() < 2 {
warn!("file path is invalid: {}", file_path);
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "invalid_file_path",
file_path = %file_path,
"admin bucket meta state"
);
continue;
}
@@ -513,7 +558,16 @@ impl Operation for ImportBucketMetadata {
)
.await
{
warn!("create bucket failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "bucket_create_failed",
bucket = %bucket_name,
error = %e,
"admin bucket meta state"
);
continue;
}
@@ -527,7 +581,17 @@ impl Operation for ImportBucketMetadata {
let config: BucketPolicy = match serde_json::from_slice(&content) {
Ok(config) => config,
Err(e) => {
warn!("deserialize config failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
};
@@ -542,7 +606,17 @@ impl Operation for ImportBucketMetadata {
}
BUCKET_NOTIFICATION_CONFIG => {
if let Err(e) = deserialize::<s3s::dto::NotificationConfiguration>(&content) {
warn!("deserialize config failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
@@ -553,7 +627,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_LIFECYCLE_CONFIG => {
if let Err(e) = deserialize::<BucketLifecycleConfiguration>(&content) {
warn!("deserialize config failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
@@ -564,7 +648,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_SSECONFIG => {
if let Err(e) = deserialize::<ServerSideEncryptionConfiguration>(&content) {
warn!("deserialize config failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
@@ -575,7 +669,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_TAGGING_CONFIG => {
if let Err(e) = deserialize::<Tagging>(&content) {
warn!("deserialize config failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
@@ -586,7 +690,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_QUOTA_CONFIG_FILE => {
if let Err(e) = serde_json::from_slice::<BucketQuota>(&content) {
warn!("deserialize config failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
@@ -597,7 +711,17 @@ impl Operation for ImportBucketMetadata {
OBJECT_LOCK_CONFIG => {
if let Err(e) = deserialize::<ObjectLockConfiguration>(&content) {
warn!("deserialize config failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
@@ -608,7 +732,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_VERSIONING_CONFIG => {
if let Err(e) = deserialize::<VersioningConfiguration>(&content) {
warn!("deserialize config failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
@@ -619,7 +753,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_REPLICATION_CONFIG => {
if let Err(e) = deserialize::<ReplicationConfiguration>(&content) {
warn!("deserialize config failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
@@ -630,7 +774,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_TARGETS_FILE => {
if let Err(e) = serde_json::from_slice::<BucketTargets>(&content) {
warn!("deserialize config failed: {e}");
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
rustfs/src/admin/handlers/group.rs
+214 -38
View File
@@ -41,6 +41,10 @@ use serde::Deserialize;
use serde_urlencoded::from_bytes;
use tracing::warn;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_GROUP: &str = "group";
const EVENT_ADMIN_GROUP_STATE: &str = "admin_group_state";
pub fn register_group_management_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
Method::GET,
@@ -85,10 +89,17 @@ pub struct ListGroups {}
#[async_trait::async_trait]
impl Operation for ListGroups {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle ListGroups");
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "list_groups",
state = "requested",
"admin group state"
);
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -104,14 +115,24 @@ impl Operation for ListGroups {
)
.await?;
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let groups = iam_store.list_groups_load().await.map_err(|e| {
warn!("list groups failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "list_groups",
result = "load_failed",
error = ?e,
"admin group state"
);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
let body = serde_json::to_vec(&groups).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?;
let body = serde_json::to_vec(&groups).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -124,10 +145,17 @@ pub struct GetGroup {}
#[async_trait::async_trait]
impl Operation for GetGroup {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle GetGroup");
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "get_group",
state = "requested",
"admin group state"
);
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -146,20 +174,31 @@ impl Operation for GetGroup {
let query = {
if let Some(query) = req.uri.query() {
let input: GroupQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
GroupQuery::default()
}
};
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let g = iam_store.get_group_description(&query.group).await.map_err(|e| {
warn!("get group failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "get_group",
group = %query.group,
result = "load_failed",
error = ?e,
"admin group state"
);
iam_error_to_s3_error(e)
})?;
let body = serde_json::to_vec(&g).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?;
let body = serde_json::to_vec(&g).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -189,10 +228,17 @@ pub struct DeleteGroup {}
#[async_trait::async_trait]
impl Operation for DeleteGroup {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle DeleteGroup");
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "delete_group",
state = "requested",
"admin group state"
);
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -210,22 +256,33 @@ impl Operation for DeleteGroup {
let group = decode_delete_group_name(&params)?;
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let updated_at = iam_store.remove_users_from_group(&group, vec![]).await.map_err(|e| {
warn!("delete group failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "delete_group",
group = %group,
result = "delete_failed",
error = ?e,
"admin group state"
);
match e {
rustfs_iam::error::Error::GroupNotEmpty => {
s3_error!(InvalidRequest, "group is not empty")
}
rustfs_iam::error::Error::InvalidArgument => {
s3_error!(InvalidArgument, "{e}")
s3_error!(InvalidArgument, "invalid group request: {e}")
}
_ => {
if is_err_no_such_group(&e) {
iam_error_to_s3_error(e)
} else {
s3_error!(InternalError, "{e}")
s3_error!(InternalError, "failed to delete group: {e}")
}
}
}
@@ -248,7 +305,15 @@ impl Operation for DeleteGroup {
})
.await
{
warn!("site replication group delete hook failed, err: {err}");
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "delete_group",
result = "site_replication_hook_failed",
error = %err,
"admin group state"
);
}
let mut header = HeaderMap::new();
@@ -289,10 +354,17 @@ pub struct SetGroupStatus {}
#[async_trait::async_trait]
impl Operation for SetGroupStatus {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle SetGroupStatus");
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "set_group_status",
state = "requested",
"admin group state"
);
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -311,7 +383,7 @@ impl Operation for SetGroupStatus {
let query = {
if let Some(query) = req.uri.query() {
let input: GroupQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
GroupQuery::default()
@@ -322,16 +394,38 @@ impl Operation for SetGroupStatus {
return Err(s3_error!(InvalidArgument, "group is required"));
}
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let updated_at = if let Some(status) = query.status.as_deref() {
match status {
"enabled" => iam_store.set_group_status(&query.group, true).await.map_err(|e| {
warn!("enable group failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "set_group_status",
group = %query.group,
status = "enabled",
result = "update_failed",
error = ?e,
"admin group state"
);
iam_error_to_s3_error(e)
})?,
"disabled" => iam_store.set_group_status(&query.group, false).await.map_err(|e| {
warn!("enable group failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "set_group_status",
group = %query.group,
status = "disabled",
result = "update_failed",
error = ?e,
"admin group state"
);
iam_error_to_s3_error(e)
})?,
_ => {
@@ -363,7 +457,15 @@ impl Operation for SetGroupStatus {
})
.await
{
warn!("site replication group status hook failed, err: {err}");
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "set_group_status",
result = "site_replication_hook_failed",
error = %err,
"admin group state"
);
}
let mut header = HeaderMap::new();
@@ -377,10 +479,17 @@ pub struct UpdateGroupMembers {}
#[async_trait::async_trait]
impl Operation for UpdateGroupMembers {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle UpdateGroupMembers");
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "update_group_members",
state = "requested",
"admin group state"
);
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -400,17 +509,37 @@ impl Operation for UpdateGroupMembers {
let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "update_group_members",
result = "body_read_failed",
error = ?e,
"admin group state"
);
return Err(s3_error!(InvalidRequest, "group configuration body too large or failed to read"));
}
};
let args: GroupAddRemove = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?;
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
warn!("UpdateGroupMembers args {:?}", args);
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "update_group_members",
group = %args.group,
member_count = args.members.len(),
remove = args.is_remove,
state = "decoded",
"admin group state"
);
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
for member in args.members.iter() {
match iam_store.is_temp_user(member).await {
@@ -433,7 +562,10 @@ impl Operation for UpdateGroupMembers {
Ok(())
})
.unwrap_or_else(|| {
Err(S3Error::with_message(S3ErrorCode::InternalError, "get global cred failed".to_string()))
Err(S3Error::with_message(
S3ErrorCode::InternalError,
"failed to load global credentials".to_string(),
))
})?;
}
Err(e) => {
@@ -445,29 +577,65 @@ impl Operation for UpdateGroupMembers {
}
let updated_at = if args.is_remove {
warn!("remove group members");
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "remove_group_members",
group = %args.group,
member_count = args.members.len(),
state = "requested",
"admin group state"
);
iam_store
.remove_users_from_group(&args.group, args.members.clone())
.await
.map_err(|e| {
warn!("remove group members failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "remove_group_members",
group = %args.group,
result = "update_failed",
error = ?e,
"admin group state"
);
iam_error_to_s3_error(e)
})?
} else {
warn!("add group members");
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "add_group_members",
group = %args.group,
member_count = args.members.len(),
state = "requested",
"admin group state"
);
if let Err(err) = iam_store.get_group_description(&args.group).await
&& is_err_no_such_group(&err)
&& has_space_be(&args.group)
{
return Err(s3_error!(InvalidArgument, "not such group"));
return Err(s3_error!(InvalidArgument, "group not found"));
}
iam_store
.add_users_to_group(&args.group, args.members.clone())
.await
.map_err(|e| {
warn!("add group members failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "add_group_members",
group = %args.group,
result = "update_failed",
error = ?e,
"admin group state"
);
iam_error_to_s3_error(e)
})?
};
@@ -484,7 +652,15 @@ impl Operation for UpdateGroupMembers {
})
.await
{
warn!("site replication group membership hook failed, err: {err}");
warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "update_group_members",
result = "site_replication_hook_failed",
error = %err,
"admin group state"
);
}
let mut header = HeaderMap::new();
rustfs/src/admin/handlers/kms_dynamic.rs
+96 -36
View File
@@ -36,6 +36,7 @@ use tracing::{error, info, instrument, warn};
const KMS_CONFIG_PATH: &str = "config/kms_config.json";
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_KMS: &str = "kms";
const EVENT_ADMIN_KMS_DYNAMIC_STATE: &str = "admin_kms_dynamic_state";
fn kms_service_manager_from_context() -> std::sync::Arc<rustfs_kms::KmsServiceManager> {
resolve_kms_runtime_service_manager().unwrap_or_else(|| {
@@ -43,7 +44,8 @@ fn kms_service_manager_from_context() -> std::sync::Arc<rustfs_kms::KmsServiceMa
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_manager_fallback",
"KMS service manager fallback initialized"
result = "service_manager_fallback_initialized",
"admin kms dynamic state"
);
rustfs_kms::init_global_kms_service_manager()
})
@@ -117,7 +119,8 @@ async fn save_kms_config(config: &KmsConfig) -> Result<(), String> {
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_persisted",
storage_path = KMS_CONFIG_PATH,
"Persisted KMS configuration"
state = "config_persisted",
"admin kms dynamic state"
);
Ok(())
}
@@ -131,7 +134,8 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_load_skipped",
reason = "storage_uninitialized",
"Skipped KMS configuration load"
result = "config_load_skipped",
"admin kms dynamic state"
);
return None;
};
@@ -144,7 +148,8 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_loaded",
storage_path = KMS_CONFIG_PATH,
"Loaded persisted KMS configuration"
state = "config_loaded",
"admin kms dynamic state"
);
Some(config)
}
@@ -154,8 +159,9 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_deserialize_failed",
storage_path = KMS_CONFIG_PATH,
result = "config_deserialize_failed",
error = %e,
"Failed to deserialize KMS configuration"
"admin kms dynamic state"
);
None
}
@@ -169,7 +175,7 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
event = "kms_config_loaded",
state = "not_found",
storage_path = KMS_CONFIG_PATH,
"Persisted KMS configuration not found"
"admin kms dynamic state"
);
} else {
warn!(
@@ -177,8 +183,9 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_load_failed",
storage_path = KMS_CONFIG_PATH,
result = "config_load_failed",
error = %e,
"Failed to load KMS configuration"
"admin kms dynamic state"
);
}
None
@@ -263,8 +270,9 @@ impl Operation for ConfigureKmsHandler {
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_request_decode_failed",
operation = "configure",
result = "request_decode_failed",
error = %e,
"Failed to decode KMS admin request"
"admin kms dynamic state"
);
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}")))));
}
@@ -277,7 +285,7 @@ impl Operation for ConfigureKmsHandler {
event = "kms_service_state",
operation = "configure",
state = "requested",
"KMS service state changed"
"admin kms dynamic state"
);
let service_manager = kms_service_manager_from_context();
@@ -303,7 +311,7 @@ impl Operation for ConfigureKmsHandler {
operation = "configure",
state = "persist_failed",
error = %e,
"KMS service state changed"
"admin kms dynamic state"
);
let status = service_manager.get_status().await;
(false, error_msg, status)
@@ -316,7 +324,7 @@ impl Operation for ConfigureKmsHandler {
operation = "configure",
state = "configured",
status = ?status,
"KMS service state changed"
"admin kms dynamic state"
);
(true, "KMS configured successfully".to_string(), status)
}
@@ -330,7 +338,7 @@ impl Operation for ConfigureKmsHandler {
operation = "configure",
state = "configure_failed",
error = %e,
"KMS service state changed"
"admin kms dynamic state"
);
let status = service_manager.get_status().await;
(false, error_msg, status)
@@ -346,7 +354,15 @@ impl Operation for ConfigureKmsHandler {
let json_response = match serde_json::to_string(&response) {
Ok(json) => json,
Err(e) => {
error!("Failed to serialize response: {}", e);
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "configure",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()),
@@ -398,8 +414,9 @@ impl Operation for StartKmsHandler {
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_request_decode_failed",
operation = "start",
result = "request_decode_failed",
error = %e,
"Failed to decode KMS admin request"
"admin kms dynamic state"
);
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}")))));
}
@@ -413,7 +430,7 @@ impl Operation for StartKmsHandler {
operation = "start",
state = "requested",
force = start_request.force.unwrap_or(false),
"KMS service state changed"
"admin kms dynamic state"
);
let service_manager = kms_service_manager_from_context();
@@ -427,7 +444,7 @@ impl Operation for StartKmsHandler {
event = "kms_service_state",
operation = "start",
state = "already_running",
"KMS service state changed"
"admin kms dynamic state"
);
let response = StartKmsResponse {
success: false,
@@ -437,7 +454,15 @@ impl Operation for StartKmsHandler {
let json_response = match serde_json::to_string(&response) {
Ok(json) => json,
Err(e) => {
error!("Failed to serialize response: {}", e);
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "start",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()),
@@ -462,7 +487,7 @@ impl Operation for StartKmsHandler {
operation = "restart",
state = "running",
status = ?status,
"KMS service state changed"
"admin kms dynamic state"
);
(true, "KMS service restarted successfully".to_string(), status)
}
@@ -475,7 +500,7 @@ impl Operation for StartKmsHandler {
operation = "restart",
state = "start_failed",
error = %e,
"KMS service state changed"
"admin kms dynamic state"
);
let status = service_manager.get_status().await;
(false, error_msg, status)
@@ -490,7 +515,7 @@ impl Operation for StartKmsHandler {
operation = "restart",
state = "stop_failed",
error = %e,
"KMS service state changed"
"admin kms dynamic state"
);
let status = service_manager.get_status().await;
(false, error_msg, status)
@@ -508,7 +533,7 @@ impl Operation for StartKmsHandler {
operation = "start",
state = "running",
status = ?status,
"KMS service state changed"
"admin kms dynamic state"
);
(true, "KMS service started successfully".to_string(), status)
}
@@ -521,7 +546,7 @@ impl Operation for StartKmsHandler {
operation = "start",
state = "start_failed",
error = %e,
"KMS service state changed"
"admin kms dynamic state"
);
let status = service_manager.get_status().await;
(false, error_msg, status)
@@ -538,7 +563,15 @@ impl Operation for StartKmsHandler {
let json_response = match serde_json::to_string(&response) {
Ok(json) => json,
Err(e) => {
error!("Failed to serialize response: {}", e);
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "start",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()),
@@ -579,7 +612,7 @@ impl Operation for StopKmsHandler {
event = "kms_service_state",
operation = "stop",
state = "requested",
"KMS service state changed"
"admin kms dynamic state"
);
let service_manager = kms_service_manager_from_context();
@@ -594,7 +627,7 @@ impl Operation for StopKmsHandler {
operation = "stop",
state = "stopped",
status = ?status,
"KMS service state changed"
"admin kms dynamic state"
);
(true, "KMS service stopped successfully".to_string(), status)
}
@@ -607,7 +640,7 @@ impl Operation for StopKmsHandler {
operation = "stop",
state = "stop_failed",
error = %e,
"KMS service state changed"
"admin kms dynamic state"
);
let status = service_manager.get_status().await;
(false, error_msg, status)
@@ -623,7 +656,15 @@ impl Operation for StopKmsHandler {
let json_response = match serde_json::to_string(&response) {
Ok(json) => json,
Err(e) => {
error!("Failed to serialize response: {}", e);
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "stop",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()),
@@ -662,7 +703,8 @@ impl Operation for GetKmsStatusHandler {
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_status_requested",
"KMS status requested"
state = "status_requested",
"admin kms dynamic state"
);
let service_manager = kms_service_manager_from_context();
@@ -699,13 +741,22 @@ impl Operation for GetKmsStatusHandler {
backend_type = ?response.backend_type,
healthy = response.healthy,
has_config_summary = response.config_summary.is_some(),
"KMS status resolved"
state = "status_resolved",
"admin kms dynamic state"
);
let json_response = match serde_json::to_string(&response) {
Ok(json) => json,
Err(e) => {
error!("Failed to serialize response: {}", e);
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "status",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()),
@@ -760,8 +811,9 @@ impl Operation for ReconfigureKmsHandler {
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_request_decode_failed",
operation = "reconfigure",
result = "request_decode_failed",
error = %e,
"Failed to decode KMS admin request"
"admin kms dynamic state"
);
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}")))));
}
@@ -774,7 +826,7 @@ impl Operation for ReconfigureKmsHandler {
event = "kms_service_state",
operation = "reconfigure",
state = "requested",
"KMS service state changed"
"admin kms dynamic state"
);
let service_manager = kms_service_manager_from_context();
@@ -800,7 +852,7 @@ impl Operation for ReconfigureKmsHandler {
operation = "reconfigure",
state = "persist_failed",
error = %e,
"KMS service state changed"
"admin kms dynamic state"
);
let status = service_manager.get_status().await;
(false, error_msg, status)
@@ -813,7 +865,7 @@ impl Operation for ReconfigureKmsHandler {
operation = "reconfigure",
state = "reconfigured",
status = ?status,
"KMS service state changed"
"admin kms dynamic state"
);
(true, "KMS reconfigured and restarted successfully".to_string(), status)
}
@@ -827,7 +879,7 @@ impl Operation for ReconfigureKmsHandler {
operation = "reconfigure",
state = "reconfigure_failed",
error = %e,
"KMS service state changed"
"admin kms dynamic state"
);
let status = service_manager.get_status().await;
(false, error_msg, status)
@@ -843,7 +895,15 @@ impl Operation for ReconfigureKmsHandler {
let json_response = match serde_json::to_string(&response) {
Ok(json) => json,
Err(e) => {
error!("Failed to serialize response: {}", e);
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "reconfigure",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()),
rustfs/src/admin/handlers/kms_keys.rs
+157 -37
View File
@@ -33,6 +33,10 @@ use std::collections::HashMap;
use tracing::{error, info};
use urlencoding;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_KMS_KEYS: &str = "kms_keys";
const EVENT_ADMIN_KMS_KEYS_STATE: &str = "admin_kms_keys_state";
#[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CreateKmsKeyRequest {
@@ -227,7 +231,7 @@ impl Operation for CreateKeyHandler {
};
let Some(service) = kms_encryption_service_from_context().await else {
return Err(s3_error!(InternalError, "KMS service not initialized"));
return Err(s3_error!(InternalError, "kms service is not initialized"));
};
// Extract key name from tags if provided
@@ -259,7 +263,15 @@ impl Operation for CreateKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
}
Err(e) => {
error!("Failed to create KMS key: {}", e);
error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "create_key",
result = "failed",
error = %e,
"admin kms keys state"
);
Err(s3_error!(InternalError, "failed to create key: {}", e))
}
}
@@ -290,11 +302,11 @@ impl Operation for DescribeKeyHandler {
.await?;
let Some(key_id) = extract_key_id(&req.uri) else {
return Err(s3_error!(InvalidRequest, "missing keyId parameter"));
return Err(s3_error!(InvalidRequest, "missing required parameter: 'keyId'"));
};
let Some(service) = kms_encryption_service_from_context().await else {
return Err(s3_error!(InternalError, "KMS service not initialized"));
return Err(s3_error!(InternalError, "kms service is not initialized"));
};
let request = DescribeKeyRequest { key_id: key_id.clone() };
@@ -314,7 +326,16 @@ impl Operation for DescribeKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
}
Err(e) => {
error!("Failed to describe KMS key {}: {}", key_id, e);
error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "describe_key",
key_id = %key_id,
result = "failed",
error = %e,
"admin kms keys state"
);
Err(s3_error!(InternalError, "failed to describe key: {}", e))
}
}
@@ -451,7 +472,7 @@ impl Operation for ListKeysHandler {
let marker = query_params.get("marker").cloned();
let Some(service) = kms_encryption_service_from_context().await else {
return Err(s3_error!(InternalError, "KMS service not initialized"));
return Err(s3_error!(InternalError, "kms service is not initialized"));
};
let request = ListKeysRequest {
@@ -478,7 +499,15 @@ impl Operation for ListKeysHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
}
Err(e) => {
error!("Failed to list KMS keys: {}", e);
error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "list_keys",
result = "failed",
error = %e,
"admin kms keys state"
);
Err(s3_error!(InternalError, "failed to list keys: {}", e))
}
}
@@ -544,7 +573,15 @@ impl Operation for GenerateDataKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
}
Err(e) => {
error!("Failed to generate data key: {}", e);
error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "generate_data_key",
result = "failed",
error = %e,
"admin kms keys state"
);
Err(s3_error!(InternalError, "failed to generate data key: {}", e))
}
}
@@ -593,7 +630,7 @@ impl Operation for CreateKmsKeyHandler {
let Some(service_manager) = kms_service_manager_from_context() else {
let response = CreateKmsKeyResponse {
success: false,
message: "KMS service manager not initialized".to_string(),
message: "kms service manager is not initialized".to_string(),
key_id: "".to_string(),
key_metadata: None,
};
@@ -607,7 +644,7 @@ impl Operation for CreateKmsKeyHandler {
let Some(manager) = service_manager.get_manager().await else {
let response = CreateKmsKeyResponse {
success: false,
message: "KMS service not running".to_string(),
message: "kms service is not running".to_string(),
key_id: "".to_string(),
key_metadata: None,
};
@@ -633,10 +670,18 @@ impl Operation for CreateKmsKeyHandler {
match manager.create_key(kms_request).await {
Ok(kms_response) => {
info!("Created KMS key: {}", kms_response.key_id);
info!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "create_key",
key_id = %kms_response.key_id,
state = "completed",
"admin kms keys state"
);
let response = CreateKmsKeyResponse {
success: true,
message: "Key created successfully".to_string(),
message: "key created successfully".to_string(),
key_id: kms_response.key_id,
key_metadata: Some(kms_response.key_metadata),
};
@@ -650,10 +695,18 @@ impl Operation for CreateKmsKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
}
Err(e) => {
error!("Failed to create KMS key: {}", e);
error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "create_key",
result = "failed",
error = %e,
"admin kms keys state"
);
let response = CreateKmsKeyResponse {
success: false,
message: format!("Failed to create key: {e}"),
message: format!("failed to create key: {e}"),
key_id: "".to_string(),
key_metadata: None,
};
@@ -720,7 +773,7 @@ impl Operation for DeleteKmsKeyHandler {
let Some(key_id) = query_params.get("keyId") else {
let response = DeleteKmsKeyResponse {
success: false,
message: "missing keyId parameter".to_string(),
message: "missing required parameter: 'keyId'".to_string(),
key_id: "".to_string(),
deletion_date: None,
};
@@ -747,7 +800,7 @@ impl Operation for DeleteKmsKeyHandler {
let Some(service_manager) = kms_service_manager_from_context() else {
let response = DeleteKmsKeyResponse {
success: false,
message: "KMS service manager not initialized".to_string(),
message: "kms service manager is not initialized".to_string(),
key_id: request.key_id,
deletion_date: None,
};
@@ -761,7 +814,7 @@ impl Operation for DeleteKmsKeyHandler {
let Some(manager) = service_manager.get_manager().await else {
let response = DeleteKmsKeyResponse {
success: false,
message: "KMS service not running".to_string(),
message: "kms service is not running".to_string(),
key_id: request.key_id,
deletion_date: None,
};
@@ -780,10 +833,18 @@ impl Operation for DeleteKmsKeyHandler {
match manager.delete_key(kms_request).await {
Ok(kms_response) => {
info!("Successfully deleted KMS key: {}", kms_response.key_id);
info!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "delete_key",
key_id = %kms_response.key_id,
state = "completed",
"admin kms keys state"
);
let response = DeleteKmsKeyResponse {
success: true,
message: "Key deleted successfully".to_string(),
message: "key deleted successfully".to_string(),
key_id: kms_response.key_id,
deletion_date: kms_response.deletion_date,
};
@@ -797,7 +858,16 @@ impl Operation for DeleteKmsKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
}
Err(e) => {
error!("Failed to delete KMS key {}: {}", request.key_id, e);
error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "delete_key",
key_id = %request.key_id,
result = "failed",
error = %e,
"admin kms keys state"
);
let status = match &e {
KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND,
KmsError::InvalidOperation { .. } | KmsError::ValidationError { .. } => StatusCode::BAD_REQUEST,
@@ -870,7 +940,7 @@ impl Operation for CancelKmsKeyDeletionHandler {
let Some(key_id) = query_params.get("keyId") else {
let response = CancelKmsKeyDeletionResponse {
success: false,
message: "missing keyId parameter".to_string(),
message: "missing required parameter: 'keyId'".to_string(),
key_id: "".to_string(),
key_metadata: None,
};
@@ -888,7 +958,7 @@ impl Operation for CancelKmsKeyDeletionHandler {
let Some(service_manager) = kms_service_manager_from_context() else {
let response = CancelKmsKeyDeletionResponse {
success: false,
message: "KMS service manager not initialized".to_string(),
message: "kms service manager is not initialized".to_string(),
key_id: request.key_id,
key_metadata: None,
};
@@ -902,7 +972,7 @@ impl Operation for CancelKmsKeyDeletionHandler {
let Some(manager) = service_manager.get_manager().await else {
let response = CancelKmsKeyDeletionResponse {
success: false,
message: "KMS service not running".to_string(),
message: "kms service is not running".to_string(),
key_id: request.key_id,
key_metadata: None,
};
@@ -919,10 +989,18 @@ impl Operation for CancelKmsKeyDeletionHandler {
match manager.cancel_key_deletion(kms_request).await {
Ok(kms_response) => {
info!("Cancelled deletion for KMS key: {}", kms_response.key_id);
info!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "cancel_delete_key",
key_id = %kms_response.key_id,
state = "completed",
"admin kms keys state"
);
let response = CancelKmsKeyDeletionResponse {
success: true,
message: "Key deletion cancelled successfully".to_string(),
message: "key deletion cancelled successfully".to_string(),
key_id: kms_response.key_id,
key_metadata: Some(kms_response.key_metadata),
};
@@ -936,7 +1014,16 @@ impl Operation for CancelKmsKeyDeletionHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
}
Err(e) => {
error!("Failed to cancel deletion for KMS key {}: {}", request.key_id, e);
error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "cancel_delete_key",
key_id = %request.key_id,
result = "failed",
error = %e,
"admin kms keys state"
);
let response = CancelKmsKeyDeletionResponse {
success: false,
message: format!("Failed to cancel key deletion: {e}"),
@@ -995,7 +1082,7 @@ impl Operation for ListKmsKeysHandler {
let Some(service_manager) = kms_service_manager_from_context() else {
let response = ListKmsKeysResponse {
success: false,
message: "KMS service manager not initialized".to_string(),
message: "kms service manager is not initialized".to_string(),
keys: vec![],
truncated: false,
next_marker: None,
@@ -1010,7 +1097,7 @@ impl Operation for ListKmsKeysHandler {
let Some(manager) = service_manager.get_manager().await else {
let response = ListKmsKeysResponse {
success: false,
message: "KMS service not running".to_string(),
message: "kms service is not running".to_string(),
keys: vec![],
truncated: false,
next_marker: None,
@@ -1031,10 +1118,18 @@ impl Operation for ListKmsKeysHandler {
match manager.list_keys(kms_request).await {
Ok(kms_response) => {
info!("Listed {} KMS keys", kms_response.keys.len());
info!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "list_keys",
state = "completed",
key_count = kms_response.keys.len(),
"admin kms keys state"
);
let response = ListKmsKeysResponse {
success: true,
message: "Keys listed successfully".to_string(),
message: "keys listed successfully".to_string(),
keys: kms_response.keys,
truncated: kms_response.truncated,
next_marker: kms_response.next_marker,
@@ -1049,10 +1144,18 @@ impl Operation for ListKmsKeysHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
}
Err(e) => {
error!("Failed to list KMS keys: {}", e);
error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "list_keys",
result = "failed",
error = %e,
"admin kms keys state"
);
let response = ListKmsKeysResponse {
success: false,
message: format!("Failed to list keys: {e}"),
message: format!("failed to list keys: {e}"),
keys: vec![],
truncated: false,
next_marker: None,
@@ -1103,7 +1206,7 @@ impl Operation for DescribeKmsKeyHandler {
let Some(key_id) = params.get("key_id") else {
let response = DescribeKmsKeyResponse {
success: false,
message: "missing keyId parameter".to_string(),
message: "missing required parameter: 'keyId'".to_string(),
key_metadata: None,
};
let data =
@@ -1116,7 +1219,7 @@ impl Operation for DescribeKmsKeyHandler {
let Some(service_manager) = kms_service_manager_from_context() else {
let response = DescribeKmsKeyResponse {
success: false,
message: "KMS service manager not initialized".to_string(),
message: "kms service manager is not initialized".to_string(),
key_metadata: None,
};
let data =
@@ -1129,7 +1232,7 @@ impl Operation for DescribeKmsKeyHandler {
let Some(manager) = service_manager.get_manager().await else {
let response = DescribeKmsKeyResponse {
success: false,
message: "KMS service not running".to_string(),
message: "kms service is not running".to_string(),
key_metadata: None,
};
let data =
@@ -1145,7 +1248,15 @@ impl Operation for DescribeKmsKeyHandler {
match manager.describe_key(kms_request).await {
Ok(kms_response) => {
info!("Described KMS key: {}", key_id);
info!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "describe_key",
key_id = %key_id,
state = "completed",
"admin kms keys state"
);
let response = DescribeKmsKeyResponse {
success: true,
message: "Key described successfully".to_string(),
@@ -1161,7 +1272,16 @@ impl Operation for DescribeKmsKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
}
Err(e) => {
error!("Failed to describe KMS key {}: {}", key_id, e);
error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "describe_key",
key_id = %key_id,
result = "failed",
error = %e,
"admin kms keys state"
);
let status = match &e {
KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND,
KmsError::InvalidOperation { .. } => StatusCode::BAD_REQUEST,
rustfs/src/admin/handlers/oidc.rs
+51 -8
View File
@@ -39,6 +39,10 @@ use time::OffsetDateTime;
use tracing::{error, info, warn};
use url::Url;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_OIDC: &str = "oidc";
const EVENT_ADMIN_OIDC_STATE: &str = "admin_oidc_state";
const OIDC_PUBLIC_PROVIDERS_SUFFIX: &str = "/v3/oidc/providers";
const OIDC_AUTHORIZE_SUFFIX: &str = "/v3/oidc/authorize/";
const OIDC_CALLBACK_SUFFIX: &str = "/v3/oidc/callback/";
@@ -448,7 +452,14 @@ impl Operation for OidcAuthorizeHandler {
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("authorize failed: {e}")))?;
info!("OIDC authorize redirect for provider '{}' to IdP", provider_id);
info!(
event = EVENT_ADMIN_OIDC_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
provider_id = %provider_id,
state = "authorize_redirect",
"admin oidc state"
);
// Return 302 redirect
let mut resp = S3Response::new((StatusCode::FOUND, Body::empty()));
@@ -486,7 +497,15 @@ impl Operation for OidcCallbackHandler {
// Check for error response from IdP
if let Some(error) = extract_query_param(&req.uri, "error") {
let desc = extract_query_param(&req.uri, "error_description").unwrap_or_default();
warn!("OIDC callback received error from IdP: {} - {}", error, desc);
warn!(
event = EVENT_ADMIN_OIDC_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
result = "idp_callback_error",
error_code = %error,
error_description = %desc,
"admin oidc state"
);
return Err(S3Error::with_message(
S3ErrorCode::AccessDenied,
format!("OIDC authentication failed: {error} - {desc}"),
@@ -500,21 +519,38 @@ impl Operation for OidcCallbackHandler {
// Exchange authorization code for tokens and extract claims
let (claims, actual_provider_id, session, id_token) =
oidc_sys.exchange_code(&state, &code, &redirect_uri).await.map_err(|e| {
error!("OIDC code exchange failed: {}", e);
error!(
event = EVENT_ADMIN_OIDC_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
result = "code_exchange_failed",
error = %e,
"admin oidc state"
);
S3Error::with_message(S3ErrorCode::AccessDenied, format!("code exchange failed: {e}"))
})?;
info!(
"OIDC login successful: username='{}', email='{}', sub='{}' (provider: {})",
claims.username, claims.email, claims.sub, actual_provider_id
event = EVENT_ADMIN_OIDC_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
provider_id = %actual_provider_id,
state = "authentication_succeeded",
"admin oidc state"
);
// Map claims to policies and groups
let (policies, groups) = oidc_sys.map_claims_to_policies(&actual_provider_id, &claims);
info!(
"OIDC claim mapping: user='{}', policies={:?}, groups={:?}",
claims.username, policies, groups
event = EVENT_ADMIN_OIDC_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
provider_id = %actual_provider_id,
policy_count = policies.len(),
group_count = groups.len(),
state = "claims_mapped",
"admin oidc state"
);
// Generate STS credentials using the shared helper.
@@ -568,7 +604,14 @@ impl Operation for OidcLogoutHandler {
Ok(Some(url)) => url,
Ok(None) => fallback_location.clone(),
Err(err) => {
warn!("OIDC logout fallback triggered: {}", err);
warn!(
event = EVENT_ADMIN_OIDC_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
result = "logout_fallback_triggered",
error = %err,
"admin oidc state"
);
fallback_location.clone()
}
},
rustfs/src/admin/handlers/policies.rs
+116 -56
View File
@@ -52,6 +52,7 @@ use url::form_urlencoded;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_POLICY: &str = "policy";
const EVENT_ADMIN_POLICY_STATE: &str = "admin_policy_state";
pub fn register_iam_policy_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
@@ -117,7 +118,7 @@ pub struct ListCannedPolicies {}
impl Operation for ListCannedPolicies {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -136,23 +137,26 @@ impl Operation for ListCannedPolicies {
let query = {
if let Some(query) = req.uri.query() {
let input: BucketQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
BucketQuery::default()
}
};
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let policies = iam_store.list_polices(&query.bucket).await.map_err(|e| {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_list_failed",
event = EVENT_ADMIN_POLICY_STATE,
bucket = %query.bucket,
result = "list_failed",
error = ?e,
"Failed to list canned policies"
"admin policy state"
);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
@@ -162,7 +166,7 @@ impl Operation for ListCannedPolicies {
.filter(|(_, v)| serde_json::to_string(v).is_ok())
.collect();
let body = serde_json::to_vec(&kvs).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?;
let body = serde_json::to_vec(&kvs).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -181,7 +185,7 @@ pub struct AddCannedPolicy {}
impl Operation for AddCannedPolicy {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -200,7 +204,7 @@ impl Operation for AddCannedPolicy {
let query = {
if let Some(query) = req.uri.query() {
let input: PolicyNameQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
PolicyNameQuery::default()
@@ -208,11 +212,11 @@ impl Operation for AddCannedPolicy {
};
if query.name.is_empty() {
return Err(s3_error!(InvalidArgument, "policy name is empty"));
return Err(s3_error!(InvalidArgument, "policy name is required"));
}
if has_space_be(&query.name) {
return Err(s3_error!(InvalidArgument, "policy name has space"));
return Err(s3_error!(InvalidArgument, "policy name contains spaces"));
}
let mut input = req.input;
@@ -222,10 +226,11 @@ impl Operation for AddCannedPolicy {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_body_read_failed",
event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name,
result = "body_read_failed",
error = ?e,
"Failed to read policy request body"
"admin policy state"
);
return Err(s3_error!(InvalidRequest, "policy configuration body too large or failed to read"));
}
@@ -235,27 +240,31 @@ impl Operation for AddCannedPolicy {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_parse_failed",
event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name,
result = "parse_failed",
error = ?e,
"Failed to parse canned policy"
"admin policy state"
);
S3Error::with_message(S3ErrorCode::InvalidRequest, e.to_string())
})?;
if policy.version.is_empty() {
return Err(s3_error!(InvalidArgument, "policy version is empty"));
return Err(s3_error!(InvalidArgument, "policy version is required"));
}
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let updated_at = iam_store.set_policy(&query.name, policy.clone()).await.map_err(|e| {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_set_failed",
event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name,
result = "persist_failed",
error = ?e,
"Failed to persist canned policy"
"admin policy state"
);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
@@ -272,7 +281,16 @@ impl Operation for AddCannedPolicy {
})
.await
{
warn!(policy = %query.name, error = ?err, "site replication policy add hook failed");
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name,
action = "add_policy",
result = "site_replication_hook_failed",
error = ?err,
"admin policy state"
);
}
let mut header = HeaderMap::new();
@@ -287,7 +305,7 @@ pub struct InfoCannedPolicy {}
impl Operation for InfoCannedPolicy {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -306,7 +324,7 @@ impl Operation for InfoCannedPolicy {
let query = {
if let Some(query) = req.uri.query() {
let input: PolicyNameQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
PolicyNameQuery::default()
@@ -314,7 +332,7 @@ impl Operation for InfoCannedPolicy {
};
if query.name.is_empty() {
return Err(s3_error!(InvalidArgument, "policy name is empty"));
return Err(s3_error!(InvalidArgument, "policy name is required"));
}
let policies = MappedPolicy::new(&query.name).to_slice();
@@ -322,21 +340,24 @@ impl Operation for InfoCannedPolicy {
return Err(s3_error!(InvalidArgument, "too many policies"));
}
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let pd = iam_store.info_policy(&query.name).await.map_err(|e| {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_info_failed",
event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name,
result = "info_failed",
error = ?e,
"Failed to load canned policy"
"admin policy state"
);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
let body = serde_json::to_vec(&pd).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?;
let body = serde_json::to_vec(&pd).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -350,7 +371,7 @@ pub struct RemoveCannedPolicy {}
impl Operation for RemoveCannedPolicy {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -369,7 +390,7 @@ impl Operation for RemoveCannedPolicy {
let query = {
if let Some(query) = req.uri.query() {
let input: PolicyNameQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
PolicyNameQuery::default()
@@ -377,19 +398,22 @@ impl Operation for RemoveCannedPolicy {
};
if query.name.is_empty() {
return Err(s3_error!(InvalidArgument, "policy name is empty"));
return Err(s3_error!(InvalidArgument, "policy name is required"));
}
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
iam_store.delete_policy(&query.name, true).await.map_err(|e| {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_delete_failed",
event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name,
result = "delete_failed",
error = ?e,
"Failed to delete canned policy"
"admin policy state"
);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
@@ -403,7 +427,16 @@ impl Operation for RemoveCannedPolicy {
})
.await
{
warn!(policy = %query.name, error = ?err, "site replication policy delete hook failed");
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name,
action = "delete_policy",
result = "site_replication_hook_failed",
error = ?err,
"admin policy state"
);
}
let mut header = HeaderMap::new();
@@ -429,7 +462,7 @@ pub struct SetPolicyForUserOrGroup {}
impl Operation for SetPolicyForUserOrGroup {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -448,7 +481,7 @@ impl Operation for SetPolicyForUserOrGroup {
let query = {
if let Some(query) = req.uri.query() {
let input: SetPolicyForUserOrGroupQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
SetPolicyForUserOrGroupQuery::default()
@@ -456,10 +489,12 @@ impl Operation for SetPolicyForUserOrGroup {
};
if query.user_or_group.is_empty() {
return Err(s3_error!(InvalidArgument, "user or group is empty"));
return Err(s3_error!(InvalidArgument, "user or group is required"));
}
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
if !query.is_group {
match iam_store.is_temp_user(&query.user_or_group).await {
@@ -473,11 +508,12 @@ impl Operation for SetPolicyForUserOrGroup {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_target_validation_failed",
event = EVENT_ADMIN_POLICY_STATE,
target = %query.user_or_group,
check = "is_temp_user",
result = "target_validation_failed",
error = ?err,
"Failed to validate policy target"
"admin policy state"
);
return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string()));
}
@@ -485,28 +521,29 @@ impl Operation for SetPolicyForUserOrGroup {
};
let Some(sys_cred) = get_global_action_cred() else {
return Err(s3_error!(InternalError, "get global action cred failed"));
return Err(s3_error!(InternalError, "failed to load global credentials"));
};
if query.user_or_group == sys_cred.access_key {
return Err(s3_error!(InvalidArgument, "can't set policy for system user"));
return Err(s3_error!(InvalidArgument, "cannot set a policy for the system user"));
}
}
if !query.is_group {
if iam_store.get_user(&query.user_or_group).await.is_none() {
return Err(s3_error!(InvalidArgument, "user not exist"));
return Err(s3_error!(InvalidArgument, "user not found"));
}
} else {
iam_store.get_group_description(&query.user_or_group).await.map_err(|e| {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_target_validation_failed",
event = EVENT_ADMIN_POLICY_STATE,
target = %query.user_or_group,
check = "group_description",
result = "target_validation_failed",
error = ?e,
"Failed to validate policy target"
"admin policy state"
);
iam_error_to_s3_error(e)
})?;
@@ -519,12 +556,13 @@ impl Operation for SetPolicyForUserOrGroup {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_mapping_set_failed",
event = EVENT_ADMIN_POLICY_STATE,
target = %query.user_or_group,
policy = %query.policy_name,
is_group = query.is_group,
result = "mapping_set_failed",
error = ?e,
"Failed to update policy mapping"
"admin policy state"
);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
@@ -546,7 +584,16 @@ impl Operation for SetPolicyForUserOrGroup {
})
.await
{
warn!(target = %query.user_or_group, error = ?err, "site replication policy mapping hook failed");
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = EVENT_ADMIN_POLICY_STATE,
target = %query.user_or_group,
action = "set_policy_mapping",
result = "site_replication_hook_failed",
error = ?err,
"admin policy state"
);
}
let mut header = HeaderMap::new();
@@ -923,11 +970,12 @@ async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_target_validation_failed",
event = EVENT_ADMIN_POLICY_STATE,
target = %assoc_req.user,
check = "is_temp_user",
result = "target_validation_failed",
error = ?err,
"Failed to validate policy association target"
"admin policy state"
);
return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string()));
}
@@ -950,10 +998,11 @@ async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_user_info_failed",
event = EVENT_ADMIN_POLICY_STATE,
target = %assoc_req.user,
result = "user_info_failed",
error = ?e,
"Failed to load policy association target"
"admin policy state"
);
iam_error_to_s3_error(e)
})?;
@@ -964,10 +1013,11 @@ async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_group_lookup_failed",
event = EVENT_ADMIN_POLICY_STATE,
group = %assoc_req.group,
result = "group_lookup_failed",
error = ?e,
"Failed to load policy association target"
"admin policy state"
);
iam_error_to_s3_error(e)
})?;
@@ -988,12 +1038,13 @@ async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_mapping_set_failed",
event = EVENT_ADMIN_POLICY_STATE,
target = %target_name,
policy_count = updated_policies.len(),
is_group,
result = "mapping_set_failed",
error = ?e,
"Failed to update policy association"
"admin policy state"
);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
@@ -1015,7 +1066,16 @@ async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool
})
.await
{
warn!(target = %target_name, error = ?err, "site replication policy association hook failed");
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = EVENT_ADMIN_POLICY_STATE,
target = %target_name,
action = if is_attach { "attach_policy" } else { "detach_policy" },
result = "site_replication_hook_failed",
error = ?err,
"admin policy state"
);
}
let policies_attached = if is_attach { changed_policies.clone() } else { Vec::new() };
rustfs/src/admin/handlers/quota.rs
+92 -30
View File
@@ -33,6 +33,10 @@ use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use url::form_urlencoded;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_QUOTA: &str = "quota";
const EVENT_ADMIN_QUOTA_STATE: &str = "admin_quota_state";
#[derive(Debug, Deserialize)]
pub struct SetBucketQuotaRequest {
pub quota: Option<u64>,
@@ -231,7 +235,14 @@ pub fn register_quota_route(r: &mut S3Router<AdminOperation>) -> std::io::Result
impl Operation for SetBucketQuotaHandler {
#[tracing::instrument(skip_all)]
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle SetBucketQuota");
warn!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "set_bucket_quota",
state = "requested",
"admin quota state"
);
let Some(ref cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -276,7 +287,7 @@ impl Operation for SetBucketQuotaHandler {
quota_checker
.set_quota_config(&bucket, quota.clone())
.await
.map_err(|e| s3_error!(InternalError, "Failed to set quota: {}", e))?;
.map_err(|e| s3_error!(InternalError, "failed to set quota: {}", e))?;
// Get real-time usage from data usage system
let current_usage = current_usage_from_context(&bucket).await;
@@ -291,7 +302,7 @@ impl Operation for SetBucketQuotaHandler {
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
};
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
};
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
@@ -302,7 +313,14 @@ impl Operation for SetBucketQuotaHandler {
impl Operation for GetBucketQuotaHandler {
#[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle GetBucketQuota");
warn!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "get_bucket_quota",
state = "requested",
"admin quota state"
);
let Some(ref cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -327,21 +345,21 @@ impl Operation for GetBucketQuotaHandler {
)
.await?;
let metadata_sys_lock =
bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?;
let metadata_sys_lock = bucket_metadata_from_context()
.ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?;
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
let (quota, current_usage) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e {
QuotaError::ConfigNotFound { .. } => {
s3_error!(NoSuchBucket, "Bucket not found: {}", bucket)
s3_error!(NoSuchBucket, "bucket not found: {}", bucket)
}
_ => s3_error!(InternalError, "Failed to get quota: {}", e),
_ => s3_error!(InternalError, "failed to get quota: {}", e),
})?;
let json = if is_compat_get_bucket_quota_path(req.uri.path()) {
serde_json::to_string(&compat_bucket_quota_response(&quota))
.map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?
.map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
} else {
let response = BucketQuotaResponse {
bucket,
@@ -350,7 +368,7 @@ impl Operation for GetBucketQuotaHandler {
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
};
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
};
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
@@ -361,7 +379,14 @@ impl Operation for GetBucketQuotaHandler {
impl Operation for ClearBucketQuotaHandler {
#[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle ClearBucketQuota");
warn!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "clear_bucket_quota",
state = "requested",
"admin quota state"
);
let Some(ref cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -385,10 +410,18 @@ impl Operation for ClearBucketQuotaHandler {
return Err(s3_error!(InvalidRequest, "bucket name is required"));
}
info!("Clearing quota for bucket: {}", bucket);
info!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "clear_bucket_quota",
bucket = %bucket,
state = "clearing",
"admin quota state"
);
let metadata_sys_lock =
bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?;
let metadata_sys_lock = bucket_metadata_from_context()
.ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?;
let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
@@ -397,9 +430,17 @@ impl Operation for ClearBucketQuotaHandler {
quota_checker
.set_quota_config(&bucket, quota.clone())
.await
.map_err(|e| s3_error!(InternalError, "Failed to clear quota: {}", e))?;
.map_err(|e| s3_error!(InternalError, "failed to clear quota: {}", e))?;
info!("Successfully cleared quota for bucket: {}", bucket);
info!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "clear_bucket_quota",
bucket = %bucket,
state = "cleared",
"admin quota state"
);
// Get real-time usage from data usage system
let current_usage = current_usage_from_context(&bucket).await;
@@ -412,7 +453,7 @@ impl Operation for ClearBucketQuotaHandler {
};
let json =
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?;
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
}
@@ -422,7 +463,14 @@ impl Operation for ClearBucketQuotaHandler {
impl Operation for GetBucketQuotaStatsHandler {
#[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle GetBucketQuotaStats");
warn!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "get_bucket_quota_stats",
state = "requested",
"admin quota state"
);
let Some(ref cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -447,16 +495,16 @@ impl Operation for GetBucketQuotaStatsHandler {
)
.await?;
let metadata_sys_lock =
bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?;
let metadata_sys_lock = bucket_metadata_from_context()
.ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?;
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
let (quota, current_usage_opt) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e {
QuotaError::ConfigNotFound { .. } => {
s3_error!(NoSuchBucket, "Bucket not found: {}", bucket)
s3_error!(NoSuchBucket, "bucket not found: {}", bucket)
}
_ => s3_error!(InternalError, "Failed to get quota stats: {}", e),
_ => s3_error!(InternalError, "failed to get quota stats: {}", e),
})?;
let current_usage = current_usage_opt.unwrap_or(0);
@@ -479,7 +527,7 @@ impl Operation for GetBucketQuotaStatsHandler {
};
let json =
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?;
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
}
@@ -489,7 +537,14 @@ impl Operation for GetBucketQuotaStatsHandler {
impl Operation for CheckBucketQuotaHandler {
#[tracing::instrument(skip_all)]
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle CheckBucketQuota");
warn!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "check_bucket_quota",
state = "requested",
"admin quota state"
);
let Some(ref cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -527,12 +582,19 @@ impl Operation for CheckBucketQuotaHandler {
};
debug!(
"Checking quota for bucket: {}, operation: {}, size: {}",
bucket, request.operation_type, request.operation_size
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "check_bucket_quota",
bucket = %bucket,
operation = %request.operation_type,
operation_size = request.operation_size,
state = "checking",
"admin quota state"
);
let metadata_sys_lock =
bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?;
let metadata_sys_lock = bucket_metadata_from_context()
.ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?;
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
@@ -547,7 +609,7 @@ impl Operation for CheckBucketQuotaHandler {
let result = quota_checker
.check_quota_with_usage_reporting(&bucket, operation, request.operation_size, true)
.await
.map_err(|e| s3_error!(InternalError, "Failed to check quota: {}", e))?;
.map_err(|e| s3_error!(InternalError, "failed to check quota: {}", e))?;
let response = CheckQuotaResponse {
bucket,
@@ -560,7 +622,7 @@ impl Operation for CheckBucketQuotaHandler {
};
let json =
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?;
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
}
rustfs/src/admin/handlers/rebalance.rs
+117 -36
View File
@@ -41,7 +41,11 @@ use s3s::{
use serde::{Deserialize, Serialize};
use std::time::Duration;
use time::OffsetDateTime;
use tracing::warn;
use tracing::info;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_REBALANCE: &str = "rebalance";
const EVENT_ADMIN_REBALANCE_STATE: &str = "admin_rebalance_state";
pub fn register_rebalance_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
@@ -219,10 +223,17 @@ pub struct RebalanceStart {}
impl Operation for RebalanceStart {
#[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle RebalanceStart");
info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "start",
state = "requested",
"admin rebalance state"
);
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "Failed to start rebalance: missing credentials"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -239,7 +250,7 @@ impl Operation for RebalanceStart {
.await?;
let Some(store) = new_object_layer_fn() else {
return Err(s3_error!(InternalError, "Failed to start rebalance: object layer not initialized"));
return Err(s3_error!(InternalError, "object layer is not initialized"));
};
if store.pools.len() == 1 {
@@ -247,47 +258,74 @@ impl Operation for RebalanceStart {
}
if store.is_decommission_running().await {
return Err(s3_error!(
InvalidRequest,
"Rebalance cannot be started, decommission is already in progress"
));
return Err(s3_error!(InvalidRequest, "cannot start rebalance while decommission is in progress"));
}
if store.is_rebalance_conflicting_with_decommission().await {
return Err(s3_error!(OperationAborted, "Rebalance already in progress"));
return Err(s3_error!(OperationAborted, "rebalance is already in progress"));
}
let bucket_infos = store
.list_bucket(&BucketOptions::default())
.await
.map_err(|e| s3_error!(InternalError, "Failed to list buckets for rebalance: {}", e))?;
.map_err(|e| s3_error!(InternalError, "failed to list buckets for rebalance: {}", e))?;
let buckets: Vec<String> = bucket_infos.into_iter().map(|bucket| bucket.name).collect();
let id = match store.init_rebalance_meta(buckets).await {
Ok(id) => id,
Err(e) => {
return Err(s3_error!(InternalError, "Failed to initialize rebalance metadata: {}", e));
return Err(s3_error!(InternalError, "failed to initialize rebalance metadata: {}", e));
}
};
store
.start_rebalance()
.await
.map_err(|e| s3_error!(InternalError, "Failed to start rebalance: {}", e))?;
.map_err(|e| s3_error!(InternalError, "failed to start rebalance: {}", e))?;
warn!("Rebalance started with id: {}", id);
info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "start",
state = "started",
rebalance_id = %id,
"admin rebalance state"
);
if let Some(notification_sys) = get_global_notification_sys() {
warn!("RebalanceStart Loading rebalance meta start");
info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "start",
state = "propagation_started",
"admin rebalance state"
);
if let Err(err) = notification_sys.load_rebalance_meta(true).await {
warn!("rebalance start propagation failed after local state update: {err}");
info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "start",
result = "propagation_failed",
error = %err,
"admin rebalance state"
);
}
warn!("RebalanceStart Loading rebalance meta done");
info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "start",
state = "propagation_completed",
"admin rebalance state"
);
}
let resp = RebalanceResp { id };
let data = serde_json::to_string(&resp)
.map_err(|e| s3_error!(InternalError, "Failed to serialize rebalance start response: {}", e))?;
.map_err(|e| s3_error!(InternalError, "failed to serialize rebalance start response: {}", e))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
@@ -303,10 +341,17 @@ pub struct RebalanceStatus {}
impl Operation for RebalanceStatus {
#[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle RebalanceStatus");
info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "status",
state = "requested",
"admin rebalance state"
);
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "Failed to load rebalance status: missing credentials"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -323,26 +368,26 @@ impl Operation for RebalanceStatus {
.await?;
let Some(store) = new_object_layer_fn() else {
return Err(s3_error!(InternalError, "Failed to load rebalance status: object layer not initialized"));
return Err(s3_error!(InternalError, "object layer is not initialized"));
};
if store.pools.is_empty() {
return Err(s3_error!(InternalError, "Failed to load rebalance status: no storage pools available"));
return Err(s3_error!(InternalError, "no storage pools are available"));
}
let first_pool = store
.pools
.first()
.cloned()
.ok_or_else(|| s3_error!(InternalError, "Failed to load rebalance status: no storage pools available"))?;
.ok_or_else(|| s3_error!(InternalError, "no storage pools are available"))?;
let mut meta = RebalanceMeta::new();
if let Err(err) = meta.load(first_pool).await {
if err == StorageError::ConfigNotFound {
return Err(s3_error!(NoSuchResource, "Pool rebalance is not started"));
return Err(s3_error!(NoSuchResource, "pool rebalance is not started"));
}
return Err(s3_error!(InternalError, "Failed to load rebalance metadata from pool 0: {}", err));
return Err(s3_error!(InternalError, "failed to load rebalance metadata from pool 0: {}", err));
}
// Compute disk usage percentage
@@ -366,7 +411,7 @@ impl Operation for RebalanceStatus {
};
let data = serde_json::to_string(&admin_status)
.map_err(|e| s3_error!(InternalError, "Failed to serialize rebalance status response: {}", e))?;
.map_err(|e| s3_error!(InternalError, "failed to serialize rebalance status response: {}", e))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
@@ -381,10 +426,17 @@ pub struct RebalanceStop {}
impl Operation for RebalanceStop {
#[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle RebalanceStop");
info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "stop",
state = "requested",
"admin rebalance state"
);
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "Failed to stop rebalance: missing credentials"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -401,37 +453,66 @@ impl Operation for RebalanceStop {
.await?;
let Some(store) = new_object_layer_fn() else {
return Err(s3_error!(InternalError, "Failed to stop rebalance: object layer not initialized"));
return Err(s3_error!(InternalError, "object layer is not initialized"));
};
if !store.is_rebalance_conflicting_with_decommission().await {
return Err(s3_error!(NoSuchResource, "Pool rebalance is not started"));
return Err(s3_error!(NoSuchResource, "pool rebalance is not started"));
}
if let Some(notification_sys) = get_global_notification_sys() {
notification_sys
.stop_rebalance()
.await
.map_err(|e| s3_error!(InternalError, "Failed to stop rebalance via notification system: {}", e))?;
.map_err(|e| s3_error!(InternalError, "failed to stop rebalance via notification system: {}", e))?;
} else {
store
.stop_rebalance()
.await
.map_err(|e| s3_error!(InternalError, "Failed to stop rebalance: {}", e))?;
.map_err(|e| s3_error!(InternalError, "failed to stop rebalance: {}", e))?;
store
.save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt)
.await
.map_err(|e| s3_error!(InternalError, "Failed to persist rebalance stop metadata: {}", e))?;
.map_err(|e| s3_error!(InternalError, "failed to persist rebalance stop metadata: {}", e))?;
}
warn!("handle RebalanceStop save_rebalance_stats done ");
info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "stop",
state = "local_stop_persisted",
"admin rebalance state"
);
if let Some(notification_sys) = get_global_notification_sys() {
warn!("handle RebalanceStop notification_sys load_rebalance_meta");
info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "stop",
state = "propagation_started",
"admin rebalance state"
);
if let Err(err) = notification_sys.load_rebalance_meta(false).await {
warn!("rebalance stop propagation failed after local state update: {err}");
info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "stop",
result = "propagation_failed",
error = %err,
"admin rebalance state"
);
}
warn!("handle RebalanceStop notification_sys load_rebalance_meta done");
info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "stop",
state = "propagation_completed",
"admin rebalance state"
);
}
let mut header = HeaderMap::new();
rustfs/src/admin/handlers/service_account.rs
+68 -14
View File
@@ -50,6 +50,7 @@ use url::form_urlencoded;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_SERVICE_ACCOUNT: &str = "service_account";
const EVENT_ADMIN_SERVICE_ACCOUNT_STATE: &str = "admin_service_account_state";
fn sr_session_policy_from_value(value: Option<&serde_json::Value>) -> S3Result<SRSessionPolicy> {
let Some(value) = value else {
@@ -110,10 +111,11 @@ fn map_service_account_lookup_error(err: rustfs_iam::error::Error, action: &str)
debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = "service_account_lookup_failed",
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
action,
result = "lookup_failed",
error = ?err,
"Service account lookup failed"
"admin service account state"
);
if is_err_no_such_service_account(&err) {
iam_error_to_s3_error(err)
@@ -126,10 +128,11 @@ fn map_temp_account_lookup_error(err: rustfs_iam::error::Error, action: &str) ->
debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = "temporary_account_lookup_failed",
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
action,
result = "temporary_lookup_failed",
error = ?err,
"Temporary account lookup failed"
"admin service account state"
);
if is_err_no_such_temp_account(&err) {
iam_error_to_s3_error(err)
@@ -144,9 +147,10 @@ fn parse_service_account_policy(policy: &serde_json::Value) -> S3Result<Policy>
debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = "service_account_policy_parse_failed",
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
result = "policy_parse_failed",
error = ?e,
"Failed to parse service account policy"
"admin service account state"
);
match e {
rustfs_policy::error::Error::PolicyError(rustfs_policy::policy::Error::NonResource) => {
@@ -372,10 +376,11 @@ impl Operation for AddServiceAccount {
debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = "service_account_create_failed",
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
target_user = %target_user,
result = "create_failed",
error = ?e,
"Failed to create service account"
"admin service account state"
);
s3_error!(InternalError, "create service account failed, e: {:?}", e)
})?;
@@ -410,7 +415,16 @@ impl Operation for AddServiceAccount {
})
.await
{
warn!(access_key = %new_cred.access_key, error = ?err, "site replication add service account hook failed");
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
access_key = %new_cred.access_key,
action = "create",
result = "site_replication_hook_failed",
error = ?err,
"admin service account state"
);
}
let resp = AddServiceAccountResp {
@@ -871,7 +885,14 @@ impl Operation for ListServiceAccount {
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key)
.await
.map_err(|e| {
debug!("check key failed: {e:?}");
debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
result = "check_key_failed",
error = ?e,
"admin service account state"
);
s3_error!(InternalError, "check key failed")
})?;
@@ -927,7 +948,15 @@ impl Operation for ListServiceAccount {
};
let service_accounts = iam_store.list_service_accounts(&target_account).await.map_err(|e| {
debug!("list service account failed: {e:?}");
debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
target_user = %target_account,
result = "list_service_accounts_failed",
error = ?e,
"admin service account state"
);
s3_error!(InternalError, "list service account failed")
})?;
@@ -1114,7 +1143,15 @@ impl Operation for ListAccessKeysBulk {
if list_sts_keys {
let sts_keys = iam_store.list_sts_accounts(&user).await.map_err(|e| {
debug!("list sts account failed: {e:?}");
debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
target_user = %user,
result = "list_sts_accounts_failed",
error = ?e,
"admin service account state"
);
s3_error!(InternalError, "list sts account failed")
})?;
@@ -1135,7 +1172,15 @@ impl Operation for ListAccessKeysBulk {
if list_service_accounts {
let service_accounts = iam_store.list_service_accounts(&user).await.map_err(|e| {
debug!("list service account failed: {e:?}");
debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
target_user = %user,
result = "list_service_accounts_failed",
error = ?e,
"admin service account state"
);
s3_error!(InternalError, "list service account failed")
})?;
@@ -1279,7 +1324,16 @@ impl Operation for DeleteServiceAccount {
})
.await
{
warn!(access_key = %query.access_key, error = ?err, "site replication delete service account hook failed");
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
access_key = %query.access_key,
action = "delete",
result = "site_replication_hook_failed",
error = ?err,
"admin service account state"
);
}
let mut header = HeaderMap::new();
rustfs/src/admin/handlers/site_replication.rs
+95 -11
View File
@@ -93,6 +93,10 @@ use tracing::warn;
use url::{Url, form_urlencoded};
use uuid::Uuid;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_SITE_REPLICATION: &str = "site_replication";
const EVENT_ADMIN_SITE_REPLICATION_STATE: &str = "admin_site_replication_state";
const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json";
const SITE_REPL_ADD_SUCCESS: &str = "Requested sites were configured for replication successfully.";
const SITE_REPL_EDIT_SUCCESS: &str = "Requested site was updated successfully.";
@@ -2000,7 +2004,15 @@ async fn build_status_info(state: &SiteReplicationState, local_peer: &PeerInfo,
reachable_peers.insert(deployment_id.clone());
}
Err(err) => {
warn!(peer = %peer.endpoint, error = ?err, "site replication peer metainfo fetch failed");
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
peer = %peer.endpoint,
result = "peer_metainfo_fetch_failed",
error = ?err,
"admin site replication state"
);
site_infos.insert(deployment_id.clone(), SRInfo::default());
}
}
@@ -2536,7 +2548,14 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local
let buckets = match store.list_bucket(&BucketOptions::default()).await {
Ok(b) => b,
Err(err) => {
warn!(error = ?err, "site replication backfill: failed to list buckets");
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
result = "backfill_list_buckets_failed",
error = ?err,
"admin site replication state"
);
return;
}
};
@@ -2546,14 +2565,38 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local
let name = &bucket.name;
if let Err(err) = ensure_site_replication_bucket_versioning(name).await {
warn!(bucket = %name, error = ?err, "site replication backfill: versioning setup failed");
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_versioning_setup_failed",
error = ?err,
"admin site replication state"
);
continue;
}
if let Err(err) = ensure_site_replication_bucket_targets(name, state, local_peer, None).await {
warn!(bucket = %name, error = ?err, "site replication backfill: targets setup failed");
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_targets_setup_failed",
error = ?err,
"admin site replication state"
);
}
if let Err(err) = ensure_site_replication_bucket_replication_config(name, state, local_peer).await {
warn!(bucket = %name, error = ?err, "site replication backfill: replication config setup failed");
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_replication_config_setup_failed",
error = ?err,
"admin site replication state"
);
}
// Broadcast the bucket to peers so they create it too (idempotent on the peer side).
// Read the real lock_enabled flag so peers recreate the bucket with the same object-lock
@@ -2561,12 +2604,29 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local
let lock_enabled = match metadata_sys::get(name).await {
Ok(bm) => bm.lock_enabled,
Err(err) => {
warn!(bucket = %name, error = ?err, "site replication backfill: failed to read bucket metadata, assuming lock_enabled=false");
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_metadata_read_failed",
fallback = "lock_enabled=false",
error = ?err,
"admin site replication state"
);
false
}
};
if let Err(err) = site_replication_make_bucket_hook(name, lock_enabled).await {
warn!(bucket = %name, error = ?err, "site replication backfill: make-bucket broadcast failed");
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_make_bucket_broadcast_failed",
error = ?err,
"admin site replication state"
);
}
// Kick a resync toward every remote peer so existing objects travel across.
for peer in state.peers.values() {
@@ -2575,8 +2635,16 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local
}
let result = start_site_bucket_resync(name, peer, &resync_id).await;
if result.status == "failed" {
warn!(bucket = %name, peer = %peer.endpoint, detail = %result.err_detail,
"site replication backfill: resync kick failed");
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
peer = %peer.endpoint,
result = "backfill_resync_kick_failed",
detail = %result.err_detail,
"admin site replication state"
);
}
}
}
@@ -3173,7 +3241,15 @@ impl Operation for SiteReplicationRemoveHandler {
.await
{
let err_detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint));
warn!(peer = %peer.endpoint, error = %err_detail, "site replication peer remove notification failed");
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
peer = %peer.endpoint,
result = "peer_remove_notification_failed",
error = %err_detail,
"admin site replication state"
);
peer_errors.push(err_detail);
}
}
@@ -3767,7 +3843,15 @@ impl Operation for SRRotateServiceAccountHandler {
send_peer_admin_request(&peer.endpoint, SITE_REPLICATION_PEER_JOIN_PATH, &svc_ak, &new_sk, &join_req).await
{
let detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint));
warn!(peer = %peer.endpoint, error = %detail, "site replication service account rotation failed for peer");
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
peer = %peer.endpoint,
result = "service_account_rotation_failed",
error = %detail,
"admin site replication state"
);
peer_errors.push(detail);
}
}
rustfs/src/admin/handlers/target_descriptor.rs
+35 -35
View File
@@ -472,7 +472,7 @@ where
if !allowed_keys.contains(key) {
return Err(s3_error!(
InvalidArgument,
"key '{}' not allowed for {} type '{}'",
"key '{}' is not allowed for {} type '{}'",
key,
target_label,
target_type
@@ -492,7 +492,7 @@ where
pub(crate) async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> {
if !queue_dir.is_empty() {
if !Path::new(queue_dir).is_absolute() {
return Err(s3_error!(InvalidArgument, "queue_dir must be absolute path"));
return Err(s3_error!(InvalidArgument, "queue_dir must be an absolute path"));
}
retry_with_backoff(
|| async { tokio::fs::metadata(queue_dir).await.map(|_| ()) },
@@ -502,7 +502,7 @@ pub(crate) async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> {
.await
.map_err(|e| match e.kind() {
ErrorKind::NotFound => s3_error!(InvalidArgument, "queue_dir does not exist"),
ErrorKind::PermissionDenied => s3_error!(InvalidArgument, "queue_dir exists but permission denied"),
ErrorKind::PermissionDenied => s3_error!(InvalidArgument, "queue_dir exists but permission is denied"),
_ => s3_error!(InvalidArgument, "failed to access queue_dir: {}", e),
})?;
}
@@ -533,7 +533,7 @@ where
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?;
timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, default_queue_dir))
.await
.map_err(|_| s3_error!(InvalidArgument, "target validation timed out"))??;
.map_err(|_| s3_error!(InvalidArgument, "target validation timed out after 10s"))??;
let mut kvs = KVS::new();
for (key, value) in kv_map {
@@ -652,13 +652,13 @@ async fn validate_webhook_request(kv_map: &HashMap<String, String>) -> S3Result<
.get("endpoint")
.map(String::as_str)
.ok_or_else(|| s3_error!(InvalidArgument, "endpoint is required"))?;
let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?;
let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint URL: {}", e))?;
match parsed_endpoint.scheme() {
"http" | "https" => {}
other => {
return Err(s3_error!(
InvalidArgument,
"unsupported endpoint scheme: {} (only http and https are allowed)",
"unsupported endpoint scheme '{}'; only http and https are allowed",
other
));
}
@@ -683,7 +683,7 @@ async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()>
let endpoint = kv_map
.get(MQTT_BROKER)
.map(String::as_str)
.ok_or_else(|| s3_error!(InvalidArgument, "broker endpoint is required"))?;
.ok_or_else(|| s3_error!(InvalidArgument, "broker URL is required"))?;
let topic = kv_map
.get(MQTT_TOPIC)
.map(String::as_str)
@@ -701,12 +701,12 @@ async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()>
.map_err(|e| s3_error!(InvalidArgument, "invalid MQTT TLS settings: {}", e))?;
let parsed_broker = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid broker URL: {}", e))?;
rustfs_targets::target::mqtt::validate_mqtt_broker_url(&parsed_broker, &tls)
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
.map_err(|e| s3_error!(InvalidArgument, "invalid broker URL: {}", e))?;
check_mqtt_broker_available_with_tls(parsed_broker.as_str(), topic, username, password, &tls)
.await
.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "MQTT broker check failed: {}", e),
TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid MQTT configuration: {}", e),
_ => s3_error!(InvalidArgument, "mqtt connectivity check failed: {}", e),
})?;
if let Some(queue_dir) = kv_map.get("queue_dir") {
@@ -755,10 +755,10 @@ async fn validate_nats_request_impl(
validate_queue_dir(queue_dir.as_str()).await?;
}
let args = build_nats_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
.map_err(|e| s3_error!(InvalidArgument, "invalid NATS configuration: {}", e))?;
check_nats_server_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "NATS server check failed: {}", e),
TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid NATS configuration: {}", e),
_ => s3_error!(InvalidArgument, "nats connectivity check failed: {}", e),
})
}
@@ -780,17 +780,17 @@ async fn validate_kafka_request_impl(
}
if !kv_map.contains_key(KAFKA_BROKERS) {
return Err(s3_error!(InvalidArgument, "Kafka brokers are required"));
return Err(s3_error!(InvalidArgument, "kafka brokers are required"));
}
if !kv_map.contains_key(KAFKA_TOPIC) {
return Err(s3_error!(InvalidArgument, "Kafka topic is required"));
return Err(s3_error!(InvalidArgument, "kafka topic is required"));
}
let args = build_kafka_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
.map_err(|e| s3_error!(InvalidArgument, "invalid Kafka configuration: {}", e))?;
check_kafka_broker_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "Kafka broker check failed: {}", e),
TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid Kafka configuration: {}", e),
_ => s3_error!(InvalidArgument, "kafka connectivity check failed: {}", e),
})
}
@@ -811,10 +811,10 @@ async fn validate_amqp_request_impl(
validate_queue_dir(queue_dir.as_str()).await?;
}
let args = build_amqp_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
.map_err(|e| s3_error!(InvalidArgument, "invalid AMQP configuration: {}", e))?;
check_amqp_broker_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "AMQP broker check failed: {}", e),
TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid AMQP configuration: {}", e),
_ => s3_error!(InvalidArgument, "amqp connectivity check failed: {}", e),
})
}
@@ -835,10 +835,10 @@ async fn validate_pulsar_request_impl(
validate_queue_dir(queue_dir.as_str()).await?;
}
let args = build_pulsar_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
.map_err(|e| s3_error!(InvalidArgument, "invalid Pulsar configuration: {}", e))?;
check_pulsar_broker_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "Pulsar broker check failed: {}", e),
TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid Pulsar configuration: {}", e),
_ => s3_error!(InvalidArgument, "pulsar connectivity check failed: {}", e),
})
}
@@ -851,11 +851,11 @@ async fn validate_mysql_request(
validate_queue_dir(queue_dir.as_str()).await?;
}
let args =
build_mysql_args(&to_kvs(kv_map), default_queue_dir, target_type).map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
let args = build_mysql_args(&to_kvs(kv_map), default_queue_dir, target_type)
.map_err(|e| s3_error!(InvalidArgument, "invalid MySQL configuration: {}", e))?;
check_mysql_server_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "MySQL server check failed: {}", e),
TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid MySQL configuration: {}", e),
_ => s3_error!(InvalidArgument, "mysql connectivity check failed: {}", e),
})
}
@@ -876,10 +876,10 @@ async fn validate_postgres_request_impl(
validate_queue_dir(queue_dir.as_str()).await?;
}
let args = build_postgres_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
.map_err(|e| s3_error!(InvalidArgument, "invalid PostgreSQL configuration: {}", e))?;
check_postgres_server_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "PostgreSQL server check failed: {}", e),
TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid PostgreSQL configuration: {}", e),
_ => s3_error!(InvalidArgument, "postgres connectivity check failed: {}", e),
})
}
@@ -894,13 +894,13 @@ async fn validate_redis_request(
}
validate_redis_config(&to_kvs(kv_map), default_queue_dir, default_channel)
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
.map_err(|e| s3_error!(InvalidArgument, "invalid Redis configuration: {}", e))?;
let args = build_redis_args(&to_kvs(kv_map), default_queue_dir, default_channel, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
.map_err(|e| s3_error!(InvalidArgument, "invalid Redis configuration: {}", e))?;
check_redis_server_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "Redis server check failed: {}", e),
TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid Redis configuration: {}", e),
_ => s3_error!(InvalidArgument, "redis connectivity check failed: {}", e),
})
}
rustfs/src/admin/handlers/tier.rs
+215 -47
View File
@@ -57,6 +57,10 @@ use time::OffsetDateTime;
use tokio::spawn;
use tracing::{debug, warn};
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_TIER: &str = "tier";
const EVENT_ADMIN_TIER_STATE: &str = "admin_tier_state";
#[derive(Debug, Clone, serde::Deserialize, Default)]
pub struct AddTierQuery {
#[serde(rename = "accessKey")]
@@ -91,9 +95,14 @@ fn spawn_transition_tier_config_propagation(action: &'static str) {
for peer_result in notification_sys.load_transition_tier_config().await {
if let Some(err) = peer_result.err {
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = action,
host = if peer_result.host.is_empty() { "<unknown>" } else { peer_result.host.as_str() },
result = "propagation_failed",
error = %err,
"tier {action} propagation failed after local save"
"admin tier state"
);
}
}
@@ -174,7 +183,7 @@ impl Operation for AddTier {
let query = {
if let Some(query) = req.uri.query() {
let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
AddTierQuery::default()
@@ -182,7 +191,7 @@ impl Operation for AddTier {
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -202,13 +211,21 @@ impl Operation for AddTier {
let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
result = "body_read_failed",
error = ?e,
"admin tier state"
);
return Err(s3_error!(InvalidRequest, "tier configuration body too large or failed to read"));
}
};
let mut args: TierConfig = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?;
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
match args.tier_type {
TierType::S3 => {
@@ -240,33 +257,67 @@ impl Operation for AddTier {
}
_ => (),
}
debug!("add tier args {:?}", args);
debug!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
tier_name = %args.name,
tier_type = ?args.tier_type,
state = "decoded",
"admin tier state"
);
let tier_name_for_log = args.name.clone();
let mut force: bool = false;
let force_str = query.force.clone().unwrap_or_default();
if !force_str.is_empty() {
force = force_str.parse().map_err(|e| {
warn!("parse force failed, e: {:?}", e);
s3_error!(InvalidRequest, "parse force failed")
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
result = "force_parse_failed",
error = ?e,
"admin tier state"
);
s3_error!(InvalidRequest, "invalid force flag")
})?;
}
match args.name.as_str() {
storageclass::STANDARD | storageclass::RRS => {
warn!("tier reserved name, args.name: {}", args.name);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
tier_name = %args.name,
result = "reserved_name_rejected",
"admin tier state"
);
return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name"));
}
&_ => (),
}
let Some(store) = resolve_object_store_handle() else {
return Err(s3_error!(InvalidRequest, "object store not init"));
return Err(s3_error!(InternalError, "object store is not initialized"));
};
{
let tier_config_mgr_handle = resolve_tier_config_handle();
let mut tier_config_mgr = tier_config_mgr_handle.write().await;
if let Err(err) = tier_config_mgr.reload(store).await {
warn!("tier_config_mgr reload failed, e: {:?}", err);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
result = "reload_failed",
error = ?err,
"admin tier state"
);
return Err(S3Error::with_message(
S3ErrorCode::Custom("TierAddFailed".into()),
format!("tier reload failed. {err}"),
@@ -276,27 +327,36 @@ impl Operation for AddTier {
return if err.code == ERR_TIER_ALREADY_EXISTS.code {
Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameAlreadyExist".into()),
"tier name already exists!",
"tier name already exists",
))
} else if err.code == ERR_TIER_NAME_NOT_UPPERCASE.code {
Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameNotUppercase".into()),
"tier name not uppercase!",
"tier name must be uppercase",
))
} else if err.code == ERR_TIER_BACKEND_IN_USE.code {
Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameBackendInUse!".into()),
"tier name backend in use!",
"tier backend is already in use",
))
} else if err.code == ERR_TIER_CONNECT_ERR.code {
Err(S3Error::with_message(
S3ErrorCode::Custom("TierConnectError".into()),
"tier connect error!",
"tier connectivity check failed",
))
} else if err.code == ERR_TIER_INVALID_CREDENTIALS.code {
Err(S3Error::with_message(S3ErrorCode::Custom(err.code.clone().into()), err.message))
} else {
warn!("tier_config_mgr add failed, e: {:?}", err);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
tier_name = %tier_name_for_log,
result = "add_failed",
error = ?err,
"admin tier state"
);
Err(S3Error::with_message(
S3ErrorCode::Custom("TierAddFailed".into()),
format!("tier add failed. {err}"),
@@ -304,7 +364,15 @@ impl Operation for AddTier {
};
}
if let Err(e) = tier_config_mgr.save().await {
warn!("tier_config_mgr save failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
result = "save_failed",
error = ?e,
"admin tier state"
);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier save failed"));
}
}
@@ -324,7 +392,7 @@ impl Operation for EditTier {
let query = {
if let Some(query) = req.uri.query() {
let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
AddTierQuery::default()
@@ -332,7 +400,7 @@ impl Operation for EditTier {
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -352,27 +420,51 @@ impl Operation for EditTier {
let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "edit_tier",
result = "body_read_failed",
error = ?e,
"admin tier state"
);
return Err(s3_error!(InvalidRequest, "tier configuration body too large or failed to read"));
}
};
let creds: TierCreds = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?;
debug!("edit tier args {:?}", creds);
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default();
debug!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "edit_tier",
tier_name = %tier_name,
state = "decoded",
"admin tier state"
);
let Some(store) = resolve_object_store_handle() else {
return Err(s3_error!(InvalidRequest, "object store not init"));
return Err(s3_error!(InternalError, "object store is not initialized"));
};
{
let tier_config_mgr_handle = resolve_tier_config_handle();
let mut tier_config_mgr = tier_config_mgr_handle.write().await;
if let Err(err) = tier_config_mgr.reload(store).await {
warn!("tier_config_mgr reload failed, e: {:?}", err);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "edit_tier",
result = "reload_failed",
error = ?err,
"admin tier state"
);
return Err(S3Error::with_message(
S3ErrorCode::Custom("TierEditFailed".into()),
format!("tier reload failed. {err}"),
@@ -380,14 +472,23 @@ impl Operation for EditTier {
}
if let Err(err) = tier_config_mgr.edit(&tier_name, creds).await {
return if err.code == ERR_TIER_NOT_FOUND.code {
Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!"))
Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found"))
} else if err.code == ERR_TIER_MISSING_CREDENTIALS.code {
Err(S3Error::with_message(
S3ErrorCode::Custom("TierMissingCredentials".into()),
"tier missing credentials!",
"tier credentials are required",
))
} else {
warn!("tier_config_mgr edit failed, e: {:?}", err);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "edit_tier",
tier_name = %tier_name,
result = "edit_failed",
error = ?err,
"admin tier state"
);
Err(S3Error::with_message(
S3ErrorCode::Custom("TierEditFailed".into()),
format!("tier edit failed. {err}"),
@@ -395,7 +496,15 @@ impl Operation for EditTier {
};
}
if let Err(e) = tier_config_mgr.save().await {
warn!("tier_config_mgr save failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "edit_tier",
result = "save_failed",
error = ?e,
"admin tier state"
);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed"));
}
}
@@ -466,7 +575,7 @@ impl Operation for RemoveTier {
let query = {
if let Some(query) = req.uri.query() {
let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
AddTierQuery::default()
@@ -474,7 +583,7 @@ impl Operation for RemoveTier {
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -494,22 +603,38 @@ impl Operation for RemoveTier {
let force_str = query.force.clone().unwrap_or_default();
if !force_str.is_empty() {
force = force_str.parse().map_err(|e| {
warn!("parse force failed, e: {:?}", e);
s3_error!(InvalidRequest, "parse force failed")
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "remove_tier",
result = "force_parse_failed",
error = ?e,
"admin tier state"
);
s3_error!(InvalidRequest, "invalid force flag")
})?;
}
let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default();
let Some(store) = resolve_object_store_handle() else {
return Err(s3_error!(InvalidRequest, "object store not init"));
return Err(s3_error!(InternalError, "object store is not initialized"));
};
{
let tier_config_mgr_handle = resolve_tier_config_handle();
let mut tier_config_mgr = tier_config_mgr_handle.write().await;
if let Err(err) = tier_config_mgr.reload(store).await {
warn!("tier_config_mgr reload failed, e: {:?}", err);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "remove_tier",
result = "reload_failed",
error = ?err,
"admin tier state"
);
return Err(S3Error::with_message(
S3ErrorCode::Custom("TierRemoveFailed".into()),
format!("tier reload failed. {err}"),
@@ -517,11 +642,23 @@ impl Operation for RemoveTier {
}
if let Err(err) = tier_config_mgr.remove(&tier_name, force).await {
return if err.code == ERR_TIER_NOT_FOUND.code {
Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found."))
Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found"))
} else if err.code == ERR_TIER_BACKEND_NOT_EMPTY.code {
Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse".into()), "tier is used."))
Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameBackendInUse".into()),
"tier backend is not empty",
))
} else {
warn!("tier_config_mgr remove failed, e: {:?}", err);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "remove_tier",
tier_name = %tier_name,
result = "remove_failed",
error = ?err,
"admin tier state"
);
Err(S3Error::with_message(
S3ErrorCode::Custom("TierRemoveFailed".into()),
format!("tier remove failed. {err}"),
@@ -530,7 +667,15 @@ impl Operation for RemoveTier {
}
if let Err(e) = tier_config_mgr.save().await {
warn!("tier_config_mgr save failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "remove_tier",
result = "save_failed",
error = ?e,
"admin tier state"
);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier save failed"));
}
}
@@ -549,7 +694,7 @@ pub struct VerifyTier {}
impl Operation for VerifyTier {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -601,7 +746,7 @@ impl Operation for GetTierInfo {
let query = {
if let Some(query) = req.uri.query() {
let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
AddTierQuery::default()
@@ -683,7 +828,7 @@ impl Operation for ClearTier {
let query = {
if let Some(query) = req.uri.query() {
let input: ClearTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
ClearTierQuery::default()
@@ -691,7 +836,7 @@ impl Operation for ClearTier {
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -718,23 +863,46 @@ impl Operation for ClearTier {
rand.insert_str(3, &t.day().to_string());
rand.insert_str(17, &t.month().to_string());
rand.insert_str(23, &t.year().to_string());
warn!("tier_config_mgr rand: {}", rand);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "clear_tier",
state = "challenge_generated",
"admin tier state"
);
if query.rand != Some(rand) {
return Err(s3_error!(InvalidRequest, "get rand failed"));
return Err(s3_error!(InvalidRequest, "invalid clear-tier confirmation token"));
};
let tier_config_mgr_handle = resolve_tier_config_handle();
let mut tier_config_mgr = tier_config_mgr_handle.write().await;
//tier_config_mgr.reload(api);
if let Err(err) = tier_config_mgr.clear_tier(force).await {
warn!("tier_config_mgr clear failed, e: {:?}", err);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "clear_tier",
result = "clear_failed",
error = ?err,
"admin tier state"
);
return Err(S3Error::with_message(
S3ErrorCode::Custom("TierClearFailed".into()),
format!("tier clear failed. {err}"),
));
}
if let Err(e) = tier_config_mgr.save().await {
warn!("tier_config_mgr save failed, e: {:?}", e);
warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "clear_tier",
result = "save_failed",
error = ?e,
"admin tier state"
);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed"));
}
rustfs/src/admin/handlers/user.rs
+82 -48
View File
@@ -51,6 +51,10 @@ use std::{collections::HashMap, io::Cursor, str::from_utf8};
use tracing::{debug, warn};
use zip::{ZipArchive, ZipWriter, result::ZipError, write::SimpleFileOptions};
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_USER: &str = "user";
const EVENT_ADMIN_USER_STATE: &str = "admin_user_state";
#[derive(Debug, Deserialize, Default)]
pub struct AddUserQuery {
#[serde(rename = "accessKey", alias = "access-key")]
@@ -184,7 +188,7 @@ impl Operation for AddUser {
let query = {
if let Some(query) = req.uri.query() {
let input: AddUserQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
AddUserQuery::default()
@@ -192,7 +196,7 @@ impl Operation for AddUser {
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -210,39 +214,41 @@ impl Operation for AddUser {
// .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, format!("decrypt_data err {}", e)))?;
let args: AddOrUpdateUserReq = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?;
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
if args.secret_key.is_empty() {
return Err(s3_error!(InvalidArgument, "access key is empty"));
return Err(s3_error!(InvalidArgument, "secret key is required"));
}
if let Some(sys_cred) = get_global_action_cred()
&& constant_time_eq(&sys_cred.access_key, ak)
{
return Err(s3_error!(InvalidArgument, "can't create user with system access key"));
return Err(s3_error!(InvalidArgument, "cannot create a user with the system access key"));
}
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InvalidRequest, "iam not init"));
return Err(s3_error!(InternalError, "iam is not initialized"));
};
if let Some(user) = iam_store.get_user(ak).await {
if (user.credentials.is_temp() || user.credentials.is_service_account()) && cred.parent_user == ak {
return Err(s3_error!(InvalidArgument, "can't create user with service account access key"));
return Err(s3_error!(InvalidArgument, "cannot create a user with a service account access key"));
}
} else if has_space_be(ak) {
return Err(s3_error!(InvalidArgument, "access key has space"));
return Err(s3_error!(InvalidArgument, "access key contains spaces"));
}
if from_utf8(ak.as_bytes()).is_err() {
return Err(s3_error!(InvalidArgument, "access key is not utf8"));
return Err(s3_error!(InvalidArgument, "access key is not valid UTF-8"));
}
let check_deny_only = should_check_deny_only(ak, &cred);
debug!(
target = "rustfs::admin::handlers::user",
operation = "AddUser",
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_USER,
event = EVENT_ADMIN_USER_STATE,
action = "add_user",
query_access_key = %ak,
signer_access_key = %cred.access_key,
is_temp = cred.is_temp(),
@@ -252,7 +258,8 @@ impl Operation for AddUser {
jwt_parent_claim_present = cred.claims.as_ref().and_then(|c| c.get("parent")).is_some(),
check_deny_only,
is_owner = owner,
"authorization context before validate_admin_request (no secrets)"
state = "authorization_context",
"admin user state"
);
// For eligible self operations, only explicit Deny should block the request.
@@ -269,7 +276,7 @@ impl Operation for AddUser {
let updated_at = iam_store
.create_user(ak, &args)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("create_user err {e}")))?;
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to create user: {e}")))?;
if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
r#type: "iam-user".to_string(),
@@ -285,7 +292,16 @@ impl Operation for AddUser {
})
.await
{
warn!(access_key = %ak, error = ?err, "site replication create user hook failed");
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_USER,
event = EVENT_ADMIN_USER_STATE,
access_key = %ak,
action = "create_user",
result = "site_replication_hook_failed",
error = ?err,
"admin user state"
);
}
let mut header = HeaderMap::new();
@@ -302,7 +318,7 @@ impl Operation for SetUserStatus {
let query = {
if let Some(query) = req.uri.query() {
let input: AddUserQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
AddUserQuery::default()
@@ -316,11 +332,11 @@ impl Operation for SetUserStatus {
}
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
if constant_time_eq(&input_cred.access_key, ak) {
return Err(s3_error!(InvalidArgument, "can't change status of self"));
return Err(s3_error!(InvalidArgument, "cannot change the status of the current user"));
}
let (cred, owner) =
@@ -340,13 +356,13 @@ impl Operation for SetUserStatus {
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, e))?;
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InvalidRequest, "iam not init"));
return Err(s3_error!(InternalError, "iam is not initialized"));
};
iam_store
.set_user_status(ak, status)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("set_user_status err {e}")))?;
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to set user status: {e}")))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -365,7 +381,7 @@ pub struct ListUsers {}
impl Operation for ListUsers {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -384,7 +400,7 @@ impl Operation for ListUsers {
let query = {
if let Some(query) = req.uri.query() {
let input: BucketQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
BucketQuery::default()
@@ -392,7 +408,7 @@ impl Operation for ListUsers {
};
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InvalidRequest, "iam not init"));
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let users = {
@@ -410,7 +426,7 @@ impl Operation for ListUsers {
};
let data = serde_json::to_vec(&users)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal users err {e}")))?;
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize response: {e}")))?;
let (data, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, data)?;
let mut header = HeaderMap::new();
@@ -425,7 +441,7 @@ pub struct RemoveUser {}
impl Operation for RemoveUser {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -444,7 +460,7 @@ impl Operation for RemoveUser {
let query = {
if let Some(query) = req.uri.query() {
let input: AddUserQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
AddUserQuery::default()
@@ -458,37 +474,35 @@ impl Operation for RemoveUser {
}
let sys_cred = get_global_action_cred()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "get_global_action_cred failed"))?;
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "failed to load global credentials"))?;
if ak == sys_cred.access_key || ak == cred.access_key || cred.parent_user == ak {
return Err(s3_error!(InvalidArgument, "can't remove self"));
return Err(s3_error!(InvalidArgument, "cannot remove the current user"));
}
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InvalidRequest, "iam not init"));
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let (is_temp, _) = iam_store
.is_temp_user(ak)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("is_temp_user err {e}")))?;
let (is_temp, _) = iam_store.is_temp_user(ak).await.map_err(|e| {
S3Error::with_message(S3ErrorCode::InternalError, format!("failed to query temporary user state: {e}"))
})?;
if is_temp {
return Err(s3_error!(InvalidArgument, "can't remove temp user"));
return Err(s3_error!(InvalidArgument, "cannot remove a temporary user"));
}
let (is_service_account, _) = iam_store
.is_service_account(ak)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("is_service_account err {e}")))?;
let (is_service_account, _) = iam_store.is_service_account(ak).await.map_err(|e| {
S3Error::with_message(S3ErrorCode::InternalError, format!("failed to query service account state: {e}"))
})?;
if is_service_account {
return Err(s3_error!(InvalidArgument, "can't remove service account"));
return Err(s3_error!(InvalidArgument, "cannot remove a service account"));
}
iam_store
.delete_user(ak, true)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("delete_user err {e}")))?;
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to delete user: {e}")))?;
if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
r#type: "iam-user".to_string(),
@@ -504,7 +518,16 @@ impl Operation for RemoveUser {
})
.await
{
warn!(access_key = %ak, error = ?err, "site replication delete user hook failed");
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_USER,
event = EVENT_ADMIN_USER_STATE,
access_key = %ak,
action = "delete_user",
result = "site_replication_hook_failed",
error = ?err,
"admin user state"
);
}
let mut header = HeaderMap::new();
@@ -521,7 +544,7 @@ impl Operation for GetUserInfo {
let query = {
if let Some(query) = req.uri.query() {
let input: AddUserQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input
} else {
AddUserQuery::default()
@@ -535,11 +558,11 @@ impl Operation for GetUserInfo {
}
let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InvalidRequest, "iam not init"));
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
@@ -548,8 +571,10 @@ impl Operation for GetUserInfo {
let check_deny_only = should_check_deny_only(ak, &cred);
debug!(
target = "rustfs::admin::handlers::user",
operation = "GetUserInfo",
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_USER,
event = EVENT_ADMIN_USER_STATE,
action = "get_user_info",
query_access_key = %ak,
signer_access_key = %cred.access_key,
is_temp = cred.is_temp(),
@@ -559,7 +584,8 @@ impl Operation for GetUserInfo {
jwt_parent_claim_present = cred.claims.as_ref().and_then(|c| c.get("parent")).is_some(),
check_deny_only,
is_owner = owner,
"authorization context before validate_admin_request (no secrets)"
state = "authorization_context",
"admin user state"
);
// For eligible self operations, only explicit Deny should block the request.
@@ -576,7 +602,7 @@ impl Operation for GetUserInfo {
let info = iam_store.get_user_info(ak).await.map_err(iam_error_to_s3_error)?;
let data = serde_json::to_vec(&info)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal user err {e}")))?;
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize response: {e}")))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -845,7 +871,15 @@ impl Operation for ImportIam {
let body = match input.store_all_limited(MAX_IAM_IMPORT_SIZE).await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_USER,
event = EVENT_ADMIN_USER_STATE,
action = "import_iam",
result = "body_read_failed",
error = ?e,
"admin user state"
);
return Err(s3_error!(InvalidRequest, "get body failed"));
}
};
rustfs/src/admin/router.rs
+41 -18
View File
@@ -104,6 +104,7 @@ use uuid::Uuid;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_OBJECT_LAMBDA: &str = "object_lambda";
const LOG_SUBSYSTEM_LIVE_EVENTS: &str = "live_events";
const EVENT_ADMIN_ROUTER_STATE: &str = "admin_router_state";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ReplicationExtRoute {
@@ -634,11 +635,12 @@ async fn load_current_server_config() -> S3Result<Config> {
Ok(config) => return Ok(config),
Err(err) => {
warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
event = "object_lambda_config_reload_failed",
result = "config_reload_failed",
error = %err,
"Failed to reload current server config for object lambda request"
"admin router state"
);
}
}
@@ -663,11 +665,12 @@ fn build_object_lambda_http_client(config: &ObjectLambdaWebhookConfig) -> S3Resu
if config.skip_tls_verify {
warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
event = "object_lambda_tls_verification_disabled",
result = "tls_verification_disabled",
endpoint = %config.endpoint,
"Object Lambda target is configured to skip TLS certificate verification"
"admin router state"
);
builder = builder.danger_accept_invalid_certs(true);
} else if !config.client_ca.is_empty() {
@@ -1220,23 +1223,25 @@ async fn fan_in_remote_live_events(
Ok(Ok(batch)) => batch,
Ok(Err(err)) => {
warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "peer_live_events_fetch_failed",
peer = %peer.client.host,
result = "peer_fetch_failed",
error = %err,
"Failed to fetch live events from peer"
"admin router state"
);
break;
}
Err(_) => {
warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "peer_live_events_fetch_failed",
peer = %peer.client.host,
result = "peer_fetch_timeout",
error = "timeout",
"Failed to fetch live events from peer"
"admin router state"
);
break;
}
@@ -1259,13 +1264,14 @@ async fn fan_in_remote_live_events(
}
Err(err) => {
warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "live_event_serialize_failed",
source = "remote_peer",
peer = %peer.client.host,
result = "event_serialize_failed",
error = %err,
"Failed to serialize live event"
"admin router state"
);
}
}
@@ -1273,12 +1279,13 @@ async fn fan_in_remote_live_events(
}
Err(err) => {
warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "peer_live_events_decode_failed",
peer = %peer.client.host,
result = "peer_decode_failed",
error = %err,
"Failed to decode live events from peer"
"admin router state"
);
}
}
@@ -1333,23 +1340,25 @@ fn build_listen_notification_response(uri: &Uri, bucket: Option<&str>) -> S3Resu
}
Err(err) => {
warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "live_event_serialize_failed",
source = "local_stream",
result = "event_serialize_failed",
error = %err,
"Failed to serialize live event"
"admin router state"
);
}
}
}
Err(broadcast::error::RecvError::Lagged(skipped)) => {
warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "live_event_stream_lagged",
result = "stream_lagged",
skipped,
"Live event stream lagged"
"admin router state"
);
}
Err(broadcast::error::RecvError::Closed) => break,
@@ -1509,7 +1518,14 @@ async fn authorize_replication_extension_request(req: &mut S3Request<Body>, ext_
license_check().map_err(|er| match er.kind() {
std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"),
_ => {
error!("license check failed due to unexpected error: {er}");
error!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
result = "license_check_failed",
error = %er,
"admin router state"
);
s3_error!(InternalError, "License validation failed")
}
})?;
@@ -2231,7 +2247,14 @@ async fn authorize_misc_extension_request(req: &mut S3Request<Body>, route: &Mis
license_check().map_err(|er| match er.kind() {
std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"),
_ => {
error!("license check failed due to unexpected error: {er}");
error!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
result = "license_check_failed",
error = %er,
"admin router state"
);
s3_error!(InternalError, "License validation failed")
}
})?;
rustfs/src/app/object_usecase.rs
+32 -32
View File
@@ -127,7 +127,7 @@ use tokio::io::{AsyncRead, ReadBuf};
use tokio::sync::RwLock;
use tokio_tar::Archive;
use tokio_util::io::{ReaderStream, StreamReader};
use tracing::{debug, error, info, instrument, warn};
use tracing::{debug, error, instrument, warn};
use uuid::Uuid;
const ACCEPT_RANGES_BYTES: &str = "bytes";
@@ -1120,15 +1120,18 @@ fn is_post_object_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap)
async fn resolve_put_object_expiration(bucket: &str, obj_info: &ObjectInfo) -> Option<String> {
let Ok((lifecycle_config, _)) = metadata_sys::get_lifecycle_config(bucket).await else {
debug!("resolve_put_object_expiration: lifecycle config not found for bucket {bucket}");
debug!(bucket, state = "config_missing", "PUT object expiration config missing");
return None;
};
let obj_opts = lifecycle::ObjectOpts::from_object_info(obj_info);
let event = lifecycle_config.predict_expiration(&obj_opts).await;
debug!(
"resolve_put_object_expiration: bucket={bucket}, action={:?}, rule_id={}, due={:?}",
event.action, event.rule_id, event.due
bucket,
action = ?event.action,
rule_id = %event.rule_id,
due = ?event.due,
"PUT object expiration resolved"
);
build_put_object_expiration_header(&event)
}
@@ -1184,7 +1187,7 @@ impl DefaultObjectUsecase {
),
)),
Err(e) => {
warn!("Quota check failed for bucket {bucket}: {e}, allowing operation");
warn!(bucket, error = %e, state = "checker_failed", "Bucket quota check degraded to allow");
Ok(())
}
_ => Ok(()),
@@ -1427,7 +1430,7 @@ impl DefaultObjectUsecase {
match ContentType::from_str(content_type) {
Ok(res) => Some(res),
Err(err) => {
error!("parse content-type err {} {:?}", content_type, err);
error!(content_type, error = ?err, "GET object content-type parse failed");
None
}
}
@@ -1624,7 +1627,7 @@ impl DefaultObjectUsecase {
{
let (decrypted_checksums, _is_multipart) =
info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| {
error!("decrypt_checksums error: {}", e);
error!(error = %e, "GetObject checksum decryption failed");
ApiError::from(e)
})?;
@@ -1667,25 +1670,22 @@ impl DefaultObjectUsecase {
if should_buffer_encrypted_object {
let mut buf = Vec::with_capacity(response_content_length as usize);
if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await {
error!("Failed to read decrypted object into memory: {}", e);
error!(error = %e, "GetObject decrypted object buffering failed");
return Err(ApiError::from(StorageError::other(format!("Failed to read decrypted object: {e}"))).into());
}
if buf.len() != response_content_length as usize {
warn!(
"Encrypted object size mismatch during read: expected={} actual={}",
response_content_length,
buf.len()
expected = response_content_length,
actual = buf.len(),
"Encrypted object size mismatch during read"
);
}
return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size));
}
info!(
"Encrypted object: Using unlimited stream for decryption with buffer size {}",
optimal_buffer_size
);
debug!(buffer_size = optimal_buffer_size, "Encrypted object uses streaming decrypt path");
return Ok(Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size));
}
@@ -1698,16 +1698,16 @@ impl DefaultObjectUsecase {
Ok(_) => {
if buf.len() != response_content_length as usize {
warn!(
"Object size mismatch during seek support read: expected={} actual={}",
response_content_length,
buf.len()
expected = response_content_length,
actual = buf.len(),
"Object size mismatch during seek-support read"
);
}
return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size));
}
Err(e) => {
error!("Failed to read object into memory for seek support: {}", e);
error!(error = %e, "GetObject seek-support buffering failed");
}
}
}
@@ -2780,7 +2780,7 @@ impl DefaultObjectUsecase {
&& src_bucket == bucket
&& src_key == key
{
error!("Rejected self-copy operation: bucket={}, key={}", bucket, key);
error!(bucket, key, "Rejected self-copy operation");
return Err(s3_error!(
InvalidRequest,
"Cannot copy an object to itself. Source and destination must be different."
@@ -3679,7 +3679,7 @@ impl DefaultObjectUsecase {
let has_children = match probe_prefix_has_children(store, &bucket, &key, false).await {
Ok(has_children) => has_children,
Err(e) => {
error!("Failed to probe children for prefix (bucket: {}, key: {}): {}", bucket, key, e);
error!(bucket, key, error = %e, "Failed to probe children for prefix");
false
}
};
@@ -3751,7 +3751,7 @@ impl DefaultObjectUsecase {
match ContentType::from_str(content_type) {
Ok(res) => Some(res),
Err(err) => {
error!("parse content-type err {} {:?}", &content_type, err);
error!(content_type = %content_type, error = ?err, "Archive content-type parse failed");
//
None
}
@@ -3765,7 +3765,7 @@ impl DefaultObjectUsecase {
// TODO: range download
let content_length = info.get_actual_size().map_err(|e| {
error!("get_actual_size error: {}", e);
error!(error = %e, "Failed to resolve actual object size");
ApiError::from(e)
})?;
@@ -3881,7 +3881,7 @@ impl DefaultObjectUsecase {
if let Ok(header_value) = tag_count.to_string().parse::<HeaderValue>() {
response.headers.insert(header_name, header_value);
} else {
warn!("Failed to parse x-amz-tagging-count header value, skipping");
warn!("Failed to parse x-amz-tagging-count header; skipping");
}
}
if let Some(retain_date) = metadata_map
@@ -4127,7 +4127,7 @@ impl DefaultObjectUsecase {
err.to_string()
);
} else {
info!("successfully restored transitioned object: {}/{}", bucket_clone, object_clone);
debug!(bucket = %bucket_clone, object = %object_clone, "Transitioned object restored");
}
});
@@ -4302,13 +4302,13 @@ impl DefaultObjectUsecase {
let decoder = CompressionFormat::from_extension(&ext)
.get_decoder(ExtractArchiveEtagReader::new(archive_reader, archive_etag.clone()))
.map_err(|e| {
error!("get_decoder err {:?}", e);
error!(error = ?e, "Archive decoder creation failed");
s3_error!(InvalidArgument, "get_decoder err")
})?;
let mut ar = Archive::new(decoder);
let mut entries = ar.entries().map_err(|e| {
error!("get entries err {:?}", e);
error!(error = ?e, "Archive entry listing failed");
s3_error!(InvalidArgument, "get entries err")
})?;
@@ -4337,10 +4337,10 @@ impl DefaultObjectUsecase {
Ok(f) => f,
Err(e) => {
if extract_options.ignore_errors {
warn!("Skipping archive entry because read failed and ignore-errors is enabled: {e}");
warn!(error = %e, "Archive entry read skipped due to ignore-errors");
continue;
}
error!("Failed to read archive entry: {}", e);
error!(error = %e, "Archive entry read failed");
return Err(s3_error!(InvalidArgument, "Failed to read archive entry: {:?}", e));
}
};
@@ -4349,7 +4349,7 @@ impl DefaultObjectUsecase {
Ok(path) => path,
Err(e) => {
if extract_options.ignore_errors {
warn!("Skipping archive entry because path decode failed and ignore-errors is enabled: {e}");
warn!(error = %e, "Archive path decode skipped due to ignore-errors");
continue;
}
return Err(s3_error!(InvalidArgument, "Failed to decode archive entry path"));
@@ -4361,7 +4361,7 @@ impl DefaultObjectUsecase {
Ok(fpath) => fpath,
Err(err) => {
if extract_options.ignore_errors {
warn!("Skipping archive entry because path is unsafe and ignore-errors is enabled: {err}");
warn!(error = %err, "Unsafe archive path skipped due to ignore-errors");
continue;
}
return Err(err);
@@ -4491,7 +4491,7 @@ impl DefaultObjectUsecase {
Ok(info) => info,
Err(e) => {
if extract_options.ignore_errors {
warn!("Skipping archive entry because object write failed and ignore-errors is enabled: {e}");
warn!(error = %e, "Archive object write skipped due to ignore-errors");
continue;
}
return Err(ApiError::from(e).into());
rustfs/src/auth.rs
+13 -13
View File
@@ -151,7 +151,7 @@ impl S3Auth for IAMAuth {
subsystem = LOG_SUBSYSTEM_KEYSTONE,
principal = %MaskedAccessKey(&creds.parent_user),
result = "token_auth",
"Keystone credentials found in task-local storage"
"Keystone task-local credentials detected"
);
// Return empty secret key - Keystone uses token validation, not AWS signatures
return Ok(SecretKey::from(String::new()));
@@ -170,7 +170,7 @@ impl S3Auth for IAMAuth {
subsystem = LOG_SUBSYSTEM_KEYSTONE,
access_key = %MaskedAccessKey(access_key),
result = "token_auth",
"Keystone access key detected for token-based auth"
"Keystone token-auth access key detected"
);
// Return empty secret key - Keystone uses token validation, not AWS signatures
// The actual credentials are stored in task-local storage by KeystoneAuthMiddleware
@@ -202,7 +202,7 @@ impl S3Auth for IAMAuth {
subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key),
reason = "no_such_user",
"Secret key lookup failed"
"Secret key lookup rejected"
);
}
Err(e) => {
@@ -213,7 +213,7 @@ impl S3Auth for IAMAuth {
access_key = %MaskedAccessKey(access_key),
error = ?e,
reason = "check_key_error",
"Secret key lookup failed"
"Secret key lookup errored"
);
return Err(iam_lookup_error_to_s3_error(&e));
}
@@ -225,7 +225,7 @@ impl S3Auth for IAMAuth {
subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key),
reason = "iam_not_initialized",
"Secret key lookup failed"
"Secret key lookup skipped"
);
}
@@ -257,7 +257,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key),
has_session_token = !session_token.is_empty(),
"Starting access key validation"
"Access key validation started"
);
if let Ok(Some(credentials)) = KEYSTONE_CREDENTIALS.try_with(|creds| creds.clone()) {
debug!(
@@ -265,7 +265,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
component = LOG_COMPONENT_AUTH,
subsystem = LOG_SUBSYSTEM_KEYSTONE,
result = "task_local",
"Keystone credentials found in task-local storage"
"Keystone task-local credentials detected"
);
if !auth_keystone::is_keystone_enabled() {
@@ -283,7 +283,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
.and_then(|c| c.get("keystone_project_name"))
.and_then(|v| v.as_str())
.is_some(),
"Validated Keystone credentials from task-local storage"
"Keystone task-local credentials validated"
);
// Determine if user is admin (owner-level access)
@@ -307,7 +307,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
subsystem = LOG_SUBSYSTEM_KEYSTONE,
principal = %MaskedAccessKey(&credentials.parent_user),
is_owner,
"Evaluated Keystone owner permissions"
"Keystone owner permissions evaluated"
);
return Ok((credentials, is_owner));
@@ -320,7 +320,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
component = LOG_COMPONENT_AUTH,
subsystem = LOG_SUBSYSTEM_KEYSTONE,
access_key = %MaskedAccessKey(access_key),
"Keystone access key detected without task-local credentials"
"Keystone context missing for access key"
);
if !auth_keystone::is_keystone_enabled() {
@@ -360,7 +360,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key),
reason = "user_not_found",
"Access key validation failed"
"Access key validation rejected"
);
return Err(s3_error!(InvalidAccessKeyId, "check key failed"));
};
@@ -372,7 +372,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key),
reason = "account_disabled",
"Access key validation failed"
"Access key validation rejected"
);
return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled"));
}
@@ -383,7 +383,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key),
reason = "validation_failed",
"Access key validation failed"
"Access key validation rejected"
);
return Err(s3_error!(InvalidRequest, "check key failed"));
}
rustfs/src/init.rs
+216 -28
View File
@@ -36,6 +36,8 @@ const LOG_SUBSYSTEM_KMS: &str = "kms";
const LOG_SUBSYSTEM_BUFFER: &str = "buffer_profile";
const LOG_SUBSYSTEM_AUTOTUNER: &str = "autotuner";
const LOG_SUBSYSTEM_PROTOCOL: &str = "protocol";
const EVENT_PROTOCOL_RUNTIME_STATE: &str = "protocol_runtime_state";
const EVENT_PROTOCOL_SERVER_STATE: &str = "protocol_server_state";
#[instrument]
pub fn print_server_info() {
@@ -666,7 +668,7 @@ where
protocol = protocol_name,
state = "runtime_failed",
error = %e,
"Protocol server state changed"
"Protocol server failed"
);
}
info!(
@@ -676,7 +678,7 @@ where
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = protocol_name,
state = "stopped",
"Protocol server state changed"
"Protocol server stopped"
);
});
@@ -786,7 +788,15 @@ pub async fn init_ftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::er
// Check if FTP is enabled
let ftp_enable = rustfs_utils::get_env_bool(ENV_FTP_ENABLE, false);
if !ftp_enable {
debug!("FTP system is disabled");
debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftp",
state = "disabled",
"Protocol runtime disabled"
);
return Ok(None);
}
@@ -817,12 +827,21 @@ pub async fn init_ftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::er
let fs = crate::storage::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs);
let server: FtpsServer<ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?;
let bind_addr = server.config().bind_addr;
let passive_ports = server.config().passive_ports.clone();
// Log server configuration
info!(
"FTP server configured on {} with passive ports {:?}",
server.config().bind_addr,
server.config().passive_ports
debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftp",
state = "configured",
bind_addr = %bind_addr,
passive_ports = ?passive_ports,
tls_enabled = false,
"Protocol runtime configured"
);
// Start FTP server in background task with proper shutdown support
@@ -830,12 +849,39 @@ pub async fn init_ftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::er
let task_handle = tokio::spawn(async move {
if let Err(e) = server.start(shutdown_rx).await {
error!("FTP server error: {}", e);
error!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftp",
state = "runtime_failed",
error = %e,
"Protocol server failed"
);
}
info!("FTP server shutdown completed");
info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftp",
state = "stopped",
"Protocol server stopped"
);
});
info!("FTP system initialized successfully");
info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftp",
state = "started",
bind_addr = %bind_addr,
tls_enabled = false,
"Protocol runtime started"
);
Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle)))
}
}
@@ -859,7 +905,15 @@ pub async fn init_ftps_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
// Check if FTPS is enabled
let ftps_enable = rustfs_utils::get_env_bool(ENV_FTPS_ENABLE, false);
if !ftps_enable {
debug!("FTPS system is disabled");
debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftps",
state = "disabled",
"Protocol runtime disabled"
);
return Ok(None);
}
@@ -893,12 +947,22 @@ pub async fn init_ftps_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
let fs = crate::storage::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs);
let server: FtpsServer<ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?;
let bind_addr = server.config().bind_addr;
let passive_ports = server.config().passive_ports.clone();
let tls_enabled = server.config().tls_enabled;
// Log server configuration
info!(
"FTPS server configured on {} with passive ports {:?}",
server.config().bind_addr,
server.config().passive_ports
debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftps",
state = "configured",
bind_addr = %bind_addr,
passive_ports = ?passive_ports,
tls_enabled,
"Protocol runtime configured"
);
// Start FTPS server in background task with proper shutdown support
@@ -906,12 +970,39 @@ pub async fn init_ftps_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
let task_handle = tokio::spawn(async move {
if let Err(e) = server.start(shutdown_rx).await {
error!("FTPS server error: {}", e);
error!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftps",
state = "runtime_failed",
error = %e,
"Protocol server failed"
);
}
info!("FTPS server shutdown completed");
info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftps",
state = "stopped",
"Protocol server stopped"
);
});
info!("FTPS system initialized successfully");
info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftps",
state = "started",
bind_addr = %bind_addr,
tls_enabled,
"Protocol runtime started"
);
Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle)))
}
}
@@ -935,7 +1026,15 @@ pub async fn init_webdav_system() -> Result<Option<ShutdownHandle>, Box<dyn std:
// Check if WebDAV is enabled
let webdav_enable = rustfs_utils::get_env_bool(ENV_WEBDAV_ENABLE, false);
if !webdav_enable {
debug!("WebDAV system is disabled");
debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "webdav",
state = "disabled",
"Protocol runtime disabled"
);
return Ok(None);
}
@@ -966,21 +1065,64 @@ pub async fn init_webdav_system() -> Result<Option<ShutdownHandle>, Box<dyn std:
let fs = crate::storage::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs);
let server: WebDavServer<crate::protocols::ProtocolStorageClient> = WebDavServer::new(config, storage_client).await?;
let bind_addr = server.config().bind_addr;
let tls_enabled = server.config().tls_enabled;
let max_body_size = server.config().max_body_size;
let request_timeout_secs = server.config().request_timeout_secs;
// Log server configuration
info!("WebDAV server configured on {}", server.config().bind_addr);
debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "webdav",
state = "configured",
bind_addr = %bind_addr,
tls_enabled,
max_body_size,
request_timeout_secs,
"Protocol runtime configured"
);
// Start WebDAV server in background task with proper shutdown support
let (shutdown_tx, shutdown_rx) = tokio::sync::broadcast::channel(1);
let task_handle = tokio::spawn(async move {
if let Err(e) = server.start(shutdown_rx).await {
error!("WebDAV server error: {}", e);
error!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "webdav",
state = "runtime_failed",
error = %e,
"Protocol server failed"
);
}
info!("WebDAV server shutdown completed");
info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "webdav",
state = "stopped",
"Protocol server stopped"
);
});
info!("WebDAV system initialized successfully");
info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "webdav",
state = "started",
bind_addr = %bind_addr,
tls_enabled,
"Protocol runtime started"
);
Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle)))
}
}
@@ -1003,7 +1145,15 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
let enabled = rustfs_utils::get_env_bool(ENV_SFTP_ENABLE, false);
if !enabled {
debug!("SFTP system is disabled");
debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "sftp",
state = "disabled",
"Protocol runtime disabled"
);
return Ok(None);
}
@@ -1051,7 +1201,18 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
let server = SftpServer::new(config.clone(), storage_client, host_keys)?;
info!("SFTP server configured on {}", config.bind_addr);
debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "sftp",
state = "configured",
bind_addr = %config.bind_addr,
read_only = config.read_only,
host_key_dir = %config.host_key_dir.display(),
"Protocol runtime configured"
);
// Hook into shutdown support
let (shutdown_tx, shutdown_rx) = tokio::sync::broadcast::channel(1);
@@ -1059,12 +1220,39 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
// Start SFTP server in background task
let task_handle = tokio::spawn(async move {
if let Err(e) = server.start(shutdown_rx).await {
error!("SFTP server error: {}", e);
error!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "sftp",
state = "runtime_failed",
error = %e,
"Protocol server failed"
);
}
info!("SFTP server shutdown completed");
info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "sftp",
state = "stopped",
"Protocol server stopped"
);
});
info!("SFTP system initialized successfully");
info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "sftp",
state = "started",
bind_addr = %config.bind_addr,
read_only = config.read_only,
"Protocol runtime started"
);
Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle)))
}
}
rustfs/src/main.rs
+63 -43
View File
@@ -89,6 +89,7 @@ const EVENT_DEFAULT_CREDENTIALS_DETECTED: &str = "default_credentials_detected";
const EVENT_SERVER_CONFIG_SANITIZED: &str = "server_config_sanitized";
const EVENT_SERVER_STARTING: &str = "server_starting";
const EVENT_SERVER_RUNTIME_FAILED: &str = "server_runtime_failed";
const EVENT_ACTION_CREDENTIALS_INITIALIZED: &str = "action_credentials_initialized";
const EVENT_ACTION_CREDENTIALS_INITIALIZATION_FAILED: &str = "action_credentials_initialization_failed";
const EVENT_OBSERVABILITY_GUARD_SET: &str = "observability_guard_set";
const EVENT_CRYPTO_PROVIDER_STATE: &str = "crypto_provider_state";
@@ -384,7 +385,14 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
// Set up AK and SK
match init_global_action_credentials(Some(config.access_key.clone()), Some(config.secret_key.clone())) {
Ok(_) => {
debug!(target: "rustfs::main::run", "Global action credentials initialized successfully.");
debug!(
target: "rustfs::main::run",
event = EVENT_ACTION_CREDENTIALS_INITIALIZED,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_AUTH,
result = "ok",
"Initialized global action credentials"
);
}
Err(e) => {
let msg = format!("init global action credentials failed: {e:?}");
@@ -419,9 +427,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
.inspect_err(|err| {
error!(
target: "rustfs::main::run",
event = EVENT_STARTUP_STORAGE_STAGE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STORAGE,
stage = "endpoint_parsing",
state = "failed",
error = ?err,
"startup storage stage failed"
"Endpoint parsing failed"
);
})
.map_err(Error::other)?;
@@ -445,9 +457,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
.inspect_err(|err| {
error!(
target: "rustfs::main::run",
event = EVENT_STARTUP_STORAGE_STAGE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STORAGE,
stage = "local_disk_initialization",
state = "failed",
error = ?err,
"startup storage stage failed"
"Local disk initialization failed"
);
})
.map_err(Error::other)?;
@@ -540,9 +556,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
.inspect_err(|err| {
error!(
target: "rustfs::main::run",
event = EVENT_STARTUP_STORAGE_STAGE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STORAGE,
stage = "ecstore_initialization",
state = "failed",
error = ?err,
"startup storage stage failed"
"ECStore initialization failed"
);
})?;
@@ -561,7 +581,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
state = "retrying",
retry_count = retry_count + 1,
error = ?e,
"Startup storage stage failed"
"Global config initialization retry failed"
);
// TODO: check error type
retry_count += 1;
@@ -580,13 +600,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
#[cfg(feature = "ftps")]
let ftp_shutdown_tx = match init_ftp_system().await {
Ok(Some(tx)) => {
info!(
debug!(
event = EVENT_PROTOCOL_SYSTEM_STATE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftp",
state = "initialized",
"Protocol system state changed"
state = "started",
"Protocol runtime started"
);
Some(tx)
}
@@ -597,7 +617,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftp",
state = "disabled",
"Protocol system state changed"
"Protocol runtime disabled"
);
None
}
@@ -609,7 +629,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
protocol = "ftp",
state = "initialization_failed",
error = %e,
"Protocol system state changed"
"Protocol runtime initialization failed"
);
return Err(Error::other(e));
}
@@ -622,13 +642,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
#[cfg(feature = "ftps")]
let ftps_shutdown_tx = match init_ftps_system().await {
Ok(Some(tx)) => {
info!(
debug!(
event = EVENT_PROTOCOL_SYSTEM_STATE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftps",
state = "initialized",
"Protocol system state changed"
state = "started",
"Protocol runtime started"
);
Some(tx)
}
@@ -639,7 +659,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftps",
state = "disabled",
"Protocol system state changed"
"Protocol runtime disabled"
);
None
}
@@ -651,7 +671,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
protocol = "ftps",
state = "initialization_failed",
error = %e,
"Protocol system state changed"
"Protocol runtime initialization failed"
);
return Err(Error::other(e));
}
@@ -664,13 +684,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
#[cfg(feature = "webdav")]
let webdav_shutdown_tx = match init_webdav_system().await {
Ok(Some(tx)) => {
info!(
debug!(
event = EVENT_PROTOCOL_SYSTEM_STATE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "webdav",
state = "initialized",
"Protocol system state changed"
state = "started",
"Protocol runtime started"
);
Some(tx)
}
@@ -681,7 +701,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "webdav",
state = "disabled",
"Protocol system state changed"
"Protocol runtime disabled"
);
None
}
@@ -693,7 +713,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
protocol = "webdav",
state = "initialization_failed",
error = %e,
"Protocol system state changed"
"Protocol runtime initialization failed"
);
return Err(Error::other(e));
}
@@ -706,13 +726,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
#[cfg(feature = "sftp")]
let sftp_shutdown_tx = match init_sftp_system().await {
Ok(Some(tx)) => {
info!(
debug!(
event = EVENT_PROTOCOL_SYSTEM_STATE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "sftp",
state = "initialized",
"Protocol system state changed"
state = "started",
"Protocol runtime started"
);
Some(tx)
}
@@ -723,7 +743,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "sftp",
state = "disabled",
"Protocol system state changed"
"Protocol runtime disabled"
);
None
}
@@ -735,7 +755,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
protocol = "sftp",
state = "initialization_failed",
error = %e,
"Protocol system state changed"
"Protocol runtime initialization failed"
);
return Err(Error::other(e));
}
@@ -758,7 +778,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "started",
"Audit system state changed"
"Audit runtime started"
),
Err(e) => error!(
target: "rustfs::main::run",
@@ -767,7 +787,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "start_failed",
error = %e,
"Audit system state changed"
"Audit runtime failed to start"
),
}
@@ -781,7 +801,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "started",
"Deadlock detector state changed"
"Deadlock detector started"
);
} else {
info!(
@@ -790,7 +810,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "disabled",
"Deadlock detector state changed"
"Deadlock detector disabled"
);
}
@@ -888,7 +908,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP,
enable_scanner = enable_scanner,
enable_heal = enable_heal,
"Configured background services"
"Background services configured"
);
// Scanner depends on the heal channel/manager, so scanner implies heal.
@@ -907,7 +927,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
enable_heal = false,
ahm_state = "skipped",
reason = "disabled",
"Background services configured"
"Background services disabled"
);
}
@@ -935,7 +955,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
server_address = %server_address,
started_at = %jiff::Zoned::now(),
iam_bootstrap = ?iam_bootstrap,
"RustFS server started successfully"
"RustFS server ready"
);
if iam_bootstrap == IamBootstrapDisposition::ReadyInline {
rustfs::server::publish_ready_when_runtime_ready(readiness.as_ref(), Some(state_manager.as_ref())).await?;
@@ -971,7 +991,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP,
state = ?state_manager.current_state(),
result = "stopped",
"Server shutdown state changed"
"RustFS server stopped"
);
Ok(())
}
@@ -1051,7 +1071,7 @@ async fn handle_shutdown(
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
signal = shutdown_signal.log_label(),
"Shutdown signal received in main thread"
"Shutdown signal received"
);
// update the status to stopping first
state_manager.update(ServiceState::Stopping);
@@ -1114,7 +1134,7 @@ async fn handle_shutdown(
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftp",
state = "stopping",
"Protocol system state changed"
"Protocol runtime stopping"
);
protocol_shutdowns.push(ftp_shutdown_tx.shutdown());
}
@@ -1127,7 +1147,7 @@ async fn handle_shutdown(
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftps",
state = "stopping",
"Protocol system state changed"
"Protocol runtime stopping"
);
protocol_shutdowns.push(ftps_shutdown_tx.shutdown());
}
@@ -1141,7 +1161,7 @@ async fn handle_shutdown(
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "webdav",
state = "stopping",
"Protocol system state changed"
"Protocol runtime stopping"
);
protocol_shutdowns.push(webdav_shutdown_tx.shutdown());
}
@@ -1155,7 +1175,7 @@ async fn handle_shutdown(
subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "sftp",
state = "stopping",
"Protocol system state changed"
"Protocol runtime stopping"
);
protocol_shutdowns.push(sftp_shutdown_tx.shutdown());
}
@@ -1178,7 +1198,7 @@ async fn handle_shutdown(
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stopping",
"Audit system state changed"
"Audit runtime stopping"
);
match stop_audit_system().await {
Ok(_) => info!(
@@ -1187,7 +1207,7 @@ async fn handle_shutdown(
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stopped",
"Audit system state changed"
"Audit runtime stopped"
),
Err(e) => error!(
target: "rustfs::main::handle_shutdown",
@@ -1196,7 +1216,7 @@ async fn handle_shutdown(
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stop_failed",
error = %e,
"Audit system state changed"
"Audit runtime failed to stop"
),
}
@@ -1217,7 +1237,7 @@ async fn handle_shutdown(
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stopping",
"Server shutdown state changed"
"RustFS server stopping"
);
if let Some(s3_shutdown_handle) = s3_shutdown_handle {
s3_shutdown_handle.shutdown().await;
@@ -1235,7 +1255,7 @@ async fn handle_shutdown(
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stopped",
"Server shutdown state changed"
"RustFS server stopped"
);
}
rustfs/src/profiling.rs
+201 -50
View File
@@ -16,26 +16,37 @@
mod unsupported_impl {
use std::path::PathBuf;
use std::time::Duration;
use tracing::info;
use tracing::{debug, info};
const LOG_COMPONENT_PROFILING: &str = "profiling";
const LOG_SUBSYSTEM_PLATFORM: &str = "platform";
pub async fn init_from_env() {
let target_env = option_env!("CARGO_CFG_TARGET_ENV").unwrap_or("unknown");
info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PLATFORM,
event = "profiling_runtime_skipped",
reason = "unsupported_platform",
target_os = std::env::consts::OS,
target_env,
target_arch = std::env::consts::ARCH,
"Profiling initialization skipped on unsupported platform"
"Profiling runtime skipped"
);
}
/// Stop all background profiling tasks
pub fn shutdown_profiling() {
let target_env = option_env!("CARGO_CFG_TARGET_ENV").unwrap_or("unknown");
info!(
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PLATFORM,
event = "profiling_shutdown_skipped",
reason = "unsupported_platform",
target_os = std::env::consts::OS,
target_env,
target_arch = std::env::consts::ARCH,
"profiling: shutdown called on unsupported platform (no-op)"
"Profiling shutdown skipped"
);
}
@@ -80,7 +91,10 @@ mod linux_impl {
use tracing::{debug, info, warn};
const LOG_COMPONENT_PROFILING: &str = "profiling";
const LOG_SUBSYSTEM_PROFILING: &str = "profiling";
const LOG_SUBSYSTEM_CPU: &str = "cpu";
const LOG_SUBSYSTEM_MEMORY: &str = "memory";
const LOG_SUBSYSTEM_JEMALLOC: &str = "jemalloc";
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
static CPU_CONT_GUARD: OnceLock<Arc<Mutex<Option<pprof::ProfilerGuard<'static>>>>> = OnceLock::new();
static PROFILING_CANCEL_TOKEN: OnceLock<CancellationToken> = OnceLock::new();
@@ -100,7 +114,7 @@ mod linux_impl {
if let Err(e) = create_dir_all(&p) {
warn!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_RUNTIME,
event = "profiling_output_dir_fallback",
path = %p.display(),
error = %e,
@@ -143,7 +157,7 @@ mod linux_impl {
write_pprof_report_pb(&report, &out)?;
info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_dump_exported",
profile_type = "cpu",
path = %out.display(),
@@ -159,7 +173,7 @@ mod linux_impl {
if let Some(ref guard) = *guard_slot {
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_dump_source",
profile_type = "cpu",
source = "continuous_guard",
@@ -195,7 +209,7 @@ mod linux_impl {
f.write_all(&bytes).map_err(|e| format!("write file failed: {e}"))?;
info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_exported",
profile_type = "memory",
path = %out.display(),
@@ -215,24 +229,69 @@ mod linux_impl {
use tikv_jemalloc_ctl::{config, epoch, stats};
if let Err(e) = epoch::advance() {
warn!("jemalloc epoch advance failed: {e}");
warn!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_epoch_advance_failed",
error = %e,
"Jemalloc profiling state changed"
);
}
match config::malloc_conf::read() {
Ok(conf) => debug!("jemalloc malloc_conf: {}", conf),
Err(e) => debug!("jemalloc read malloc_conf failed: {e}"),
Ok(conf) => debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_malloc_conf",
result = "ok",
malloc_conf = %conf,
"Jemalloc profiling state checked"
),
Err(e) => debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_malloc_conf",
result = "read_failed",
error = %e,
"Jemalloc profiling state checked"
),
}
match std::env::var("MALLOC_CONF") {
Ok(v) => debug!("MALLOC_CONF={}", v),
Err(_) => debug!("MALLOC_CONF is not set"),
Ok(v) => debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_malloc_conf_env",
state = "set",
malloc_conf = %v,
"Jemalloc profiling state checked"
),
Err(_) => debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_malloc_conf_env",
state = "unset",
"Jemalloc profiling state checked"
),
}
if let Some(lock) = jemalloc_pprof::PROF_CTL.as_ref() {
let ctl = lock.lock().await;
info!(activated = ctl.activated(), "jemalloc profiling status");
info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_profiling_status",
activated = ctl.activated(),
"Jemalloc profiling status checked"
);
} else {
info!("jemalloc profiling controller is NOT available");
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_profiling_status",
state = "unavailable",
"Jemalloc profiling status checked"
);
}
let _ = epoch::advance();
@@ -253,7 +312,14 @@ mod linux_impl {
#[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
pub async fn check_jemalloc_profiling() {
debug!("jemalloc profiling status check skipped on unsupported target");
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_profiling_status",
result = "skipped",
reason = "unsupported_target",
"Jemalloc profiling status checked"
);
}
// Internal: start continuous CPU profiling
@@ -263,11 +329,11 @@ mod linux_impl {
if slot.is_some() {
warn!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state",
profile_type = "cpu_continuous",
state = "already_running",
"Profiling state changed"
"CPU profiling already running"
);
return;
}
@@ -280,22 +346,22 @@ mod linux_impl {
*slot = Some(guard);
info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state",
profile_type = "cpu_continuous",
state = "started",
freq_hz,
"Profiling state changed"
"CPU profiling started"
);
}
Err(e) => warn!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state",
profile_type = "cpu_continuous",
state = "start_failed",
error = %e,
"Profiling state changed"
"CPU profiling failed to start"
),
}
}
@@ -304,14 +370,14 @@ mod linux_impl {
async fn start_cpu_periodic(freq_hz: i32, interval: Duration, duration: Duration, token: CancellationToken) {
info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state",
profile_type = "cpu_periodic",
state = "started",
freq_hz,
?interval,
?duration,
"Profiling state changed"
"Periodic CPU profiling started"
);
tokio::spawn(async move {
loop {
@@ -319,11 +385,11 @@ mod linux_impl {
_ = token.cancelled() => {
info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state",
profile_type = "cpu_periodic",
state = "cancelled",
"Profiling state changed"
"Periodic CPU profiling cancelled"
);
break;
}
@@ -337,7 +403,15 @@ mod linux_impl {
let guard = match pprof::ProfilerGuard::new(freq_hz) {
Ok(g) => g,
Err(e) => {
warn!("periodic CPU profiler create failed: {e}");
warn!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_capture_failed",
profile_type = "cpu_periodic",
stage = "create_guard",
error = %e,
"Profiling capture failed"
);
continue;
}
};
@@ -346,11 +420,11 @@ mod linux_impl {
_ = token.cancelled() => {
info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state",
profile_type = "cpu_periodic",
state = "cancelled_during_capture",
"Profiling state changed"
"Periodic CPU profiling cancelled during capture"
);
break;
}
@@ -361,11 +435,20 @@ mod linux_impl {
Ok(report) => {
let out = output_dir().join(format!("cpu_profile_{}.pb", ts()));
if let Err(e) = write_pprof_report_pb(&report, &out) {
warn!("write periodic CPU pprof failed: {e}");
} else {
info!(
warn!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_dump_failed",
profile_type = "cpu_periodic",
stage = "write_dump",
path = %out.display(),
error = %e,
"Periodic CPU dump write failed"
);
} else {
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_dump_exported",
profile_type = "cpu_periodic",
path = %out.display(),
@@ -373,7 +456,15 @@ mod linux_impl {
);
}
}
Err(e) => warn!("periodic CPU report build failed: {e}"),
Err(e) => warn!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_capture_failed",
profile_type = "cpu_periodic",
stage = "build_report",
error = %e,
"Profiling capture failed"
),
}
}
});
@@ -384,12 +475,12 @@ mod linux_impl {
async fn start_memory_periodic(interval: Duration, token: CancellationToken) {
info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_state",
profile_type = "memory_periodic",
state = "started",
?interval,
"Profiling state changed"
"Periodic memory profiling started"
);
tokio::spawn(async move {
loop {
@@ -397,11 +488,11 @@ mod linux_impl {
_ = token.cancelled() => {
info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_state",
profile_type = "memory_periodic",
state = "cancelled",
"Profiling state changed"
"Periodic memory profiling cancelled"
);
break;
}
@@ -409,29 +500,61 @@ mod linux_impl {
}
let Some(lock) = jemalloc_pprof::PROF_CTL.as_ref() else {
debug!("skip memory dump: PROF_CTL not available");
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_skipped",
profile_type = "memory_periodic",
reason = "prof_ctl_unavailable",
"Profiling dump skipped"
);
continue;
};
let mut ctl = lock.lock().await;
if !ctl.activated() {
debug!("skip memory dump: jemalloc profiling not active");
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_skipped",
profile_type = "memory_periodic",
reason = "jemalloc_inactive",
"Profiling dump skipped"
);
continue;
}
let out = output_dir().join(format!("mem_profile_periodic_{}.pb", ts()));
match File::create(&out) {
Err(e) => {
tracing::error!("periodic mem dump create file failed: {}", e);
tracing::error!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_failed",
profile_type = "memory_periodic",
stage = "create_file",
path = %out.display(),
error = %e,
"Periodic memory dump file creation failed"
);
continue;
}
Ok(mut f) => match ctl.dump_pprof() {
Ok(bytes) => {
if let Err(e) = f.write_all(&bytes) {
tracing::error!("periodic mem dump write failed: {}", e);
} else {
info!(
tracing::error!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_failed",
profile_type = "memory_periodic",
stage = "write_dump",
path = %out.display(),
error = %e,
"Periodic memory dump write failed"
);
} else {
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_exported",
profile_type = "memory_periodic",
path = %out.display(),
@@ -439,7 +562,15 @@ mod linux_impl {
);
}
}
Err(e) => tracing::error!("periodic mem dump failed: {}", e),
Err(e) => tracing::error!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_failed",
profile_type = "memory_periodic",
stage = "dump_pprof",
error = %e,
"Periodic memory dump export failed"
),
},
}
}
@@ -448,14 +579,27 @@ mod linux_impl {
#[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
async fn start_memory_periodic(_interval: Duration, _token: CancellationToken) {
debug!("periodic memory profiling skipped on unsupported target");
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_runtime_skipped",
profile_type = "memory_periodic",
reason = "unsupported_target",
"Profiling runtime skipped"
);
}
// Public: unified init entry, avoid duplication/conflict
pub async fn init_from_env() {
let enabled = get_env_bool(ENV_ENABLE_PROFILING, DEFAULT_ENABLE_PROFILING);
if !enabled {
debug!("profiling: disabled by env");
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_RUNTIME,
event = "profiling_runtime_disabled",
reason = "env_flag",
"Profiling runtime disabled"
);
return;
}
@@ -472,7 +616,14 @@ mod linux_impl {
let cpu_duration = Duration::from_secs(get_env_u64(ENV_CPU_DURATION_SECS, DEFAULT_CPU_DURATION_SECS));
match cpu_mode {
CpuMode::Off => debug!("profiling: CPU mode off"),
CpuMode::Off => debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_mode_selected",
profile_type = "cpu",
state = "off",
"Profiling mode selected"
),
CpuMode::Continuous => start_cpu_continuous(cpu_freq).await,
CpuMode::Periodic => start_cpu_periodic(cpu_freq, cpu_interval, cpu_duration, token.clone()).await,
}
rustfs/src/protocols/client.rs
+2 -2
View File
@@ -207,7 +207,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
bucket,
object = %key,
start_pos = ?start_pos,
"Protocol storage client request details"
"Protocol storage client request"
);
let mut builder = GetObjectInput::builder().bucket(bucket.to_string()).key(key.to_string());
@@ -513,7 +513,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
object = %key,
range_start = start_pos,
range_length = length,
"Protocol storage client request details"
"Protocol storage client request"
);
let range = s3s::dto::Range::Int {
scripts/check_logging_guardrails.sh
+166
View File
@@ -6,15 +6,35 @@ cd "$repo_root"
checked_files=(
"rustfs/src/main.rs"
"rustfs/src/init.rs"
"rustfs/src/profiling.rs"
"rustfs/src/startup_iam.rs"
"rustfs/src/auth.rs"
"rustfs/src/protocols/client.rs"
"rustfs/src/admin/router.rs"
"rustfs/src/admin/handlers/table_catalog.rs"
"rustfs/src/admin/handlers/service_account.rs"
"rustfs/src/admin/handlers/kms_dynamic.rs"
"rustfs/src/admin/handlers/site_replication.rs"
"rustfs/src/admin/handlers/group.rs"
"rustfs/src/admin/handlers/quota.rs"
"rustfs/src/admin/handlers/rebalance.rs"
"rustfs/src/admin/handlers/tier.rs"
"crates/audit/src/pipeline.rs"
"crates/audit/src/system.rs"
"crates/audit/src/global.rs"
"crates/notify/src/config_manager.rs"
"crates/notify/src/runtime_facade.rs"
"crates/notify/src/integration.rs"
"crates/notify/src/notifier.rs"
"crates/notify/src/bucket_config_manager.rs"
"crates/notify/src/rule_engine.rs"
"crates/notify/src/global.rs"
"crates/audit/src/registry.rs"
"crates/audit/src/observability.rs"
"crates/targets/src/store.rs"
"crates/targets/src/target/mqtt.rs"
"crates/targets/src/target/webhook.rs"
"crates/ecstore/src/store/peer.rs"
"crates/ecstore/src/store/init.rs"
"crates/ecstore/src/tier/tier.rs"
@@ -100,6 +120,25 @@ forbidden_patterns=(
'info!("Loaded notification config for bucket: {}"'
'info!("Updated notification rules for bucket: {}"'
'info!("Removed all notification rules for bucket: {}"'
'info!(event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,'
'info!("Notification system instance is being dropped"'
'info!("Notification shutdown metric snapshot"'
'info!("Notification system status snapshot"'
'debug!("Audit replay stream skipped"'
'warn!("Dropped queued audit payload"'
'error!("Queued audit payload failed permanently"'
'info!(target_id = %target_id.id, "Webhook target created")'
'info!("Webhook target {} initialized"'
'info!("Webhook sending queued payload to target: {}"'
'debug!("Event saved to store for target: {}"'
'debug!("Event sent from store and deleted for target: {}"'
'info!("Webhook target closed: {}"'
'info!(target_id = %target_id, "MQTT target created")'
'info!(target_id = %self.id, "MQTT target initialized and connected.")'
'info!(target_id = %target_id, "MQTT event loop task started.")'
'info!(target_id = %self.id, "MQTT target close method finished.")'
'debug!("Wrote event to store: {}"'
'debug!("Deleted event from store: {}"'
'info!("Audit configuration reloaded"'
'info!("Audit system started"'
'info!("Audit metrics reset"'
@@ -108,17 +147,117 @@ forbidden_patterns=(
'error!("Server encountered an error and is shutting down: {}"'
'error!("Failed to initialize Keystone authentication: {}"'
'error!("new_global_notification_sys failed {:?}"'
'warn!(access_key = %new_cred.access_key, error = ?err, "site replication add service account hook failed")'
'warn!(access_key = %query.access_key, error = ?err, "site replication delete service account hook failed")'
'debug!("check key failed: {e:?}")'
'debug!("list service account failed: {e:?}")'
'debug!("list sts account failed: {e:?}")'
'error!("license check failed due to unexpected error: {er}")'
'info!("Created KMS key: {}"'
'info!("Successfully deleted KMS key: {}"'
'info!("Cancelled deletion for KMS key: {}"'
'info!("Listed {} KMS keys"'
'info!("Described KMS key: {}"'
'error!("Failed to create KMS key: {}"'
'error!("Failed to list KMS keys: {}"'
'error!("Failed to describe KMS key {}: {}"'
'warn!(access_key = %ak, error = ?err, "site replication create user hook failed")'
'info!("OIDC authorize redirect for provider'
'warn!("OIDC callback received error from IdP:'
'error!("OIDC code exchange failed: {}"'
'warn!("OIDC logout fallback triggered: {}"'
'warn!("get body failed, e: {:?}")'
'warn!("file path is invalid: {}")'
'warn!("bucket metadata not found: {e}")'
'warn!("get bucket metadata failed: {e}")'
'warn!("create bucket failed: {e}")'
'warn!("deserialize config failed: {e}")'
'warn!(policy = %query.name, error = ?err, "site replication policy add hook failed")'
'warn!(policy = %query.name, error = ?err, "site replication policy delete hook failed")'
'warn!(target = %query.user_or_group, error = ?err, "site replication policy mapping hook failed")'
'warn!(access_key = %ak, error = ?err, "site replication delete user hook failed")'
'warn!(target = %target_name, error = ?err, "site replication policy association hook failed")'
'error!("Failed to serialize response: {}"'
'warn!(peer = %peer.endpoint, error = ?err, "site replication peer metainfo fetch failed")'
'warn!(error = ?err, "site replication backfill: failed to list buckets")'
'warn!(bucket = %name, error = ?err, "site replication backfill: versioning setup failed")'
'warn!(bucket = %name, error = ?err, "site replication backfill: targets setup failed")'
'warn!(bucket = %name, error = ?err, "site replication backfill: replication config setup failed")'
'warn!(bucket = %name, error = ?err, "site replication backfill: failed to read bucket metadata, assuming lock_enabled=false")'
'warn!(bucket = %name, error = ?err, "site replication backfill: make-bucket broadcast failed")'
'warn!(bucket = %name, peer = %peer.endpoint, detail = %result.err_detail,'
'warn!(peer = %peer.endpoint, error = %err_detail, "site replication peer remove notification failed")'
'warn!(peer = %peer.endpoint, error = %detail, "site replication service account rotation failed for peer")'
'warn!("handle ListGroups")'
'warn!("handle GetGroup")'
'warn!("handle DeleteGroup")'
'warn!("handle SetGroupStatus")'
'warn!("handle UpdateGroupMembers")'
'warn!("list groups failed, e: {:?}")'
'warn!("get group failed, e: {:?}")'
'warn!("delete group failed, e: {:?}")'
'warn!("site replication group delete hook failed, err: {err}")'
'warn!("enable group failed, e: {:?}")'
'warn!("UpdateGroupMembers args {:?}")'
'warn!("remove group members")'
'warn!("remove group members failed, e: {:?}")'
'warn!("add group members")'
'warn!("add group members failed, e: {:?}")'
'warn!("site replication group membership hook failed, err: {err}")'
'warn!("handle SetBucketQuota")'
'warn!("handle GetBucketQuota")'
'warn!("handle ClearBucketQuota")'
'warn!("handle GetBucketQuotaStats")'
'warn!("handle CheckBucketQuota")'
'info!("Clearing quota for bucket: {}")'
'info!("Successfully cleared quota for bucket: {}")'
'debug!("Checking quota for bucket: {}, operation: {}, size: {}")'
'warn!("handle RebalanceStart")'
'warn!("Rebalance started with id: {}")'
'warn!("RebalanceStart Loading rebalance meta start")'
'warn!("rebalance start propagation failed after local state update: {err}")'
'warn!("RebalanceStart Loading rebalance meta done")'
'warn!("handle RebalanceStatus")'
'warn!("handle RebalanceStop")'
'warn!("handle RebalanceStop save_rebalance_stats done ")'
'warn!("handle RebalanceStop notification_sys load_rebalance_meta")'
'warn!("rebalance stop propagation failed after local state update: {err}")'
'warn!("handle RebalanceStop notification_sys load_rebalance_meta done")'
'warn!("get body failed, e: {:?}")'
'debug!("add tier args {:?}")'
'warn!("parse force failed, e: {:?}")'
'warn!("tier reserved name, args.name: {}")'
'warn!("tier_config_mgr reload failed, e: {:?}")'
'warn!("tier_config_mgr add failed, e: {:?}")'
'debug!("edit tier args {:?}")'
'warn!("tier_config_mgr edit failed, e: {:?}")'
'warn!("tier_config_mgr remove failed, e: {:?}")'
'warn!("tier_config_mgr rand: {}")'
'warn!("tier_config_mgr clear failed, e: {:?}")'
'warn!("tier_config_mgr save failed, e: {:?}")'
'warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
event = "object_lambda_tls_verification_disabled"'
'info!("FTP system initialized successfully"'
'debug!("FTP system is disabled")'
'info!("FTP system disabled"'
'info!("FTP server shutdown completed")'
'error!("Failed to initialize FTP system: {}"'
'info!("FTPS system initialized successfully"'
'debug!("FTPS system is disabled")'
'info!("FTPS system disabled"'
'info!("FTPS server shutdown completed")'
'error!("Failed to initialize FTPS system: {}"'
'info!("WebDAV system initialized successfully"'
'debug!("WebDAV system is disabled")'
'info!("WebDAV system disabled"'
'info!("WebDAV server shutdown completed")'
'error!("Failed to initialize WebDAV system: {}"'
'info!("SFTP system initialized successfully"'
'debug!("SFTP system is disabled")'
'info!("SFTP system disabled"'
'info!("SFTP server shutdown completed")'
'error!("Failed to initialize SFTP system: {}"'
'warn!("prewarm_local_disk_id_map: failed to load disk id for {}: {}"'
'info!("retrying get formats after {:?}"'
@@ -174,6 +313,12 @@ forbidden_patterns=(
'warn!("KMS initialization skipped: {e}")'
'warn!("Audit system: {e}")'
'warn!("notification system: {e}")'
'info!("jemalloc profiling controller is NOT available")'
'warn!("periodic CPU profiler create failed: {e}")'
'warn!("write periodic CPU pprof failed: {e}")'
'warn!("periodic CPU report build failed: {e}")'
'debug!("profiling: disabled by env")'
'debug!("profiling: CPU mode off")'
'info!("Starting HealManager")'
'info!("Stopping HealManager")'
'info!("HealManager started successfully")'
@@ -196,6 +341,27 @@ forbidden_patterns=(
'info!("Heal queue has {} pending requests, {} tasks active")'
'error!("Heal channel processor failed: {}")'
'info!("Heal manager with channel processor initialized successfully")'
'"Heal object workflow started"'
'"Heal object recovery started"'
'"Heal object completed"'
'"Heal object cleanup completed"'
'"Heal metadata workflow started"'
'"Heal metadata completed"'
'"Heal MRF workflow started"'
'"Heal MRF completed"'
'"Heal EC decode workflow started"'
'"Heal EC decode completed"'
'"Heal erasure set workflow started"'
'"Heal erasure set format repair completed"'
'"Heal erasure set completed"'
'"Heal storage repair finished"'
'"Heal storage admin operation finished"'
'"Heal storage request finished"'
'"Erasure set object heal completed"'
'"Erasure set heal resumed from checkpoint"'
'"Scanner disk bucket state updated"'
'"Scanner lifecycle action evaluation started"'
'"Scanner state persisted"'
'"scanner deep heal downgraded to normal during new-object cooldown"'
'"scanner detected folder with excessive direct subfolders"'
'"scan_folder: failed to get size for item {}: {}"'