gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-02 10:18:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): normalize admin telemetry and error messages (#3430)

Browse Source
This commit is contained in:
houseme
2026-06-14 13:27:10 +08:00
committed by GitHub
parent dc82efbab4
commit e8012bd1ba
70 changed files with 4807 additions and 1445 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+123 -116
View File
@@ -35,9 +35,9 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]] [[package]]
name = "aead" name = "aead"
version = "0.6.0-rc.10" version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b657e772794c6b04730ea897b66a058ccd866c16d1967da05eeeecec39043fe" checksum = "ef60ac202874e574ce7a7158cc8bca7313dd344322482e4fadee288bf4a306b8"
dependencies = [ dependencies = [
"crypto-common 0.2.2", "crypto-common 0.2.2",
"inout 0.2.2", "inout 0.2.2",
@@ -277,9 +277,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
[[package]] [[package]]
name = "ar_archive_writer" name = "ar_archive_writer"
version = "0.5.1" version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348"
dependencies = [ dependencies = [
"object 0.37.3", "object 0.37.3",
] ]
@@ -887,9 +887,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-runtime" name = "aws-runtime"
version = "1.7.4" version = "1.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ed8e8c52d2dc2390ad9f15647fe663f71e9780b4262c190fbb823a32721566" checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39"
dependencies = [ dependencies = [
"aws-credential-types", "aws-credential-types",
"aws-sigv4", "aws-sigv4",
@@ -915,9 +915,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-sdk-s3" name = "aws-sdk-s3"
version = "1.135.0" version = "1.136.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f97e3e7e7d86fd26fcdc18bc382da5ca9e8b2ff8d54030d187fd0dac8a236d96" checksum = "fbd03e531a7d981fba45114c5813a7565dd470a1bd3ef1188ca98c98ebdfc668"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"aws-credential-types", "aws-credential-types",
@@ -951,9 +951,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-sdk-sso" name = "aws-sdk-sso"
version = "1.101.0" version = "1.102.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b647baea49ff551960b904f905681e9b4765a6c4ea08631e89dc52d8bd3f5896" checksum = "8c82b3ac19f1431854f7ace3a7531674633e286bfdde21976893bfee36fd493b"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"aws-credential-types", "aws-credential-types",
@@ -976,9 +976,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-sdk-ssooidc" name = "aws-sdk-ssooidc"
version = "1.103.0" version = "1.104.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ae401c65ff288aa7873117fe535cd32b7b1bb0bc43751d28901a1d5f20636b9" checksum = "321000d2b4c5519ee573f73167f612efd7329322d9b26969ad1979f0427f1913"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"aws-credential-types", "aws-credential-types",
@@ -1001,9 +1001,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-sdk-sts" name = "aws-sdk-sts"
version = "1.106.0" version = "1.107.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c80de7bb7d03e9ca8c9fd7b489f20f3948d3f3be91a7953591347d238115408" checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"aws-credential-types", "aws-credential-types",
@@ -1086,9 +1086,9 @@ dependencies = [
[[package]] [[package]]
name = "aws-smithy-eventstream" name = "aws-smithy-eventstream"
version = "0.60.20" version = "0.60.21"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf09d74e5e32f76b8762da505a3cd59303e367a664ca67295387baa8c1d7548" checksum = "78d8391e65fcea47c586a22e1a41f173b38615b112b2c6b7a44e80cec3e6b706"
dependencies = [ dependencies = [
"aws-smithy-types", "aws-smithy-types",
"bytes", "bytes",
@@ -1509,9 +1509,9 @@ dependencies = [
[[package]] [[package]]
name = "block-buffer" name = "block-buffer"
version = "0.12.0" version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
dependencies = [ dependencies = [
"hybrid-array", "hybrid-array",
"zeroize", "zeroize",
@@ -1560,9 +1560,9 @@ dependencies = [
[[package]] [[package]]
name = "bon" name = "bon"
version = "3.9.1" version = "3.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" checksum = "b2f04f6fef12d70d42a77b1433c9e0f065238479a6cefc4f5bab105e9873a3c3"
dependencies = [ dependencies = [
"bon-macros", "bon-macros",
"rustversion", "rustversion",
@@ -1570,9 +1570,9 @@ dependencies = [
[[package]] [[package]]
name = "bon-macros" name = "bon-macros"
version = "3.9.1" version = "3.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" checksum = "7d0bd4c2f75335ad98052a37efb54f428b492f64340257143b3429c8a508fa7b"
dependencies = [ dependencies = [
"darling 0.23.0", "darling 0.23.0",
"ident_case", "ident_case",
@@ -1661,9 +1661,9 @@ dependencies = [
[[package]] [[package]]
name = "bytesize" name = "bytesize"
version = "2.3.1" version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" checksum = "49e78e506b9d7633710dab98996f22f95f3d0f488e8f1aa162830556ed9fc14d"
[[package]] [[package]]
name = "bytestring" name = "bytestring"
@@ -1742,9 +1742,9 @@ dependencies = [
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.63" version = "1.2.64"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f"
dependencies = [ dependencies = [
"find-msvc-tools", "find-msvc-tools",
"jobserver", "jobserver",
@@ -1856,7 +1856,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c"
dependencies = [ dependencies = [
"block-buffer 0.12.0", "block-buffer 0.12.1",
"crypto-common 0.2.2", "crypto-common 0.2.2",
"inout 0.2.2", "inout 0.2.2",
"zeroize", "zeroize",
@@ -2348,12 +2348,11 @@ dependencies = [
[[package]] [[package]]
name = "crypto-primes" name = "crypto-primes"
version = "0.7.0" version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21f41f23de7d24cdbda7f0c4d9c0351f99a4ceb258ef30e5c1927af8987ffe5a" checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f"
dependencies = [ dependencies = [
"crypto-bigint 0.7.3", "crypto-bigint 0.7.3",
"libm",
"rand_core 0.10.1", "rand_core 0.10.1",
] ]
@@ -3380,7 +3379,6 @@ version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
dependencies = [ dependencies = [
"powerfmt",
"serde_core", "serde_core",
] ]
@@ -3551,7 +3549,7 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
dependencies = [ dependencies = [
"block-buffer 0.12.0", "block-buffer 0.12.1",
"const-oid 0.10.2", "const-oid 0.10.2",
"crypto-common 0.2.2", "crypto-common 0.2.2",
"ctutils", "ctutils",
@@ -3576,7 +3574,7 @@ dependencies = [
"libc", "libc",
"option-ext", "option-ext",
"redox_users", "redox_users",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -3912,7 +3910,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [ dependencies = [
"libc", "libc",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -4294,11 +4292,10 @@ dependencies = [
[[package]] [[package]]
name = "getset" name = "getset"
version = "0.1.6" version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912" checksum = "6cf442baaabe4213ce7d1239afc26c039180b6456da2cededa316ae2c8a77a77"
dependencies = [ dependencies = [
"proc-macro-error2",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.117", "syn 2.0.117",
@@ -5345,7 +5342,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [ dependencies = [
"hermit-abi", "hermit-abi",
"libc", "libc",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -5512,13 +5509,12 @@ dependencies = [
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.99" version = "0.3.102"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"futures-util", "futures-util",
"once_cell",
"wasm-bindgen", "wasm-bindgen",
] ]
@@ -5978,6 +5974,29 @@ dependencies = [
"zerocopy-derive", "zerocopy-derive",
] ]
[[package]]
name = "manyhow"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587"
dependencies = [
"manyhow-macros",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "manyhow-macros"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495"
dependencies = [
"proc-macro-utils",
"proc-macro2",
"quote",
]
[[package]] [[package]]
name = "mappings" name = "mappings"
version = "0.7.2" version = "0.7.2"
@@ -6040,9 +6059,9 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.8.1" version = "2.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
[[package]] [[package]]
name = "memmap2" name = "memmap2"
@@ -6104,9 +6123,9 @@ dependencies = [
[[package]] [[package]]
name = "metrique" name = "metrique"
version = "0.1.25" version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2638fb6c2325d3e35c7b0d1ce49cf8ce74b89b7439d4dc5ce1c9fff0f7fa18a1" checksum = "ffdfd0c9954a916c5e1db230cc7593569858afec7744cebfd0fbef45de7b07c7"
dependencies = [ dependencies = [
"itoa", "itoa",
"jiff", "jiff",
@@ -6124,9 +6143,9 @@ dependencies = [
[[package]] [[package]]
name = "metrique-core" name = "metrique-core"
version = "0.1.20" version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16b99ea3484ab9afe6337413bacc842d516e65623a476699f9873803028e6a9e" checksum = "f56ef332b47a4d2dc14209bf9ce4334682dd2fec43a4b84e5ed063bf2f4ea29a"
dependencies = [ dependencies = [
"itertools 0.14.0", "itertools 0.14.0",
"metrique-writer-core", "metrique-writer-core",
@@ -6134,9 +6153,9 @@ dependencies = [
[[package]] [[package]]
name = "metrique-macro" name = "metrique-macro"
version = "0.1.16" version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a02c5cb6e55c1ec5eb3ef2d10d137805547c486ad54b587d42d0a49713e6ef9" checksum = "403c61cd847b11680ae6cf1feb384cbe06e412e4dc221f1e16ccf48038d190bb"
dependencies = [ dependencies = [
"Inflector", "Inflector",
"darling 0.23.0", "darling 0.23.0",
@@ -6147,9 +6166,9 @@ dependencies = [
[[package]] [[package]]
name = "metrique-service-metrics" name = "metrique-service-metrics"
version = "0.1.21" version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb71714973a9ba53d609a577311aae4d8beb7df37869b5a92cb777a3f4c905b8" checksum = "93045b7f5c8b998946b715de07523b1c6d81932512c5c27590bf73328aaf3832"
dependencies = [ dependencies = [
"metrique-writer", "metrique-writer",
] ]
@@ -6162,9 +6181,9 @@ checksum = "d607939211e4eaaa8cd35394fa5e57faffb7390d0ac513b39992edcaf3cc526c"
[[package]] [[package]]
name = "metrique-writer" name = "metrique-writer"
version = "0.1.22" version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04f770fe8a45a7c2f285c7ce81b9364e0e5604cbb7daf693fef1f624bf8f0bf8" checksum = "ae9e23184d81616fcdfcdc5768e14c2bca765096094d08e7895259fe658378e4"
dependencies = [ dependencies = [
"ahash 0.8.12", "ahash 0.8.12",
"crossbeam-queue", "crossbeam-queue",
@@ -6183,9 +6202,9 @@ dependencies = [
[[package]] [[package]]
name = "metrique-writer-core" name = "metrique-writer-core"
version = "0.1.16" version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a0b67a72a8849987122ca8ceacaf81d47eaba36cd673767fc3742a086dd04ac" checksum = "23421d85733c0182613b32f32094080e07cd5e98be5893f1cd3baf7eed16c4d0"
dependencies = [ dependencies = [
"derive-where", "derive-where",
"itertools 0.14.0", "itertools 0.14.0",
@@ -6337,15 +6356,15 @@ checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b"
[[package]] [[package]]
name = "mysql-common-derive" name = "mysql-common-derive"
version = "0.32.1" version = "0.32.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66f62cad7623a9cb6f8f64037f0c4f69c8db8e82914334a83c9788201c2c1bfa" checksum = "a4db8a44120571277accfaa3f3d91e7d3989d601d817c2fc01a9391b86135666"
dependencies = [ dependencies = [
"darling 0.20.11", "darling 0.23.0",
"heck", "heck",
"manyhow",
"num-bigint", "num-bigint",
"proc-macro-crate", "proc-macro-crate",
"proc-macro-error2",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.117", "syn 2.0.117",
@@ -6571,7 +6590,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [ dependencies = [
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -6743,7 +6762,7 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [ dependencies = [
"base64 0.21.7", "base64 0.22.1",
"chrono", "chrono",
"getrandom 0.2.17", "getrandom 0.2.17",
"http 1.4.2", "http 1.4.2",
@@ -7798,9 +7817,9 @@ dependencies = [
[[package]] [[package]]
name = "primefield" name = "primefield"
version = "0.14.0-rc.10" version = "0.14.0-rc.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f845ec3240cd5ed5e1e31cf3ff633a5bf47c698dc4092ba9e767415b3d393406" checksum = "b1d7e42f46a29abc16fb621a3466ee453358ebaae48a9e515f287e0af052ed8f"
dependencies = [ dependencies = [
"crypto-bigint 0.7.3", "crypto-bigint 0.7.3",
"crypto-common 0.2.2", "crypto-common 0.2.2",
@@ -7838,25 +7857,14 @@ dependencies = [
] ]
[[package]] [[package]]
name = "proc-macro-error-attr2" name = "proc-macro-utils"
version = "2.0.0" version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
] "smallvec",
[[package]]
name = "proc-macro-error2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
dependencies = [
"proc-macro-error-attr2",
"proc-macro2",
"quote",
"syn 2.0.117",
] ]
[[package]] [[package]]
@@ -8480,9 +8488,9 @@ dependencies = [
[[package]] [[package]]
name = "redis" name = "redis"
version = "1.2.2" version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a12e6b5f4d8ef33944e833e2b1859ad478deab6e431d7337b30ee2efe21f7543" checksum = "f9fd510128eda94d1d49b9f81487744d5c451422431cce41238fe2853d29f4cc"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"arcstr", "arcstr",
@@ -10216,7 +10224,7 @@ dependencies = [
"errno", "errno",
"libc", "libc",
"linux-raw-sys 0.12.1", "linux-raw-sys 0.12.1",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -10290,7 +10298,7 @@ dependencies = [
"security-framework", "security-framework",
"security-framework-sys", "security-framework-sys",
"webpki-root-certs", "webpki-root-certs",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -11177,9 +11185,9 @@ dependencies = [
[[package]] [[package]]
name = "starshard" name = "starshard"
version = "2.2.0" version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "472e5a677707be0fbe7bd851ca25ed7e7757e83dc07a05447cdb6fbdd27db5e2" checksum = "4dafd0cbefb050fa7a7f95ac229f7978ed4c99866a1a5b62e34a154b52d73d77"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"hashbrown 0.17.1", "hashbrown 0.17.1",
@@ -11441,10 +11449,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [ dependencies = [
"fastrand", "fastrand",
"getrandom 0.3.4", "getrandom 0.4.2",
"once_cell", "once_cell",
"rustix 1.1.4", "rustix 1.1.4",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -11582,12 +11590,11 @@ dependencies = [
[[package]] [[package]]
name = "time" name = "time"
version = "0.3.47" version = "0.3.49"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469"
dependencies = [ dependencies = [
"deranged", "deranged",
"itoa",
"js-sys", "js-sys",
"libc", "libc",
"num-conv", "num-conv",
@@ -11600,15 +11607,15 @@ dependencies = [
[[package]] [[package]]
name = "time-core" name = "time-core"
version = "0.1.8" version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
[[package]] [[package]]
name = "time-macros" name = "time-macros"
version = "0.2.27" version = "0.2.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d"
dependencies = [ dependencies = [
"num-conv", "num-conv",
"time-core", "time-core",
@@ -12372,9 +12379,9 @@ dependencies = [
[[package]] [[package]]
name = "wasip2" name = "wasip2"
version = "1.0.3+wasi-0.2.9" version = "1.0.4+wasi-0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
dependencies = [ dependencies = [
"wit-bindgen 0.57.1", "wit-bindgen 0.57.1",
] ]
@@ -12399,9 +12406,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen" name = "wasm-bindgen"
version = "0.2.122" version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"once_cell", "once_cell",
@@ -12412,9 +12419,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-futures" name = "wasm-bindgen-futures"
version = "0.4.72" version = "0.4.75"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280"
dependencies = [ dependencies = [
"js-sys", "js-sys",
"wasm-bindgen", "wasm-bindgen",
@@ -12422,9 +12429,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro" name = "wasm-bindgen-macro"
version = "0.2.122" version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d"
dependencies = [ dependencies = [
"quote", "quote",
"wasm-bindgen-macro-support", "wasm-bindgen-macro-support",
@@ -12432,9 +12439,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro-support" name = "wasm-bindgen-macro-support"
version = "0.2.122" version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd"
dependencies = [ dependencies = [
"bumpalo", "bumpalo",
"proc-macro2", "proc-macro2",
@@ -12445,9 +12452,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-shared" name = "wasm-bindgen-shared"
version = "0.2.122" version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f"
dependencies = [ dependencies = [
"unicode-ident", "unicode-ident",
] ]
@@ -12501,9 +12508,9 @@ dependencies = [
[[package]] [[package]]
name = "web-sys" name = "web-sys"
version = "0.3.99" version = "0.3.102"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d"
dependencies = [ dependencies = [
"js-sys", "js-sys",
"wasm-bindgen", "wasm-bindgen",
@@ -12608,7 +12615,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [ dependencies = [
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -13067,18 +13074,18 @@ dependencies = [
[[package]] [[package]]
name = "zerocopy" name = "zerocopy"
version = "0.8.50" version = "0.8.52"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
dependencies = [ dependencies = [
"zerocopy-derive", "zerocopy-derive",
] ]
[[package]] [[package]]
name = "zerocopy-derive" name = "zerocopy-derive"
version = "0.8.50" version = "0.8.52"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@@ -13108,18 +13115,18 @@ dependencies = [
[[package]] [[package]]
name = "zeroize" name = "zeroize"
version = "1.8.2" version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
dependencies = [ dependencies = [
"zeroize_derive", "zeroize_derive",
] ]
[[package]] [[package]]
name = "zeroize_derive" name = "zeroize_derive"
version = "1.4.3" version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
Cargo.toml
+6 -6
View File
@@ -159,7 +159,7 @@ tower-http = { version = "0.6.11", features = ["cors"] }
# Serialization and Data Formats # Serialization and Data Formats
bytes = { version = "1.11.1", features = ["serde"] } bytes = { version = "1.11.1", features = ["serde"] }
bytesize = "2.3.1" bytesize = "2.4.0"
byteorder = "1.5.0" byteorder = "1.5.0"
flatbuffers = "25.12.19" flatbuffers = "25.12.19"
form_urlencoded = "1.2.2" form_urlencoded = "1.2.2"
@@ -188,13 +188,13 @@ rustls-pki-types = "1.14.1"
sha1 = "0.11.0" sha1 = "0.11.0"
sha2 = "0.11.0" sha2 = "0.11.0"
subtle = "2.6" subtle = "2.6"
zeroize = { version = "1.8.2", features = ["derive"] } zeroize = { version = "1.9.0", features = ["derive"] }
# Time and Date # Time and Date
chrono = { version = "0.4.45", features = ["serde"] } chrono = { version = "0.4.45", features = ["serde"] }
humantime = "2.3.0" humantime = "2.3.0"
jiff = { version = "0.2.28", features = ["serde"] } jiff = { version = "0.2.28", features = ["serde"] }
time = { version = "=0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] } time = { version = "0.3.49", features = ["std", "parsing", "formatting", "macros", "serde"] }
# Database # Database
deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] } deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] }
@@ -209,7 +209,7 @@ atoi = "3.0.0"
atomic_enum = "0.3.0" atomic_enum = "0.3.0"
aws-config = { version = "1.8.18" } aws-config = { version = "1.8.18" }
aws-credential-types = { version = "1.2.14" } aws-credential-types = { version = "1.2.14" }
aws-sdk-s3 = { version = "1.135.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } aws-sdk-s3 = { version = "1.136.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-smithy-http-client = { version = "1.1.13", default-features = false, features = ["default-client", "rustls-aws-lc"] } aws-smithy-http-client = { version = "1.1.13", default-features = false, features = ["default-client", "rustls-aws-lc"] }
aws-smithy-types = { version = "1.5.0" } aws-smithy-types = { version = "1.5.0" }
base64 = "0.22.1" base64 = "0.22.1"
@@ -264,7 +264,7 @@ reed-solomon-erasure = { version = "6.0", default-features = false, features = [
reed-solomon-simd = "3.1.0" reed-solomon-simd = "3.1.0"
regex = { version = "1.12.4" } regex = { version = "1.12.4" }
rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket"] } rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket"] }
redis = { version = "1.2.2", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] } redis = { version = "1.2.3", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
rustix = { version = "1.1.4", features = ["fs"] } rustix = { version = "1.1.4", features = ["fs"] }
rust-embed = { version = "8.11.0" } rust-embed = { version = "8.11.0" }
rustc-hash = { version = "2.1.2" } rustc-hash = { version = "2.1.2" }
@@ -276,7 +276,7 @@ smallvec = { version = "1.15.2", features = ["serde"] }
smartstring = "1.0.1" smartstring = "1.0.1"
snafu = "0.9.1" snafu = "0.9.1"
snap = "1.1.1" snap = "1.1.1"
starshard = { version = "2.2.0", features = ["rayon", "async", "serde"] } starshard = { version = "2.2.1", features = ["rayon", "async", "serde"] }
strum = { version = "0.28.0", features = ["derive"] } strum = { version = "0.28.0", features = ["derive"] }
sysinfo = "0.39.3" sysinfo = "0.39.3"
temp-env = "0.3.6" temp-env = "0.3.6"
crates/audit/src/observability.rs
+3 -3
View File
@@ -160,7 +160,7 @@ impl AuditMetrics {
component = LOG_COMPONENT_AUDIT, component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_OBSERVABILITY, subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
state = "config_reloaded", state = "config_reloaded",
"Audit observability state updated" "audit observability state"
); );
} }
@@ -173,7 +173,7 @@ impl AuditMetrics {
component = LOG_COMPONENT_AUDIT, component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_OBSERVABILITY, subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
state = "system_started", state = "system_started",
"Audit observability state updated" "audit observability state"
); );
} }
@@ -249,7 +249,7 @@ impl AuditMetrics {
component = LOG_COMPONENT_AUDIT, component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_OBSERVABILITY, subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
state = "metrics_reset", state = "metrics_reset",
"Audit observability state updated" "audit observability state"
); );
} }
crates/audit/src/pipeline.rs
+13 -13
View File
@@ -266,7 +266,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE, subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id, target_id = %target_id,
state = "enabled", state = "enabled",
"Changed audit target state" "audit target state"
); );
Ok(()) Ok(())
} else { } else {
@@ -283,7 +283,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE, subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id, target_id = %target_id,
state = "disabled", state = "disabled",
"Changed audit target state" "audit target state"
); );
Ok(()) Ok(())
} else { } else {
@@ -300,7 +300,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE, subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id, target_id = %target_id,
state = "removed", state = "removed",
"Changed audit target state" "audit target state"
); );
Ok(()) Ok(())
} else { } else {
@@ -323,7 +323,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE, subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id, target_id = %target_id,
state = "upserted", state = "upserted",
"Changed audit target state" "audit target state"
); );
Ok(()) Ok(())
} }
@@ -349,7 +349,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE, subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(), target_id = %target.id(),
replay_key = %key, replay_key = %key,
"Delivered queued audit event" "audit replay delivery"
); );
observability::record_target_success(); observability::record_target_success();
} }
@@ -361,7 +361,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE, subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(), target_id = %target.id(),
reason = "not_connected", reason = "not_connected",
"Retrying queued audit event delivery" "audit replay delivery"
); );
} }
rustfs_targets::TargetError::Timeout(_) => { rustfs_targets::TargetError::Timeout(_) => {
@@ -371,7 +371,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE, subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(), target_id = %target.id(),
reason = "timeout", reason = "timeout",
"Retrying queued audit event delivery" "audit replay delivery"
); );
} }
_ => {} _ => {}
@@ -383,7 +383,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE, subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(), target_id = %target.id(),
reason = %reason, reason = %reason,
"Dropped queued audit payload" "audit replay delivery"
); );
observability::record_target_failure(); observability::record_target_failure();
} }
@@ -395,7 +395,7 @@ impl AuditRuntimeFacade {
target_id = %target.id(), target_id = %target.id(),
error = %error, error = %error,
reason = "permanent_failure", reason = "permanent_failure",
"Queued audit payload failed permanently" "audit replay delivery"
); );
target.record_final_failure(); target.record_final_failure();
observability::record_target_failure(); observability::record_target_failure();
@@ -408,7 +408,7 @@ impl AuditRuntimeFacade {
target_id = %target.id(), target_id = %target.id(),
replay_key = %key, replay_key = %key,
reason = "retry_exhausted", reason = "retry_exhausted",
"Dropped queued audit payload after retry exhaustion" "audit replay delivery"
); );
target.record_final_failure(); target.record_final_failure();
observability::record_target_failure(); observability::record_target_failure();
@@ -422,7 +422,7 @@ impl AuditRuntimeFacade {
replay_key = %key, replay_key = %key,
error = %error, error = %error,
reason = "unreadable_entry", reason = "unreadable_entry",
"Skipped unreadable audit store entry" "audit replay delivery"
); );
} }
} }
@@ -436,7 +436,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE, subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id, target_id = %target_id,
replay_enabled = true, replay_enabled = true,
"Audit replay stream started" "audit replay stream"
); );
} else { } else {
debug!( debug!(
@@ -446,7 +446,7 @@ impl AuditRuntimeFacade {
target_id = %target_id, target_id = %target_id,
replay_enabled = false, replay_enabled = false,
reason = "no_store_configured", reason = "no_store_configured",
"Audit replay stream skipped" "audit replay stream"
); );
} }
}), }),
crates/audit/src/registry.rs
+3 -3
View File
@@ -198,7 +198,7 @@ impl AuditRegistry {
target_type = %target_type, target_type = %target_type,
target_id = %target_id, target_id = %target_id,
registry_key = %key, registry_key = %key,
"Created audit target registry key" "audit target registry state"
); );
key.to_string() key.to_string()
} }
@@ -221,7 +221,7 @@ impl AuditRegistry {
target_type = %target_type, target_type = %target_type,
target_id = %target_id, target_id = %target_id,
state = "enabled", state = "enabled",
"Audit target registry state changed" "audit target registry state"
); );
Ok(()) Ok(())
} else { } else {
@@ -250,7 +250,7 @@ impl AuditRegistry {
target_type = %target_type, target_type = %target_type,
target_id = %target_id, target_id = %target_id,
state = "disabled", state = "disabled",
"Audit target registry state changed" "audit target registry state"
); );
Ok(()) Ok(())
} else { } else {
crates/audit/src/system.rs
+8 -8
View File
@@ -120,7 +120,7 @@ impl AuditSystem {
subsystem = LOG_SUBSYSTEM_SYSTEM, subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "targets_created", state = "targets_created",
target_count = targets.len(), target_count = targets.len(),
"Created audit targets" "audit system state"
); );
let activation = self.runtime_facade().activate_targets_with_replay(targets).await; let activation = self.runtime_facade().activate_targets_with_replay(targets).await;
@@ -159,7 +159,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT, component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM, subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "starting", state = "starting",
"Starting audit system" "audit system state"
); );
// Record system start // Record system start
@@ -268,7 +268,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT, component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM, subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "stopping", state = "stopping",
"Stopping audit system" "audit system state"
); );
// Stop all stream tasks first // Stop all stream tasks first
@@ -441,7 +441,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT, component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM, subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "reloading", state = "reloading",
"Reloading audit configuration" "audit config reload"
); );
observability::record_config_reload(); observability::record_config_reload();
@@ -465,7 +465,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT, component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM, subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "reloaded", state = "reloaded",
"Reloaded audit configuration" "audit config reload"
); );
Ok(()) Ok(())
} }
@@ -513,7 +513,7 @@ fn info_audit_state(state: &str, reason: Option<&str>, target_count: Option<usiz
state, state,
reason = reason.unwrap_or_default(), reason = reason.unwrap_or_default(),
target_count = target_count.unwrap_or_default(), target_count = target_count.unwrap_or_default(),
"Changed audit system state" "audit system state"
); );
} }
@@ -526,7 +526,7 @@ fn debug_audit_state(state: &str, reason: Option<&str>, error: Option<&str>, tar
reason = reason.unwrap_or_default(), reason = reason.unwrap_or_default(),
error = error.unwrap_or_default(), error = error.unwrap_or_default(),
target_count, target_count,
"Observed audit system state" "audit system state"
); );
} }
@@ -537,7 +537,7 @@ fn warn_audit_state(state: &str, reason: Option<&str>) {
subsystem = LOG_SUBSYSTEM_SYSTEM, subsystem = LOG_SUBSYSTEM_SYSTEM,
state, state,
reason = reason.unwrap_or_default(), reason = reason.unwrap_or_default(),
"Audit system state transition skipped" "audit system state"
); );
} }
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+9 -9
View File
@@ -77,7 +77,7 @@ use tokio::sync::mpsc::{Receiver, Sender};
use tokio::sync::{RwLock, mpsc}; use tokio::sync::{RwLock, mpsc};
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn}; use tracing::{debug, error, warn};
use uuid::Uuid; use uuid::Uuid;
use xxhash_rust::xxh64; use xxhash_rust::xxh64;
@@ -762,13 +762,13 @@ impl TransitionState {
"Transition compensation backfill failed" "Transition compensation backfill failed"
); );
} else { } else {
info!( debug!(
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION, event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE, component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE, subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %bucket, bucket = %bucket,
state = "completed", state = "completed",
"Completed transition compensation backfill" "Transition compensation completed"
); );
} }
@@ -958,7 +958,7 @@ impl TransitionState {
pub async fn init(api: Arc<ECStore>) { pub async fn init(api: Arc<ECStore>) {
let (configured, absolute_max, n) = resolve_transition_worker_count(); let (configured, absolute_max, n) = resolve_transition_worker_count();
info!( debug!(
event = EVENT_LIFECYCLE_WORKER_STATE, event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE, component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE, subsystem = LOG_SUBSYSTEM_LIFECYCLE,
@@ -968,7 +968,7 @@ impl TransitionState {
transition_queue_capacity = GLOBAL_TransitionState.transition_queue_capacity, transition_queue_capacity = GLOBAL_TransitionState.transition_queue_capacity,
transition_queue_send_timeout_ms = GLOBAL_TransitionState.transition_queue_send_timeout.as_millis() as u64, transition_queue_send_timeout_ms = GLOBAL_TransitionState.transition_queue_send_timeout.as_millis() as u64,
state = "configured", state = "configured",
"Lifecycle worker state resolved" "Lifecycle worker configuration resolved"
); );
//let mut transition_state = GLOBAL_TransitionState.write().await; //let mut transition_state = GLOBAL_TransitionState.write().await;
@@ -1158,7 +1158,7 @@ impl TransitionState {
GLOBAL_TransitionState.num_workers.store(current_workers, Ordering::SeqCst); GLOBAL_TransitionState.num_workers.store(current_workers, Ordering::SeqCst);
GLOBAL_TransitionState.record_scanner_transition_state(); GLOBAL_TransitionState.record_scanner_transition_state();
info!( debug!(
event = EVENT_LIFECYCLE_WORKER_STATE, event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE, component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE, subsystem = LOG_SUBSYSTEM_LIFECYCLE,
@@ -1169,7 +1169,7 @@ impl TransitionState {
current_transition_workers = current_workers, current_transition_workers = current_workers,
pruned_finished_transition_workers = pruned_finished_workers, pruned_finished_transition_workers = pruned_finished_workers,
state = "resized", state = "resized",
"Lifecycle worker state updated" "Lifecycle worker pool resized"
); );
} }
} }
@@ -2008,14 +2008,14 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
Arc::new(move || { Arc::new(move || {
let tier = tier.clone(); let tier = tier.clone();
Box::pin(async move { Box::pin(async move {
info!( debug!(
event = EVENT_LIFECYCLE_TIER_AUDIT, event = EVENT_LIFECYCLE_TIER_AUDIT,
component = LOG_COMPONENT_ECSTORE, component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE, subsystem = LOG_SUBSYSTEM_LIFECYCLE,
tier = %tier, tier = %tier,
bytes = bytes, bytes = bytes,
state = "transition_completed", state = "transition_completed",
"Lifecycle tier transition audit completed" "Lifecycle tier transition recorded"
); );
}) })
}) })
crates/ecstore/src/cache_value/metacache_set.rs
+87 -11
View File
@@ -29,7 +29,11 @@ use tokio::io::AsyncRead;
use tokio::spawn; use tokio::spawn;
use tokio::time::timeout; use tokio::time::timeout;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::{error, info, warn}; use tracing::{error, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
const EVENT_METACACHE_LISTING: &str = "metacache_listing";
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>; pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
pub type PartialFn = pub type PartialFn =
@@ -114,6 +118,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
return Err(DiskError::ErasureReadQuorum); return Err(DiskError::ErasureReadQuorum);
} }
let log_bucket = opts.bucket.clone();
let log_path = opts.path.clone();
let mut jobs: Vec<tokio::task::JoinHandle<std::result::Result<(), DiskError>>> = Vec::new(); let mut jobs: Vec<tokio::task::JoinHandle<std::result::Result<(), DiskError>>> = Vec::new();
let mut readers = Vec::with_capacity(opts.disks.len()); let mut readers = Vec::with_capacity(opts.disks.len());
let fds = opts.fallback_disks.iter().flatten().cloned().collect::<VecDeque<_>>(); let fds = opts.fallback_disks.iter().flatten().cloned().collect::<VecDeque<_>>();
@@ -180,7 +187,17 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
match disk.walk_dir(wakl_opts, &mut wr).await { match disk.walk_dir(wakl_opts, &mut wr).await {
Ok(_res) => {} Ok(_res) => {}
Err(err) => { Err(err) => {
info!("walk dir err {:?}", &err); warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "walk_dir_failed",
error = ?err,
"Metacache walk_dir failed"
);
last_err = Some(err); last_err = Some(err);
need_fallback = true; need_fallback = true;
} }
@@ -205,7 +222,16 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
} }
let Some(disk) = disk_op else { let Some(disk) = disk_op else {
warn!("list_path_raw: fallback disk is none"); warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_disk_missing",
"Metacache fallback disk missing"
);
let err = last_err.unwrap_or(DiskError::DiskNotFound); let err = last_err.unwrap_or(DiskError::DiskNotFound);
record_producer_error(&producer_errs_clone, disk_idx, &err); record_producer_error(&producer_errs_clone, disk_idx, &err);
return Err(err); return Err(err);
@@ -234,7 +260,17 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
last_err = None; last_err = None;
} }
Err(err) => { Err(err) => {
error!("walk dir2 err {:?}", &err); error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_walk_dir_failed",
error = ?err,
"Metacache fallback walk_dir failed"
);
last_err = Some(err); last_err = Some(err);
} }
} }
@@ -355,11 +391,15 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
) )
.increment(1); .increment(1);
warn!( warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
drive = %endpoint, drive = %endpoint,
bucket = %opts.bucket, bucket = %opts.bucket,
path = %opts.path, path = %opts.path,
timeout_ms = peek_timeout.as_millis(), timeout_ms = peek_timeout.as_millis(),
"list_path_raw reader peek timed out; excluding drive from current merge" state = "peek_timed_out",
"Metacache reader peek timed out"
); );
let (detached_rd, write_half) = tokio::io::duplex(1); let (detached_rd, write_half) = tokio::io::duplex(1);
drop(write_half); drop(write_half);
@@ -439,8 +479,14 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
}); });
error!( error!(
"list_path_raw: has_err > 0 && has_err > opts.disks.len() - opts.min_disks break, err: {:?}", event = EVENT_METACACHE_LISTING,
&combined_err.join(", ") component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts.bucket,
path = %opts.path,
state = "quorum_failed",
error = %combined_err.join(", "),
"Metacache listing quorum failed"
); );
return Err(DiskError::other(combined_err.join(", "))); return Err(DiskError::other(combined_err.join(", ")));
} }
@@ -493,7 +539,16 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
}); });
if let Err(err) = revjob.await.map_err(std::io::Error::other)? { if let Err(err) = revjob.await.map_err(std::io::Error::other)? {
error!("list_path_raw: revjob err {:?}", err); error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %log_bucket,
path = %log_path,
state = "merge_job_failed",
error = ?err,
"Metacache merge job failed"
);
cancel_rx.cancel(); cancel_rx.cancel();
for job in jobs { for job in jobs {
job.abort(); job.abort();
@@ -521,9 +576,23 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
Ok(Ok(())) => {} Ok(Ok(())) => {}
Ok(Err(err)) => { Ok(Err(err)) => {
if matches!(err, DiskError::FileNotFound | DiskError::VolumeNotFound) { if matches!(err, DiskError::FileNotFound | DiskError::VolumeNotFound) {
warn!("list_path_raw producer missing path {:?}", err); warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
state = "producer_missing_path",
error = ?err,
"Metacache producer missing path"
);
} else { } else {
error!("list_path_raw producer err {:?}", err); error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
state = "producer_failed",
error = ?err,
"Metacache producer failed"
);
} }
job_errs.push(err); job_errs.push(err);
} }
@@ -531,7 +600,14 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
if err.is_cancelled() { if err.is_cancelled() {
continue; continue;
} }
error!("list_path_raw join err {:?}", err); error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
state = "producer_join_failed",
error = ?err,
"Metacache producer join failed"
);
job_errs.push(err.into()); job_errs.push(err.into());
} }
} }
crates/ecstore/src/disk/local.rs
+28 -4
View File
@@ -472,7 +472,15 @@ fn resolve_local_disk_root(ep_path: &str) -> Result<PathBuf> {
impl LocalDisk { impl LocalDisk {
pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> { pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
debug!("Creating local disk"); debug!(
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
endpoint = %ep,
state = "create_started",
cleanup,
"Local disk creation started"
);
let endpoint_path = ep.get_file_path(); let endpoint_path = ep.get_file_path();
let root = resolve_local_disk_root(&endpoint_path).inspect_err(|err| { let root = resolve_local_disk_root(&endpoint_path).inspect_err(|err| {
log_startup_disk_error("resolve_local_disk_root", Path::new(&endpoint_path), err); log_startup_disk_error("resolve_local_disk_root", Path::new(&endpoint_path), err);
@@ -501,13 +509,21 @@ impl LocalDisk {
root = ?root, root = ?root,
state = "failed", state = "failed",
error = ?err, error = ?err,
"Disk local startup cleanup failed" "Local disk startup cleanup failed"
); );
} }
// Use optimized path resolution instead of absolutize_virtually // Use optimized path resolution instead of absolutize_virtually
let format_path = root.join(RUSTFS_META_BUCKET).join(super::FORMAT_CONFIG_FILE); let format_path = root.join(RUSTFS_META_BUCKET).join(super::FORMAT_CONFIG_FILE);
debug!("format_path: {:?}", format_path); debug!(
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
root = ?root,
format_path = ?format_path,
state = "format_path_resolved",
"Local disk format path resolved"
);
let (format_data, format_meta) = read_file_exists(&format_path).await.inspect_err(|err| { let (format_data, format_meta) = read_file_exists(&format_path).await.inspect_err(|err| {
log_startup_disk_error("read_format_json", &format_path, err); log_startup_disk_error("read_format_json", &format_path, err);
})?; })?;
@@ -639,7 +655,15 @@ impl LocalDisk {
let root = disk.root.clone(); let root = disk.root.clone();
tokio::spawn(Self::cleanup_deleted_objects_loop(root, exit_rx)); tokio::spawn(Self::cleanup_deleted_objects_loop(root, exit_rx));
debug!("LocalDisk created: {:?}", disk); debug!(
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
endpoint = %disk.endpoint,
root = ?disk.root,
state = "created",
"Local disk created"
);
Ok(disk) Ok(disk)
} }
crates/ecstore/src/pools.rs
+323 -43
View File
@@ -71,6 +71,12 @@ use time::{Duration, OffsetDateTime};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn}; use tracing::{debug, error, info, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_POOLS: &str = "pools";
const EVENT_DECOMMISSION_STATE: &str = "decommission_state";
const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket";
const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry";
pub const POOL_META_NAME: &str = "pool.bin"; pub const POOL_META_NAME: &str = "pool.bin";
pub const POOL_META_FORMAT: u16 = 1; pub const POOL_META_FORMAT: u16 = 1;
pub const POOL_META_VERSION: u16 = 1; pub const POOL_META_VERSION: u16 = 1;
@@ -1280,7 +1286,15 @@ impl ECStore {
take_decommission_canceler(cancelers.as_mut_slice(), idx) take_decommission_canceler(cancelers.as_mut_slice(), idx)
}; };
if !cancel_decommission_canceler(canceler) { if !cancel_decommission_canceler(canceler) {
warn!("decommission_cancel: no active canceler found for pool {}", idx); warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "cancel_skipped",
reason = "no_active_canceler",
"Decommission cancel skipped"
);
} }
if should_reload_pool_meta && let Some(notification_sys) = get_global_notification_sys() { if should_reload_pool_meta && let Some(notification_sys) = get_global_notification_sys() {
@@ -1338,7 +1352,15 @@ impl ECStore {
let store = store.clone(); let store = store.clone();
tokio::spawn(async move { tokio::spawn(async move {
if let Err(err) = store.do_decommission_in_routine(canceler, idx).await { if let Err(err) = store.do_decommission_in_routine(canceler, idx).await {
error!("decommission: routine failed for idx {}: {err}", idx); error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "routine_failed",
error = %err,
"Decommission routine failed"
);
} }
}); });
} }
@@ -1350,7 +1372,14 @@ impl ECStore {
pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> { pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> {
let indices = dedup_indices(&indices); let indices = dedup_indices(&indices);
warn!("decommission: {:?}", indices); info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_indices = ?indices,
state = "requested",
"Decommission requested"
);
validate_start_decommission_request(&indices, self.single_pool())?; validate_start_decommission_request(&indices, self.single_pool())?;
ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?; ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?;
@@ -1363,8 +1392,13 @@ impl ECStore {
for idx in indices { for idx in indices {
if let Err(cancel_err) = self.decommission_cancel(idx).await { if let Err(cancel_err) = self.decommission_cancel(idx).await {
error!( error!(
"decommission: failed to rollback decommission state for idx {} after spawn error: {:?}", event = EVENT_DECOMMISSION_STATE,
idx, cancel_err component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "rollback_failed",
error = ?cancel_err,
"Decommission rollback failed after spawn error"
); );
if rollback_err.is_none() { if rollback_err.is_none() {
rollback_err = Some(Error::other(format!("decommission rollback failed for idx {idx}: {cancel_err}"))); rollback_err = Some(Error::other(format!("decommission rollback failed for idx {idx}: {cancel_err}")));
@@ -1390,10 +1424,28 @@ impl ECStore {
lock_retention: Option<DefaultRetention>, lock_retention: Option<DefaultRetention>,
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
) -> Result<()> { ) -> Result<()> {
warn!("decommission_entry: {} {}", &bucket, &entry.name); debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "started",
"Decommission entry started"
);
wk.give().await; wk.give().await;
if entry.is_dir() { if entry.is_dir() {
warn!("decommission_entry: skip dir {}", &entry.name); debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "skipped_directory",
"Decommission entry skipped directory"
);
return Ok(()); return Ok(());
} }
@@ -1426,7 +1478,16 @@ impl ECStore {
if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) { if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) {
// //
decommissioned += 1; decommissioned += 1;
info!("decommission_pool: DELETE marked object with no other non-current versions will be skipped"); debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "skipped_delete_marker",
"Decommission delete marker skipped"
);
continue; continue;
} }
@@ -1447,8 +1508,16 @@ impl ECStore {
{ {
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) {
warn!( warn!(
"decommission_pool: ignore delete-marker copy for {}/{} version {:?}: {:?}", event = EVENT_DECOMMISSION_ENTRY,
&bucket, &version.name, &version_id, &err component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
version_id = ?version_id,
state = "ignored_delete_marker_copy",
error = ?err,
"Decommission delete marker copy ignored"
); );
ignore = true; ignore = true;
cleanup_ignored = true; cleanup_ignored = true;
@@ -1463,7 +1532,16 @@ impl ECStore {
if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) { if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
decommissioned += 1; decommissioned += 1;
} }
info!("decommission_pool: ignore {}", &version.name); debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "ignored",
"Decommission entry ignored"
);
continue; continue;
} }
@@ -1483,9 +1561,17 @@ impl ECStore {
decommissioned += 1; decommissioned += 1;
} }
info!( debug!(
"decommission_pool: DecomCopyDeleteMarker {} {} {:?} {:?}", event = EVENT_DECOMMISSION_ENTRY,
&bucket, &version.name, &version_id, error component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
version_id = ?version_id,
result = ?error,
state = "delete_marker_copied",
"Decommission delete marker copied"
); );
continue; continue;
} }
@@ -1571,8 +1657,15 @@ impl ECStore {
} }
warn!( warn!(
"decommission_pool: decommission_object done {}/{} {}", event = EVENT_DECOMMISSION_ENTRY,
&bucket_name, &object_name, &version.name component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket_name,
object = %object_name,
version = %version.name,
state = "object_migrated",
"Decommission object migrated"
); );
failure = false; failure = false;
@@ -1583,7 +1676,16 @@ impl ECStore {
if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) { if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
decommissioned += 1; decommissioned += 1;
} }
info!("decommission_pool: ignore {}", &version.name); debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "ignored",
"Decommission entry ignored"
);
continue; continue;
} }
@@ -1624,12 +1726,17 @@ impl ECStore {
resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())? resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())?
} else if decommissioned != fivs.versions.len() || expired > 0 { } else if decommissioned != fivs.versions.len() || expired > 0 {
warn!( warn!(
"decommission_pool: source object retained for {}/{} because only {}/{} versions were decommissioned and {} expired by lifecycle", event = EVENT_DECOMMISSION_ENTRY,
&bucket, component = LOG_COMPONENT_ECSTORE,
&entry.name, subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
decommissioned, decommissioned,
fivs.versions.len(), total_versions = fivs.versions.len(),
expired expired,
state = "source_retained",
"Decommission source object retained"
); );
} }
@@ -1667,7 +1774,16 @@ impl ECStore {
} }
} }
warn!("decommission_pool: decommission_entry done {} {}", &bucket, &entry.name); debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "completed",
"Decommission entry completed"
);
Ok(()) Ok(())
} }
@@ -1705,7 +1821,16 @@ impl ECStore {
for (set_idx, set) in pool.disk_set.iter().enumerate() { for (set_idx, set) in pool.disk_set.iter().enumerate() {
wk.clone().take().await; wk.clone().take().await;
warn!("decommission_pool: decommission_pool {} {}", set_idx, &bi.name); debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_idx,
bucket = %bi.name,
state = "listing_worker_started",
"Decommission listing worker started"
);
let decommission_entry: ListCallback = Arc::new({ let decommission_entry: ListCallback = Arc::new({
let this = Arc::clone(self); let this = Arc::clone(self);
@@ -1753,23 +1878,69 @@ impl ECStore {
let worker = tokio::spawn(async move { let worker = tokio::spawn(async move {
loop { loop {
if rx_clone.is_cancelled() { if rx_clone.is_cancelled() {
warn!("decommission_pool: cancel {}", set_id); debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_worker_cancelled",
"Decommission listing worker cancelled"
);
break; break;
} }
warn!("decommission_pool: list_objects_to_decommission {} {}", set_id, &bi.name); debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_started",
"Decommission listing started"
);
match set match set
.list_objects_to_decommission(rx_clone.clone(), bi.clone(), decommission_entry.clone()) .list_objects_to_decommission(rx_clone.clone(), bi.clone(), decommission_entry.clone())
.await .await
{ {
Ok(_) => { Ok(_) => {
warn!("decommission_pool: list_objects_to_decommission {} done", set_id); debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_completed",
"Decommission listing completed"
);
break; break;
} }
Err(err) => { Err(err) => {
error!("decommission_pool: list_objects_to_decommission {} err {:?}", set_id, &err); error!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_failed",
error = ?err,
"Decommission listing failed"
);
if is_err_bucket_not_found(&err) { if is_err_bucket_not_found(&err) {
warn!("decommission_pool: list_objects_to_decommission {} volume not found", set_id); warn!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_bucket_missing",
"Decommission listing bucket missing"
);
break; break;
} }
@@ -1783,7 +1954,15 @@ impl ECStore {
listing_workers.push((set_id, worker)); listing_workers.push((set_id, worker));
} }
warn!("decommission_pool: decommission_pool wait {} {}", idx, &bi.name); debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "waiting_for_workers",
"Decommission waiting for workers"
);
let mut listing_worker_error = None; let mut listing_worker_error = None;
for (set_id, worker) in listing_workers { for (set_id, worker) in listing_workers {
@@ -1807,11 +1986,28 @@ impl ECStore {
} }
if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) { if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
warn!("decommission_pool: canceled after wait {} {}", idx, &bi.name); warn!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "cancelled_after_wait",
error = %err,
"Decommission bucket cancelled after wait"
);
return Err(err); return Err(err);
} }
warn!("decommission_pool: decommission_pool done {} {}", idx, &bi.name); debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "completed",
"Decommission bucket completed"
);
Ok(()) Ok(())
} }
@@ -1821,7 +2017,14 @@ impl ECStore {
defer!(|| async { defer!(|| async {
let mut cancelers = self.decommission_cancelers.write().await; let mut cancelers = self.decommission_cancelers.write().await;
if take_decommission_canceler(cancelers.as_mut_slice(), idx).is_none() { if take_decommission_canceler(cancelers.as_mut_slice(), idx).is_none() {
warn!("decommission: canceler already cleared for pool {}", idx); warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "canceler_already_cleared",
"Decommission canceler already cleared"
);
} }
}); });
@@ -1830,7 +2033,14 @@ impl ECStore {
let (final_state, canceled, cmd_line) = { let (final_state, canceled, cmd_line) = {
let pool_meta = self.pool_meta.read().await; let pool_meta = self.pool_meta.read().await;
let Some(pool) = pool_meta.pools.get(idx) else { let Some(pool) = pool_meta.pools.get(idx) else {
error!("decommission: pool metadata missing for idx {}", idx); error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "pool_metadata_missing",
"Decommission pool metadata missing"
);
return Err(Error::other(format!( return Err(Error::other(format!(
"failed to resolve decommission final state: pool metadata missing for idx {idx}" "failed to resolve decommission final state: pool metadata missing for idx {idx}"
))); )));
@@ -1849,29 +2059,75 @@ impl ECStore {
}; };
if let Err(err) = result { if let Err(err) = result {
error!("decom err {:?}", &err); error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "background_failed",
error = ?err,
"Decommission background routine failed"
);
if is_err_operation_canceled(&err) || should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) { if is_err_operation_canceled(&err) || should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
warn!("decommission: canceled for pool {}, preserving canceled state", cmd_line); warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "cancelled_preserved",
"Decommission cancelled; preserving canceled state"
);
return Ok(()); return Ok(());
} }
resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?; resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?;
warn!("decommission: decommission_failed {}", idx); warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "marked_failed",
"Decommission marked failed"
);
return Ok(()); return Ok(());
} }
warn!("decommission: decommission_in_background complete {}", idx); debug!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "background_complete",
"Decommission background routine completed"
);
if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) { if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
warn!("decommission: canceled for pool {}, skipping terminal state overwrite", cmd_line); warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "terminal_state_preserved",
"Decommission terminal state preserved after cancellation"
);
return Ok(()); return Ok(());
} }
match final_state { match final_state {
DecommissionFinalState::Complete => { DecommissionFinalState::Complete => {
warn!("Decommissioning complete for pool {}, verifying for any pending objects", cmd_line); debug!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "verifying_completion",
"Decommission completion verification started"
);
if let Err(err) = self.check_after_decommission(idx).await { if let Err(err) = self.check_after_decommission(idx).await {
resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?; resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?;
return Err(Error::other(format!( return Err(Error::other(format!(
@@ -1879,16 +2135,40 @@ impl ECStore {
))); )));
} }
warn!("Decommissioning complete for pool {}, marking completed state", cmd_line); info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "marking_completed",
"Decommission marking completed state"
);
resolve_decommission_terminal_mark_result(self.complete_decommission(idx).await, "completed", &cmd_line)?; resolve_decommission_terminal_mark_result(self.complete_decommission(idx).await, "completed", &cmd_line)?;
} }
DecommissionFinalState::Failed => { DecommissionFinalState::Failed => {
warn!("Decommissioning finished with failed items for pool {}, marking failed state", cmd_line); warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "marking_failed",
"Decommission marking failed state"
);
resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?; resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?;
} }
} }
warn!("Decommissioning complete for pool {}", cmd_line); info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "completed",
"Decommission completed"
);
Ok(()) Ok(())
} }
crates/ecstore/src/rebalance.rs
+176 -30
View File
@@ -858,7 +858,14 @@ impl ECStore {
"init_rebalance_meta", "init_rebalance_meta",
)?; )?;
info!("init_rebalance_meta: rebalance meta saved"); info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "metadata_initialized",
bucket_count = bucktes.len(),
"Rebalance metadata initialized"
);
let id = meta.id.clone(); let id = meta.id.clone();
@@ -898,9 +905,16 @@ impl ECStore {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
pub async fn next_rebal_bucket(&self, pool_index: usize) -> Result<Option<String>> { pub async fn next_rebal_bucket(&self, pool_index: usize) -> Result<Option<String>> {
info!("next_rebal_bucket: pool_index: {}", pool_index);
let rebalance_meta = self.rebalance_meta.read().await; let rebalance_meta = self.rebalance_meta.read().await;
info!("next_rebal_bucket: rebalance_meta: {:?}", rebalance_meta); debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
has_meta = rebalance_meta.is_some(),
state = "next_bucket_lookup",
"Rebalance next bucket lookup"
);
resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index) resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index)
} }
@@ -931,20 +945,38 @@ impl ECStore {
let rebalance_meta = self.rebalance_meta.read().await; let rebalance_meta = self.rebalance_meta.read().await;
if let Some(meta) = rebalance_meta.as_ref() { if let Some(meta) = rebalance_meta.as_ref() {
meta.pool_stats.iter().enumerate().for_each(|(i, v)| { meta.pool_stats.iter().enumerate().for_each(|(i, v)| {
info!( debug!(
"is_rebalance_started: pool_index: {}, participating: {:?}, status: {:?}", event = EVENT_REBALANCE_STATE,
i, v.participating, v.info.status component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = i,
participating = v.participating,
status = ?v.info.status,
state = "status_inspected",
"Rebalance status inspected"
); );
}); });
let started = is_rebalance_conflicting_with_decommission(meta); let started = is_rebalance_conflicting_with_decommission(meta);
if started { if started {
info!("is_rebalance_started: rebalance started"); debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "running",
"Rebalance is running"
);
return true; return true;
} }
} }
info!("is_rebalance_started: rebalance not started"); debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "not_running",
"Rebalance is not running"
);
false false
} }
@@ -1108,15 +1140,23 @@ impl ECStore {
workers_started += 1; workers_started += 1;
tokio::spawn(async move { tokio::spawn(async move {
if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx).await { if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx).await {
error!("Rebalance failed for pool {}: {}", pool_idx, err); error!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = pool_idx,
state = "pool_failed",
error = %err,
"Rebalance pool failed"
);
} else { } else {
info!( debug!(
event = EVENT_REBALANCE_STATE, event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE, component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE, subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = pool_idx, pool_index = pool_idx,
state = "completed", state = "completed",
"Completed rebalance pool" "Rebalance pool completed"
); );
} }
}); });
@@ -1139,7 +1179,8 @@ impl ECStore {
component = LOG_COMPONENT_ECSTORE, component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE, subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "started", state = "started",
"Started rebalance" worker_count = workers_started,
"Rebalance started"
); );
Ok(()) Ok(())
} }
@@ -1240,7 +1281,7 @@ impl ECStore {
subsystem = LOG_SUBSYSTEM_REBALANCE, subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index, pool_index,
state = "pool_started", state = "pool_started",
"Started rebalance worker" "Rebalance worker started"
); );
let mut final_result: Result<()> = Ok(()); let mut final_result: Result<()> = Ok(());
let mut deferred_buckets = HashSet::new(); let mut deferred_buckets = HashSet::new();
@@ -1267,7 +1308,15 @@ impl ECStore {
let next_bucket = match self.next_rebal_bucket(pool_index).await { let next_bucket = match self.next_rebal_bucket(pool_index).await {
Ok(bucket) => bucket, Ok(bucket) => bucket,
Err(err) => { Err(err) => {
error!("next_rebal_bucket failed for pool {}: {:?}", pool_index, err); error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "next_bucket_failed",
error = ?err,
"Rebalance next bucket lookup failed"
);
final_result = Err(resolve_rebalance_terminal_error( final_result = Err(resolve_rebalance_terminal_error(
err.clone(), err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1294,7 +1343,16 @@ impl ECStore {
) { ) {
Ok(outcome) => outcome, Ok(outcome) => outcome,
Err(err) => { Err(err) => {
error!("Error rebalancing bucket {}: {:?}", bucket, err); error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "bucket_failed",
error = ?err,
"Rebalance bucket failed"
);
final_result = Err(resolve_rebalance_terminal_error( final_result = Err(resolve_rebalance_terminal_error(
err.clone(), err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1308,7 +1366,16 @@ impl ECStore {
let err = Error::other(format!( let err = Error::other(format!(
"rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}" "rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
)); ));
error!("Error rebalancing bucket {}: {:?}", bucket, err); error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "bucket_deferred_repeatedly",
error = ?err,
"Rebalance bucket failed after repeated deferral"
);
final_result = Err(resolve_rebalance_terminal_error( final_result = Err(resolve_rebalance_terminal_error(
err.clone(), err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1327,7 +1394,16 @@ impl ECStore {
"Deferred rebalance bucket after transient object failures" "Deferred rebalance bucket after transient object failures"
); );
if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await { if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await {
error!("defer_rebalance_bucket failed for pool {}: {:?}", pool_index, err); error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "defer_failed",
error = ?err,
"Rebalance bucket defer failed"
);
final_result = Err(resolve_rebalance_terminal_error( final_result = Err(resolve_rebalance_terminal_error(
err.clone(), err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1347,7 +1423,15 @@ impl ECStore {
"Completed rebalance bucket" "Completed rebalance bucket"
); );
if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await { if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await {
error!("bucket_rebalance_done failed for pool {}: {:?}", pool_index, err); error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "bucket_done_mark_failed",
error = ?err,
"Rebalance bucket completion mark failed"
);
final_result = Err(resolve_rebalance_terminal_error( final_result = Err(resolve_rebalance_terminal_error(
err.clone(), err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1374,7 +1458,7 @@ impl ECStore {
subsystem = LOG_SUBSYSTEM_REBALANCE, subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index, pool_index,
state = "pool_done", state = "pool_done",
"Finished rebalance worker" "Rebalance worker finished"
); );
if final_result.is_ok() if final_result.is_ok()
@@ -1388,7 +1472,14 @@ impl ECStore {
{ {
final_result = Err(err); final_result = Err(err);
} }
info!("Pool {} rebalancing is done2", pool_index); debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "pool_result_returned",
"Rebalance worker result returned"
);
final_result final_result
} }
@@ -2591,7 +2682,15 @@ impl ECStore {
ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?; ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
// Placeholder for actual bucket rebalance logic // Placeholder for actual bucket rebalance logic
info!("Rebalancing bucket {} in pool {}", bucket, pool_index); debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "entry_scan_started",
"Rebalance bucket entry scan started"
);
// TODO: other config // TODO: other config
// if bucket != RUSTFS_META_BUCKET{ // if bucket != RUSTFS_META_BUCKET{
@@ -2765,9 +2864,14 @@ impl ECStore {
let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?; let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?;
info!( debug!(
"save_rebalance_stats: save rebalance meta, pool_idx: {}, opt: {:?}, meta: {:?}", event = EVENT_REBALANCE_STATE,
pool_idx, opt, meta_to_save component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = pool_idx,
save_opt = ?opt,
state = "metadata_save_requested",
"Rebalance metadata save requested"
); );
let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}"); let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}");
resolve_rebalance_meta_save_result( resolve_rebalance_meta_save_result(
@@ -2838,12 +2942,27 @@ impl SetDisks {
bucket: String, bucket: String,
cb: ListCallback, cb: ListCallback,
) -> Result<()> { ) -> Result<()> {
info!("list_objects_to_rebalance: start list_objects_to_rebalance"); debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
bucket = %bucket,
state = "started",
"Rebalance listing started"
);
// Placeholder for actual object listing logic // Placeholder for actual object listing logic
let (disks, _) = self.get_online_disks_with_healing(false).await; let (disks, _) = self.get_online_disks_with_healing(false).await;
ensure_rebalance_listing_disks_available(!disks.is_empty(), &bucket)?; ensure_rebalance_listing_disks_available(!disks.is_empty(), &bucket)?;
info!("list_objects_to_rebalance: get online disks with healing"); debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
bucket = %bucket,
disk_count = disks.len(),
state = "disks_resolved",
"Rebalance listing disks resolved"
);
let listing_quorum = self.set_drive_count.div_ceil(2); let listing_quorum = self.set_drive_count.div_ceil(2);
let resolver = MetadataResolutionParams { let resolver = MetadataResolutionParams {
@@ -2863,7 +2982,14 @@ impl SetDisks {
min_disks: listing_quorum, min_disks: listing_quorum,
skip_walkdir_total_timeout: true, skip_walkdir_total_timeout: true,
agreed: Some(Box::new(move |entry: MetaCacheEntry| { agreed: Some(Box::new(move |entry: MetaCacheEntry| {
info!("list_objects_to_rebalance: agreed: {:?}", &entry.name); debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
entry = %entry.name,
state = "agreed_entry",
"Rebalance listing agreed entry"
);
Box::pin(cb1(entry)) Box::pin(cb1(entry))
})), })),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| { partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
@@ -2873,11 +2999,24 @@ impl SetDisks {
match entries.resolve(resolver) { match entries.resolve(resolver) {
Some(entry) => { Some(entry) => {
info!("list_objects_to_rebalance: list_objects_to_decommission get {}", &entry.name); debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
entry = %entry.name,
state = "resolved_partial_entry",
"Rebalance listing resolved partial entry"
);
Box::pin(async move { cb(entry).await }) Box::pin(async move { cb(entry).await })
} }
None => { None => {
info!("list_objects_to_rebalance: list_objects_to_decommission get none"); debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "partial_entry_missing",
"Rebalance listing partial entry missing"
);
Box::pin(async {}) Box::pin(async {})
} }
} }
@@ -2887,7 +3026,14 @@ impl SetDisks {
) )
.await?; .await?;
info!("list_objects_to_rebalance: list_objects_to_rebalance done"); debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
bucket = %bucket,
state = "completed",
"Rebalance listing completed"
);
Ok(()) Ok(())
} }
} }
crates/ecstore/src/rpc/remote_disk.rs
+363 -37
View File
@@ -65,7 +65,7 @@ use tokio::{
}; };
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tonic::{Request, service::interceptor::InterceptedService, transport::Channel}; use tonic::{Request, service::interceptor::InterceptedService, transport::Channel};
use tracing::{debug, info, warn}; use tracing::{debug, warn};
use uuid::Uuid; use uuid::Uuid;
#[derive(Clone, Copy, Debug, Eq, PartialEq)] #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -76,6 +76,10 @@ enum FailureHealthAction {
const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2; const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2;
const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20); const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20);
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64> async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64>
where where
@@ -275,7 +279,16 @@ impl RemoteDisk {
if initial_probe_ok { if initial_probe_ok {
health.record_operation_success(&endpoint, "connectivity_probe_success"); health.record_operation_success(&endpoint, "connectivity_probe_success");
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") { } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!("Remote disk health check failed for {}: marking as faulty", addr); warn!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "initial_probe_failed",
result = "mark_faulty",
"Remote disk initial health probe failed"
);
// Start recovery monitoring // Start recovery monitoring
let health_clone = Arc::clone(&health); let health_clone = Arc::clone(&health);
@@ -291,7 +304,15 @@ impl RemoteDisk {
loop { loop {
tokio::select! { tokio::select! {
_ = cancel_token.cancelled() => { _ = cancel_token.cancelled() => {
debug!("Health monitoring cancelled for remote disk: {}", addr); debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "monitor_cancelled",
"Remote disk health monitor cancelled"
);
return; return;
} }
_ = interval.tick() => { _ = interval.tick() => {
@@ -320,7 +341,16 @@ impl RemoteDisk {
if Self::perform_connectivity_check(&addr).await.is_ok() { if Self::perform_connectivity_check(&addr).await.is_ok() {
health.record_operation_success(&endpoint, "connectivity_probe_success"); health.record_operation_success(&endpoint, "connectivity_probe_success");
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") { } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!("Remote disk health check failed for {}: marking as faulty", addr); warn!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "probe_failed",
result = "mark_faulty",
"Remote disk health probe failed"
);
// Start recovery monitoring // Start recovery monitoring
let health_clone = Arc::clone(&health); let health_clone = Arc::clone(&health);
@@ -354,9 +384,25 @@ impl RemoteDisk {
_ = interval.tick() => { _ = interval.tick() => {
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() { if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success"); let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
info!("Remote disk recovery probe succeeded: {}", addr); debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "recovery_probe_succeeded",
"Remote disk recovery probe succeeded"
);
if became_online { if became_online {
info!("Remote disk recovered: {}", addr); debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "recovered",
"Remote disk recovered"
);
return; return;
} }
} else { } else {
@@ -469,7 +515,16 @@ impl RemoteDisk {
{ {
// Check if disk is faulty // Check if disk is faulty
if self.health.is_faulty() { if self.health.is_faulty() {
warn!("remote disk {} health is faulty, returning error", self.to_string()); warn!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
op,
state = "faulty_short_circuit",
"Remote disk operation short-circuited by faulty state"
);
return Err(DiskError::FaultyDisk); return Err(DiskError::FaultyDisk);
} }
@@ -519,10 +574,14 @@ impl RemoteDisk {
self.mark_faulty_and_evict("operation_timeout").await; self.mark_faulty_and_evict("operation_timeout").await;
} }
warn!( warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint, endpoint = %self.endpoint,
addr = %self.addr, addr = %self.addr,
op, op,
timeout_ms = timeout_duration.as_millis(), timeout_ms = timeout_duration.as_millis(),
state = "timeout",
"Remote disk operation timed out" "Remote disk operation timed out"
); );
Err(DiskError::Timeout) Err(DiskError::Timeout)
@@ -547,11 +606,15 @@ impl RemoteDisk {
) )
.increment(1); .increment(1);
warn!( warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint, endpoint = %self.endpoint,
addr = %self.addr, addr = %self.addr,
op, op,
timeout_ms = timeout_duration.as_millis(), timeout_ms = timeout_duration.as_millis(),
"Remote disk operation returned a network-like error" state = "network_like_error",
"Remote disk operation returned network-like error"
); );
if failure_health_action == FailureHealthAction::MarkFailure { if failure_health_action == FailureHealthAction::MarkFailure {
self.mark_faulty_and_evict("operation_network_error").await; self.mark_faulty_and_evict("operation_network_error").await;
@@ -574,13 +637,26 @@ impl RemoteDisk {
.increment(1); .increment(1);
if transitioned_to_offline { if transitioned_to_offline {
warn!( warn!(
"Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}", event = EVENT_REMOTE_DISK_HEALTH,
self.endpoint, self.addr, reason component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
reason,
state = "marked_faulty",
"Remote disk marked faulty"
); );
} else { } else {
warn!( warn!(
"Remote disk marked suspect after timeout: endpoint={}, addr={}, reason={}, state={:?}", event = EVENT_REMOTE_DISK_HEALTH,
self.endpoint, self.addr, reason, state component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
reason,
runtime_state = ?state,
state = "marked_suspect",
"Remote disk marked suspect"
); );
} }
counter!( counter!(
@@ -589,11 +665,15 @@ impl RemoteDisk {
"reason" => reason.to_string() "reason" => reason.to_string()
) )
.increment(1); .increment(1);
info!( debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint, endpoint = %self.endpoint,
addr = %self.addr, addr = %self.addr,
reason, reason,
"Evicting cached remote disk connection after fault transition" state = "evict_cached_connection",
"Remote disk cached connection evicted"
); );
evict_failed_connection(&self.addr).await; evict_failed_connection(&self.addr).await;
} }
@@ -705,7 +785,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn make_volume(&self, volume: &str) -> Result<()> { async fn make_volume(&self, volume: &str) -> Result<()> {
info!("make_volume"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
op = "make_volume",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -733,7 +822,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
info!("make_volumes"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume_count = volumes.len(),
op = "make_volumes",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -761,7 +859,15 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> { async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
info!("list_volumes"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
op = "list_volumes",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -794,7 +900,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> { async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
info!("stat_volume"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
op = "stat_volume",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -824,7 +939,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn delete_volume(&self, volume: &str) -> Result<()> { async fn delete_volume(&self, volume: &str) -> Result<()> {
info!("delete_volume {}/{}", self.endpoint.to_string(), volume); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
op = "delete_volume",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -909,7 +1033,17 @@ impl DiskAPI for RemoteDisk {
force_del_marker: bool, force_del_marker: bool,
opts: DeleteOptions, opts: DeleteOptions,
) -> Result<()> { ) -> Result<()> {
info!("delete_version"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "delete_version",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -946,7 +1080,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> { async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> {
info!("delete_versions"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
version_count = versions.len(),
op = "delete_versions",
state = "started",
"Remote disk RPC started"
);
if self.health.is_faulty() { if self.health.is_faulty() {
return vec![Some(DiskError::FaultyDisk); versions.len()]; return vec![Some(DiskError::FaultyDisk); versions.len()];
@@ -1041,7 +1185,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
info!("delete_paths"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path_count = paths.len(),
op = "delete_paths",
state = "started",
"Remote disk RPC started"
);
let paths = paths.to_owned(); let paths = paths.to_owned();
self.execute_with_timeout( self.execute_with_timeout(
@@ -1071,7 +1225,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
info!("write_metadata {}/{}", volume, path); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "write_metadata",
state = "started",
"Remote disk RPC started"
);
let file_info = serde_json::to_string(&fi)?; let file_info = serde_json::to_string(&fi)?;
let file_info_bin = encode_msgpack(&fi)?; let file_info_bin = encode_msgpack(&fi)?;
@@ -1134,7 +1298,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> { async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
info!("update_metadata"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "update_metadata",
state = "started",
"Remote disk RPC started"
);
let file_info = serde_json::to_string(&fi)?; let file_info = serde_json::to_string(&fi)?;
let opts_str = serde_json::to_string(&opts)?; let opts_str = serde_json::to_string(&opts)?;
let file_info_bin = encode_msgpack(&fi)?; let file_info_bin = encode_msgpack(&fi)?;
@@ -1180,7 +1354,18 @@ impl DiskAPI for RemoteDisk {
version_id: &str, version_id: &str,
opts: &ReadOptions, opts: &ReadOptions,
) -> Result<FileInfo> { ) -> Result<FileInfo> {
info!("read_version"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
version_id,
op = "read_version",
state = "started",
"Remote disk RPC started"
);
let opts_str = serde_json::to_string(opts)?; let opts_str = serde_json::to_string(opts)?;
let opts_bin = encode_msgpack(opts)?; let opts_bin = encode_msgpack(opts)?;
@@ -1217,7 +1402,18 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))] #[tracing::instrument(level = "debug", skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> { async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
info!("read_xl {}/{}/{}", self.endpoint.to_string(), volume, path); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
read_data,
op = "read_xl",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -1257,7 +1453,19 @@ impl DiskAPI for RemoteDisk {
dst_volume: &str, dst_volume: &str,
dst_path: &str, dst_path: &str,
) -> Result<RenameDataResp> { ) -> Result<RenameDataResp> {
info!("rename_data {}/{}/{}/{}", self.addr, self.endpoint.to_string(), dst_volume, dst_path); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
src_volume,
src_path,
dst_volume,
dst_path,
op = "rename_data",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout_for_op( self.execute_with_timeout_for_op(
"rename_data", "rename_data",
@@ -1325,7 +1533,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self, wr))] #[tracing::instrument(skip(self, wr))]
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> { async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
info!("walk_dir {}", self.endpoint.to_string()); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
bucket = %opts.bucket,
base_dir = %opts.base_dir,
op = "walk_dir",
state = "started",
"Remote disk RPC started"
);
let disk = self.disk_ref().await; let disk = self.disk_ref().await;
let body = serde_json::to_vec(&opts)?; let body = serde_json::to_vec(&opts)?;
@@ -1443,7 +1661,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))] #[tracing::instrument(level = "debug", skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> { async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
info!("append_file {}/{}", volume, path); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "append_file",
state = "started",
"Remote disk RPC started"
);
if self.health.is_faulty() { if self.health.is_faulty() {
return Err(DiskError::FaultyDisk); return Err(DiskError::FaultyDisk);
@@ -1487,7 +1715,19 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))] #[tracing::instrument(level = "debug", skip(self))]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
info!("rename_file"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
src_volume,
src_path,
dst_volume,
dst_path,
op = "rename_file",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -1518,7 +1758,19 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> { async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> {
info!("rename_part {}/{}", src_volume, src_path); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
src_volume,
src_path,
dst_volume,
dst_path,
op = "rename_part",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -1550,7 +1802,19 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
recursive = opt.recursive,
immediate = opt.immediate,
op = "delete",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -1581,7 +1845,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> { async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("verify_file"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "verify_file",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -1614,6 +1888,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result<Vec<ObjectPartInfo>> { async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result<Vec<ObjectPartInfo>> {
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
bucket,
path_count = paths.len(),
op = "read_parts",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
let mut client = self let mut client = self
@@ -1642,7 +1927,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> { async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("check_parts"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "check_parts",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -1675,7 +1970,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> { async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
bucket = %req.bucket,
prefix = %req.prefix,
op = "read_multiple",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -1721,7 +2026,18 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> { async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> {
info!("write_all"); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
bytes = data.len(),
op = "write_all",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
@@ -1779,7 +2095,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes> { async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes> {
info!("read_all {}/{}", volume, path); debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "read_all",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout( self.execute_with_timeout(
|| async { || async {
crates/ecstore/src/set_disk.rs
+110 -10
View File
@@ -130,6 +130,12 @@ use tracing::error;
use tracing::{debug, info, warn}; use tracing::{debug, info, warn};
use uuid::Uuid; use uuid::Uuid;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_SET_DISK: &str = "set_disk";
const EVENT_SET_DISK_MULTIPART: &str = "set_disk_multipart";
const EVENT_SET_DISK_WRITE: &str = "set_disk_write";
const EVENT_SET_DISK_HEAL: &str = "set_disk_heal";
use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _}; use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024; pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
@@ -948,7 +954,16 @@ impl ObjectIO for SetDisks {
) )
.await .await
{ {
error!("get_object_with_fileinfo {bucket}/{object} err {:?}", e); error!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
state = "read_pipeline_failed",
error = ?e,
"Set disk object read pipeline failed"
);
}; };
}); });
@@ -1065,7 +1080,15 @@ impl ObjectIO for SetDisks {
{ {
Ok(writer) => (Some(writer), None), Ok(writer) => (Some(writer), None),
Err(err) => { Err(err) => {
warn!("create_bitrot_writer disk {}, err {:?}, skipping operation", disk.to_string(), err); warn!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
disk = ?disk,
state = "bitrot_writer_skipped",
error = ?err,
"Set disk bitrot writer skipped"
);
(None, Some(err)) (None, Some(err))
} }
} }
@@ -1085,7 +1108,18 @@ impl ObjectIO for SetDisks {
let nil_count = errors.iter().filter(|&e| e.is_none()).count(); let nil_count = errors.iter().filter(|&e| e.is_none()).count();
if nil_count < write_quorum { if nil_count < write_quorum {
error!("not enough disks to write: {:?}", errors); error!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
write_quorum,
available_writers = nil_count,
state = "write_quorum_unavailable",
error = ?errors,
"Set disk write quorum unavailable"
);
if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) { if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(to_object_err(write_err.into(), vec![bucket, object])); return Err(to_object_err(write_err.into(), vec![bucket, object]));
} }
@@ -1136,7 +1170,17 @@ impl ObjectIO for SetDisks {
// } // }
if (w_size as i64) < data.size() { if (w_size as i64) < data.size() {
warn!("put_object write size < data.size(), w_size={}, data.size={}", w_size, data.size()); warn!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
written_size = w_size,
expected_size = data.size(),
state = "short_write",
"Set disk write produced fewer bytes than expected"
);
return Err(Error::other(format!( return Err(Error::other(format!(
"put_object write size < data.size(), w_size={}, data.size={}", "put_object write size < data.size(), w_size={}, data.size={}",
w_size, w_size,
@@ -2976,7 +3020,15 @@ impl MultipartOperations for SetDisks {
{ {
Ok(writer) => writer, Ok(writer) => writer,
Err(err) => { Err(err) => {
warn!("create_bitrot_writer disk {}, err {:?}, skipping operation", disk.to_string(), err); warn!(
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
disk = ?disk,
state = "bitrot_writer_skipped",
error = ?err,
"Set disk multipart bitrot writer skipped"
);
errors.push(Some(err)); errors.push(Some(err));
writers.push(None); writers.push(None);
continue; continue;
@@ -3021,7 +3073,18 @@ impl MultipartOperations for SetDisks {
let _ = mem::replace(&mut data.stream, reader); let _ = mem::replace(&mut data.stream, reader);
if (w_size as i64) < data.size() { if (w_size as i64) < data.size() {
warn!("put_object_part write size < data.size(), w_size={}, data.size={}", w_size, data.size()); warn!(
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
part_number = part_id,
written_size = w_size,
expected_size = data.size(),
state = "short_write",
"Set disk multipart write produced fewer bytes than expected"
);
return Err(Error::other(format!( return Err(Error::other(format!(
"put_object_part write size < data.size(), w_size={}, data.size={}", "put_object_part write size < data.size(), w_size={}, data.size={}",
w_size, w_size,
@@ -3657,7 +3720,17 @@ impl MultipartOperations for SetDisks {
); );
return Err(Error::InvalidPart(p.part_num, "".to_owned(), p.etag.clone().unwrap_or_default())); return Err(Error::InvalidPart(p.part_num, "".to_owned(), p.etag.clone().unwrap_or_default()));
}; };
info!(target:"rustfs_ecstore::set_disk", part_number = p.part_num, part_size = ext_part.size, part_actual_size = ext_part.actual_size, "Completing multipart part"); debug!(
target:"rustfs_ecstore::set_disk",
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
part_number = p.part_num,
part_size = ext_part.size,
part_actual_size = ext_part.actual_size,
state = "part_validated",
"Set disk multipart part validated"
);
// Normalize ETags by removing quotes before comparison (PR #592 compatibility) // Normalize ETags by removing quotes before comparison (PR #592 compatibility)
let client_etag = p.etag.as_ref().map(|e| rustfs_utils::path::trim_etag(e)); let client_etag = p.etag.as_ref().map(|e| rustfs_utils::path::trim_etag(e));
@@ -3988,7 +4061,16 @@ impl HealOperations for SetDisks {
let disks = disks.clone(); let disks = disks.clone();
let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false).await?; let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false).await?;
if DiskError::is_all_not_found(&errs) { if DiskError::is_all_not_found(&errs) {
debug!(bucket, object, version_id, "heal_object skipped missing object"); debug!(
event = EVENT_SET_DISK_HEAL,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
version_id,
state = "missing_object_skipped",
"Set disk heal skipped missing object"
);
let err = if !version_id.is_empty() { let err = if !version_id.is_empty() {
Error::FileVersionNotFound Error::FileVersionNotFound
} else { } else {
@@ -4356,7 +4438,16 @@ async fn disks_with_all_parts(
verify_resp = v; verify_resp = v;
} }
Err(err) => { Err(err) => {
info!("verify_file failed: {err:?}, object_name={}, index: {index}", object_name); debug!(
event = EVENT_SET_DISK_HEAL,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
object = %object_name,
disk_index = index,
state = "verify_failed",
error = ?err,
"Set disk verify_file failed"
);
verify_err = Some(err); verify_err = Some(err);
} }
} }
@@ -4366,7 +4457,16 @@ async fn disks_with_all_parts(
verify_resp = v; verify_resp = v;
} }
Err(err) => { Err(err) => {
info!("check_parts failed: {err:?}, object_name={}, index: {index}", object_name); debug!(
event = EVENT_SET_DISK_HEAL,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
object = %object_name,
disk_index = index,
state = "check_parts_failed",
error = ?err,
"Set disk check_parts failed"
);
verify_err = Some(err); verify_err = Some(err);
} }
} }
crates/heal/src/heal/channel.rs
+17 -17
View File
@@ -69,7 +69,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL, subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "started", state = "started",
"Heal channel processor state updated" "Heal channel started"
); );
loop { loop {
@@ -85,7 +85,7 @@ impl HealChannelProcessor {
subsystem = LOG_SUBSYSTEM_CHANNEL, subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "process_failed", state = "process_failed",
error = %e, error = %e,
"Heal channel request processing failed" "Heal channel processing failed"
); );
} }
} }
@@ -96,7 +96,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL, subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "receiver_closed", state = "receiver_closed",
"Heal channel processor state updated" "Heal channel receiver closed"
); );
break; break;
} }
@@ -105,7 +105,7 @@ impl HealChannelProcessor {
response = self.response_receiver.recv() => { response = self.response_receiver.recv() => {
if let Some(response) = response { if let Some(response) = response {
// Handle response if needed // Handle response if needed
info!( debug!(
target: "rustfs::heal::channel", target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_RESPONSE, event = EVENT_HEAL_CHANNEL_RESPONSE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -113,7 +113,7 @@ impl HealChannelProcessor {
request_id = %response.request_id, request_id = %response.request_id,
success = response.success, success = response.success,
state = "received_local", state = "received_local",
"Heal channel response observed" "Heal response observed"
); );
} }
} }
@@ -126,7 +126,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL, subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "stopped", state = "stopped",
"Heal channel processor state updated" "Heal channel stopped"
); );
Ok(()) Ok(())
} }
@@ -154,7 +154,7 @@ impl HealChannelProcessor {
request: HealChannelRequest, request: HealChannelRequest,
response_tx: oneshot::Sender<std::result::Result<HealAdmissionResult, String>>, response_tx: oneshot::Sender<std::result::Result<HealAdmissionResult, String>>,
) -> Result<()> { ) -> Result<()> {
info!( debug!(
target: "rustfs::heal::channel", target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST, event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -163,7 +163,7 @@ impl HealChannelProcessor {
bucket = %request.bucket, bucket = %request.bucket,
object_prefix = %request.object_prefix.as_deref().unwrap_or(""), object_prefix = %request.object_prefix.as_deref().unwrap_or(""),
state = "start_received", state = "start_received",
"Heal channel start request received" "Heal start received"
); );
// Convert channel request to heal request // Convert channel request to heal request
@@ -185,7 +185,7 @@ impl HealChannelProcessor {
// Submit to heal manager // Submit to heal manager
match self.heal_manager.submit_heal_request(heal_request).await { match self.heal_manager.submit_heal_request(heal_request).await {
Ok(admission) => { Ok(admission) => {
info!( debug!(
target: "rustfs::heal::channel", target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST, event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -193,7 +193,7 @@ impl HealChannelProcessor {
request_id = %request.id, request_id = %request.id,
admission = admission.result_label(), admission = admission.result_label(),
state = "admission_decided", state = "admission_decided",
"Heal channel admission decision completed" "Heal admission decided"
); );
let _ = response_tx.send(Ok(admission)); let _ = response_tx.send(Ok(admission));
@@ -225,7 +225,7 @@ impl HealChannelProcessor {
request_id = %request.id, request_id = %request.id,
state = "submit_failed", state = "submit_failed",
error = %error_text, error = %error_text,
"Heal channel start request failed" "Heal start submission failed"
); );
let _ = response_tx.send(Err(error_text.clone())); let _ = response_tx.send(Err(error_text.clone()));
@@ -251,7 +251,7 @@ impl HealChannelProcessor {
client_token: String, client_token: String,
response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>, response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
) -> Result<()> { ) -> Result<()> {
info!( debug!(
target: "rustfs::heal::channel", target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST, event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -259,7 +259,7 @@ impl HealChannelProcessor {
request_id = %client_token, request_id = %client_token,
heal_path = %heal_path, heal_path = %heal_path,
state = "query_received", state = "query_received",
"Heal channel query request received" "Heal query received"
); );
let (summary, detail, items) = match self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await { let (summary, detail, items) = match self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await {
@@ -332,7 +332,7 @@ impl HealChannelProcessor {
client_token: String, client_token: String,
response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>, response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
) -> Result<()> { ) -> Result<()> {
info!( debug!(
target: "rustfs::heal::channel", target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST, event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -340,7 +340,7 @@ impl HealChannelProcessor {
request_id = %client_token, request_id = %client_token,
heal_path = %heal_path, heal_path = %heal_path,
state = "cancel_received", state = "cancel_received",
"Heal channel cancel request received" "Heal cancel received"
); );
let request_id = if client_token.is_empty() { let request_id = if client_token.is_empty() {
@@ -451,7 +451,7 @@ impl HealChannelProcessor {
request_id = %response.request_id, request_id = %response.request_id,
state = "enqueue_local_failed", state = "enqueue_local_failed",
error = %e, error = %e,
"Heal channel response enqueue failed" "Heal response local enqueue failed"
); );
} }
// Always attempt to broadcast, even if local send failed // Always attempt to broadcast, even if local send failed
@@ -464,7 +464,7 @@ impl HealChannelProcessor {
subsystem = LOG_SUBSYSTEM_CHANNEL, subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "broadcast_failed", state = "broadcast_failed",
error = %e, error = %e,
"Heal channel response broadcast failed" "Heal response broadcast failed"
); );
} }
} }
crates/heal/src/heal/erasure_healer.rs
+27 -27
View File
@@ -27,7 +27,7 @@ use std::sync::{
atomic::{AtomicUsize, Ordering}, atomic::{AtomicUsize, Ordering},
}; };
use tokio::sync::{RwLock, Semaphore}; use tokio::sync::{RwLock, Semaphore};
use tracing::{error, info, warn}; use tracing::{debug, error, info, warn};
const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_ERASURE_HEALER: &str = "erasure_healer"; const LOG_SUBSYSTEM_ERASURE_HEALER: &str = "erasure_healer";
@@ -99,7 +99,7 @@ impl ErasureSetHealer {
/// execute erasure set heal with resume /// execute erasure set heal with resume
#[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))] #[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))]
pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> { pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> {
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE, event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -162,7 +162,7 @@ impl ErasureSetHealer {
Ok(manager) => { Ok(manager) => {
let state = manager.get_state().await; let state = manager.get_state().await;
if state.set_disk_id == set_disk_id && ResumeUtils::can_resume_task(&self.disk, &task_id).await { if state.set_disk_id == set_disk_id && ResumeUtils::can_resume_task(&self.disk, &task_id).await {
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE, event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -170,7 +170,7 @@ impl ErasureSetHealer {
task_id, task_id,
set_disk_id, set_disk_id,
state = "resume_found", state = "resume_found",
"Erasure set resume state selected" "Erasure set resume selected"
); );
return Ok(task_id); return Ok(task_id);
} }
@@ -193,7 +193,7 @@ impl ErasureSetHealer {
// create new task id // create new task id
let task_id = format!("{}_{}", set_disk_id, ResumeUtils::generate_task_id()); let task_id = format!("{}_{}", set_disk_id, ResumeUtils::generate_task_id());
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE, event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -215,7 +215,7 @@ impl ErasureSetHealer {
) -> Result<(ResumeManager, CheckpointManager)> { ) -> Result<(ResumeManager, CheckpointManager)> {
// check if resume state exists // check if resume state exists
if ResumeManager::has_resume_state(&self.disk, task_id).await { if ResumeManager::has_resume_state(&self.disk, task_id).await {
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE, event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -235,7 +235,7 @@ impl ErasureSetHealer {
Ok((resume_manager, checkpoint_manager)) Ok((resume_manager, checkpoint_manager))
} else { } else {
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE, event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -243,7 +243,7 @@ impl ErasureSetHealer {
task_id, task_id,
set_disk_id, set_disk_id,
state = "creating_new", state = "creating_new",
"Erasure set resume state creating" "Erasure set resume created"
); );
let resume_manager = ResumeManager::new( let resume_manager = ResumeManager::new(
@@ -273,7 +273,7 @@ impl ErasureSetHealer {
let state = resume_manager.get_state().await; let state = resume_manager.get_state().await;
let checkpoint = checkpoint_manager.get_checkpoint().await; let checkpoint = checkpoint_manager.get_checkpoint().await;
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE, event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -282,7 +282,7 @@ impl ErasureSetHealer {
current_bucket_index = checkpoint.current_bucket_index, current_bucket_index = checkpoint.current_bucket_index,
current_object_index = checkpoint.current_object_index, current_object_index = checkpoint.current_object_index,
state = "resuming", state = "resuming",
"Erasure set heal resumed from checkpoint" "Erasure set resumed"
); );
// 2. initialize progress // 2. initialize progress
@@ -349,7 +349,7 @@ impl ErasureSetHealer {
match bucket_result { match bucket_result {
Ok(_) => { Ok(_) => {
resume_manager.complete_bucket(bucket).await?; resume_manager.complete_bucket(bucket).await?;
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE, event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -357,7 +357,7 @@ impl ErasureSetHealer {
set_disk_id, set_disk_id,
bucket, bucket,
state = "completed", state = "completed",
"Erasure set bucket heal completed" "Erasure set bucket completed"
); );
} }
Err(e) => { Err(e) => {
@@ -383,14 +383,14 @@ impl ErasureSetHealer {
// 5. mark task completed // 5. mark task completed
resume_manager.mark_completed().await?; resume_manager.mark_completed().await?;
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE, event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
set_disk_id, set_disk_id,
state = "completed", state = "completed",
"Erasure set heal completed" "Erasure set completed"
); );
Ok(()) Ok(())
} }
@@ -411,7 +411,7 @@ impl ErasureSetHealer {
resume_manager: &ResumeManager, resume_manager: &ResumeManager,
checkpoint_manager: &CheckpointManager, checkpoint_manager: &CheckpointManager,
) -> Result<()> { ) -> Result<()> {
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE, event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -421,7 +421,7 @@ impl ErasureSetHealer {
bucket_index, bucket_index,
current_object_index = *current_object_index, current_object_index = *current_object_index,
state = "started", state = "started",
"Erasure set bucket heal started" "Erasure set bucket started"
); );
// 1. get bucket info // 1. get bucket info
@@ -584,7 +584,7 @@ impl ErasureSetHealer {
Ok(true) => { Ok(true) => {
*successful_objects += 1; *successful_objects += 1;
checkpoint_manager.add_processed_object(object.clone()).await?; checkpoint_manager.add_processed_object(object.clone()).await?;
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE, event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -593,13 +593,13 @@ impl ErasureSetHealer {
bucket, bucket,
object = %object, object = %object,
state = "healed", state = "healed",
"Erasure set object heal completed" "Erasure set object healed"
); );
} }
Ok(false) => { Ok(false) => {
checkpoint_manager.add_processed_object(object.clone()).await?; checkpoint_manager.add_processed_object(object.clone()).await?;
*successful_objects += 1; *successful_objects += 1;
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE, event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -608,7 +608,7 @@ impl ErasureSetHealer {
bucket, bucket,
object = %object, object = %object,
state = "missing_treated_as_ok", state = "missing_treated_as_ok",
"Erasure set object heal skipped because object disappeared" "Erasure set missing object treated as ok"
); );
} }
Err(Error::TaskCancelled) => { Err(Error::TaskCancelled) => {
@@ -743,14 +743,14 @@ impl ErasureSetHealer {
bucket: &str, bucket: &str,
progress: &Arc<RwLock<HealProgress>>, progress: &Arc<RwLock<HealProgress>>,
) -> Result<()> { ) -> Result<()> {
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE, event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
bucket, bucket,
state = "started", state = "started",
"Erasure set bucket heal started" "Erasure set bucket started"
); );
// 1. get bucket info // 1. get bucket info
@@ -847,7 +847,7 @@ impl ErasureSetHealer {
p.set_current_object(Some(format!("completed bucket: {bucket}"))); p.set_current_object(Some(format!("completed bucket: {bucket}")));
} }
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE, event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -857,7 +857,7 @@ impl ErasureSetHealer {
total_failed, total_failed,
total_scanned, total_scanned,
state = "completed", state = "completed",
"Erasure set bucket heal completed" "Erasure set bucket completed"
); );
Ok(()) Ok(())
@@ -890,7 +890,7 @@ impl ErasureSetHealer {
match storage.heal_object(&bucket, &object, None, &heal_opts).await { match storage.heal_object(&bucket, &object, None, &heal_opts).await {
Ok((_result, None)) => { Ok((_result, None)) => {
info!( debug!(
target: "rustfs::heal::erasure_healer", target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE, event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -898,7 +898,7 @@ impl ErasureSetHealer {
bucket, bucket,
object = %object, object = %object,
state = "healed", state = "healed",
"Erasure set object heal completed" "Erasure set object healed"
); );
Ok(()) Ok(())
} }
@@ -956,7 +956,7 @@ impl ErasureSetHealer {
success_count, success_count,
total, total,
state = "summary", state = "summary",
"Erasure set heal summary recorded" "Erasure set summary recorded"
); );
if failure_count > 0 { if failure_count > 0 {
crates/heal/src/heal/manager.rs
+56 -36
View File
@@ -558,7 +558,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER, subsystem = LOG_SUBSYSTEM_MANAGER,
state = "already_running", state = "already_running",
"Heal manager state unchanged" "Heal manager already running"
); );
return Ok(()); return Ok(());
} }
@@ -571,7 +571,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER, subsystem = LOG_SUBSYSTEM_MANAGER,
state = "starting", state = "starting",
"Heal manager state updated" "Heal manager starting"
); );
// start scheduler // start scheduler
@@ -586,7 +586,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER, subsystem = LOG_SUBSYSTEM_MANAGER,
state = "running", state = "running",
"Heal manager state updated" "Heal manager started"
); );
Ok(()) Ok(())
} }
@@ -599,7 +599,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER, subsystem = LOG_SUBSYSTEM_MANAGER,
state = "stopping", state = "stopping",
"Heal manager state updated" "Heal manager stopping"
); );
// cancel all tasks // cancel all tasks
@@ -617,7 +617,7 @@ impl HealManager {
state = "task_cancel_failed", state = "task_cancel_failed",
task_id = %task.id, task_id = %task.id,
error = %e, error = %e,
"Heal manager failed to cancel active task" "Heal active task cancellation failed"
); );
} }
} }
@@ -636,7 +636,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER, subsystem = LOG_SUBSYSTEM_MANAGER,
state = "stopped", state = "stopped",
"Heal manager state updated" "Heal manager stopped"
); );
Ok(()) Ok(())
} }
@@ -659,7 +659,7 @@ impl HealManager {
match admission { match admission {
HealAdmissionResult::Merged => { HealAdmissionResult::Merged => {
info!( debug!(
target: "rustfs::heal::manager", target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION, event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -667,7 +667,7 @@ impl HealManager {
request_id = %request.id, request_id = %request.id,
priority = ?request.priority, priority = ?request.priority,
result = "merged_duplicate", result = "merged_duplicate",
"Heal queue admission decided" "Heal queue request merged"
); );
} }
HealAdmissionResult::Dropped(reason) => { HealAdmissionResult::Dropped(reason) => {
@@ -680,7 +680,7 @@ impl HealManager {
priority = ?request.priority, priority = ?request.priority,
reason = reason.as_str(), reason = reason.as_str(),
result = "dropped_duplicate", result = "dropped_duplicate",
"Heal queue admission decided" "Heal queue request dropped"
); );
} }
HealAdmissionResult::Accepted | HealAdmissionResult::Full => {} HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
@@ -707,7 +707,7 @@ impl HealManager {
queue_len, queue_len,
queue_capacity, queue_capacity,
result = "accepted_by_displacement", result = "accepted_by_displacement",
"Heal queue admission decided" "Heal queue request accepted by displacement"
); );
drop(queue); drop(queue);
if config.event_driven_scheduler_enable { if config.event_driven_scheduler_enable {
@@ -726,7 +726,7 @@ impl HealManager {
queue_len, queue_len,
queue_capacity, queue_capacity,
result = "full_no_displacement_candidate", result = "full_no_displacement_candidate",
"Heal queue admission decided" "Heal queue request rejected without displacement"
); );
return Ok(HealAdmissionResult::Full); return Ok(HealAdmissionResult::Full);
} }
@@ -745,7 +745,7 @@ impl HealManager {
queue_capacity, queue_capacity,
reason = reason.as_str(), reason = reason.as_str(),
result = "dropped_full", result = "dropped_full",
"Heal queue admission decided" "Heal queue request dropped"
); );
} }
HealAdmissionResult::Full => { HealAdmissionResult::Full => {
@@ -759,7 +759,7 @@ impl HealManager {
queue_len, queue_len,
queue_capacity, queue_capacity,
result = "rejected_full", result = "rejected_full",
"Heal queue admission decided" "Heal queue request rejected"
); );
} }
HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {} HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {}
@@ -779,7 +779,7 @@ impl HealManager {
queue_capacity, queue_capacity,
queue_usage_pct = (queue_len * 100) / queue_capacity, queue_usage_pct = (queue_len * 100) / queue_capacity,
result = "queue_pressure_high", result = "queue_pressure_high",
"Heal queue pressure increased" "Heal queue pressure high"
); );
} }
@@ -793,7 +793,7 @@ impl HealManager {
// Log queue statistics periodically (when adding high/urgent priority items) // Log queue statistics periodically (when adding high/urgent priority items)
if matches!(priority, HealPriority::High | HealPriority::Urgent) { if matches!(priority, HealPriority::High | HealPriority::Urgent) {
let stats = queue.get_priority_stats(); let stats = queue.get_priority_stats();
info!( debug!(
target: "rustfs::heal::manager", target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION, event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -812,7 +812,7 @@ impl HealManager {
drop(queue); drop(queue);
info!( debug!(
target: "rustfs::heal::manager", target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION, event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -821,7 +821,7 @@ impl HealManager {
priority = ?priority, priority = ?priority,
queue_len = queue_len + 1, queue_len = queue_len + 1,
result = "accepted", result = "accepted",
"Heal queue admission decided" "Heal queue request accepted"
); );
if config.event_driven_scheduler_enable { if config.event_driven_scheduler_enable {
self.notify.notify_one(); self.notify.notify_one();
@@ -1119,7 +1119,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER, subsystem = LOG_SUBSYSTEM_MANAGER,
state = "shutdown", state = "shutdown",
"Heal scheduler state updated" "Heal scheduler stopped"
); );
break; break;
} }
@@ -1158,13 +1158,17 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER, subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "started", state = "started",
interval = ?duration, interval = ?duration,
"Heal auto disk scanner state updated" "Heal auto disk scanner started"
); );
tokio::spawn(async move { tokio::spawn(async move {
let mut interval = interval(duration); let mut interval = interval(duration);
loop { loop {
let mut candidate_count = 0usize;
let mut skipped_duplicate_count = 0usize;
let mut skipped_invalid_count = 0usize;
let mut enqueued_count = 0usize;
tokio::select! { tokio::select! {
_ = cancel_token.cancelled() => { _ = cancel_token.cancelled() => {
info!( info!(
@@ -1173,7 +1177,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER, subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "shutdown", state = "shutdown",
"Heal auto disk scanner state updated" "Heal auto disk scanner stopped"
); );
break; break;
} }
@@ -1185,14 +1189,15 @@ impl HealManager {
// detect unformatted disk via get_disk_id() // detect unformatted disk via get_disk_id()
match disk.get_disk_id().await { match disk.get_disk_id().await {
Err(DiskError::UnformattedDisk) => { Err(DiskError::UnformattedDisk) => {
info!( candidate_count += 1;
debug!(
target: "rustfs::heal::manager", target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_DISK, event = EVENT_HEAL_AUTO_SCAN_DISK,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER, subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
endpoint = %disk.endpoint(), endpoint = %disk.endpoint(),
disk_state = "unformatted", disk_state = "unformatted",
"Heal auto disk scanner detected candidate disk" "Heal auto-scan candidate detected"
); );
endpoints.push(disk.endpoint()); endpoints.push(disk.endpoint());
} }
@@ -1205,7 +1210,7 @@ impl HealManager {
endpoint = %disk.endpoint(), endpoint = %disk.endpoint(),
disk_state = "check_failed", disk_state = "check_failed",
error = ?e, error = ?e,
"Heal auto disk scanner failed to inspect disk" "Heal auto-scan disk inspection failed"
); );
} }
Ok(_) => { Ok(_) => {
@@ -1222,7 +1227,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER, subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "idle", state = "idle",
"Heal auto disk scanner found no candidate disks" "Heal auto disk scanner idle"
); );
continue; continue;
} }
@@ -1238,7 +1243,7 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER, subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "bucket_list_failed", state = "bucket_list_failed",
error = %e, error = %e,
"Heal auto disk scanner failed to list buckets" "Heal auto-scan bucket listing failed"
); );
continue; continue;
} }
@@ -1256,8 +1261,9 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER, subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
endpoint = %ep, endpoint = %ep,
result = "skipped_invalid_set_disk_id", result = "skipped_invalid_set_disk_id",
"Heal auto disk scanner skipped enqueue" "Heal auto-scan enqueue skipped"
); );
skipped_invalid_count += 1;
continue; continue;
}; };
// skip if already queued or healing // skip if already queued or healing
@@ -1283,7 +1289,8 @@ impl HealManager {
} }
if skip { if skip {
info!( skipped_duplicate_count += 1;
debug!(
target: "rustfs::heal::manager", target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1291,7 +1298,7 @@ impl HealManager {
endpoint = %ep, endpoint = %ep,
set_disk_id, set_disk_id,
result = "skipped_duplicate", result = "skipped_duplicate",
"Heal auto disk scanner skipped enqueue" "Heal auto-scan duplicate skipped"
); );
continue; continue;
} }
@@ -1312,7 +1319,8 @@ impl HealManager {
if config.event_driven_scheduler_enable { if config.event_driven_scheduler_enable {
notify.notify_one(); notify.notify_one();
} }
info!( enqueued_count += 1;
debug!(
target: "rustfs::heal::manager", target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1321,10 +1329,22 @@ impl HealManager {
set_disk_id, set_disk_id,
bucket_count = buckets.len(), bucket_count = buckets.len(),
result = "enqueued", result = "enqueued",
"Heal auto disk scanner enqueued task" "Heal auto-scan task enqueued"
); );
} }
} }
info!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "cycle_completed",
candidate_count,
enqueued_count,
skipped_duplicate_count,
skipped_invalid_count,
"Heal auto-scan cycle completed"
);
} }
} }
} }
@@ -1405,7 +1425,7 @@ impl HealManager {
// start heal task // start heal task
tokio::spawn(async move { tokio::spawn(async move {
info!( debug!(
target: "rustfs::heal::manager", target: "rustfs::heal::manager",
event = EVENT_HEAL_SCHEDULER_STATE, event = EVENT_HEAL_SCHEDULER_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1415,12 +1435,12 @@ impl HealManager {
heal_type = %task_type_label_for_spawn, heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn, set = %task_set_label_for_spawn,
state = "task_started", state = "task_started",
"Heal scheduler started task" "Heal scheduler task started"
); );
let result = task.execute().await; let result = task.execute().await;
match result { match result {
Ok(_) => { Ok(_) => {
info!( debug!(
target: "rustfs::heal::manager", target: "rustfs::heal::manager",
event = EVENT_HEAL_SCHEDULER_STATE, event = EVENT_HEAL_SCHEDULER_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1429,7 +1449,7 @@ impl HealManager {
heal_type = %task_type_label_for_spawn, heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn, set = %task_set_label_for_spawn,
state = "task_completed", state = "task_completed",
"Heal scheduler finished task" "Heal scheduler task completed"
); );
} }
Err(e) => { Err(e) => {
@@ -1443,7 +1463,7 @@ impl HealManager {
set = %task_set_label_for_spawn, set = %task_set_label_for_spawn,
state = "task_failed", state = "task_failed",
error = %e, error = %e,
"Heal scheduler observed task failure" "Heal scheduler task failed"
); );
} }
} }
@@ -1500,7 +1520,7 @@ impl HealManager {
queue_len = remaining, queue_len = remaining,
active_tasks = active_heals_guard.len(), active_tasks = active_heals_guard.len(),
state = "backlog_high", state = "backlog_high",
"Heal queue backlog summary recorded" "Heal queue backlog high"
); );
} }
} }
crates/heal/src/heal/resume.rs
+5 -5
View File
@@ -20,7 +20,7 @@ use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock; use tokio::sync::RwLock;
use tracing::{debug, info, warn}; use tracing::{debug, warn};
use uuid::Uuid; use uuid::Uuid;
const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_COMPONENT_HEAL: &str = "heal";
@@ -299,7 +299,7 @@ impl ResumeManager {
let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await; let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await;
} }
info!( debug!(
target: "rustfs::heal::resume", target: "rustfs::heal::resume",
event = EVENT_HEAL_RESUME_STATE, event = EVENT_HEAL_RESUME_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -524,7 +524,7 @@ impl CheckpointManager {
let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await; let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await;
} }
info!( debug!(
target: "rustfs::heal::resume", target: "rustfs::heal::resume",
event = EVENT_HEAL_CHECKPOINT_STATE, event = EVENT_HEAL_CHECKPOINT_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -649,7 +649,7 @@ impl ResumeUtils {
let age_hours = (current_time - state.last_update) / 3600; let age_hours = (current_time - state.last_update) / 3600;
if age_hours > max_age_hours { if age_hours > max_age_hours {
info!( debug!(
target: "rustfs::heal::resume", target: "rustfs::heal::resume",
event = EVENT_HEAL_RESUME_STATE, event = EVENT_HEAL_RESUME_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -657,7 +657,7 @@ impl ResumeUtils {
task_id, task_id,
age_hours, age_hours,
state = "expired_cleanup_started", state = "expired_cleanup_started",
"Heal resume state cleanup started" "Heal resume cleanup started"
); );
if let Err(e) = resume_manager.cleanup().await { if let Err(e) = resume_manager.cleanup().await {
warn!( warn!(
crates/heal/src/heal/storage.rs
+33 -33
View File
@@ -24,7 +24,7 @@ use rustfs_ecstore::{
use rustfs_madmin::heal_commands::HealResultItem; use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::{BucketInfo, DiskSetSelector, StorageAdminApi}; use rustfs_storage_api::{BucketInfo, DiskSetSelector, StorageAdminApi};
use std::sync::Arc; use std::sync::Arc;
use tracing::{debug, error, info, warn}; use tracing::{debug, error, warn};
const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_STORAGE: &str = "storage"; const LOG_SUBSYSTEM_STORAGE: &str = "storage";
@@ -206,7 +206,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket, bucket,
object, object,
result = "not_found", result = "not_found",
"Heal storage request finished" "Heal storage object metadata missing"
); );
Ok(None) Ok(None)
} else { } else {
@@ -333,7 +333,7 @@ impl HealStorageAPI for ECStoreHealStorage {
.await .await
{ {
Ok(_) => { Ok(_) => {
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_IO, event = EVENT_HEAL_STORAGE_OBJECT_IO,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -342,7 +342,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket, bucket,
object, object,
result = "ok", result = "ok",
"Heal storage request finished" "Heal storage object write completed"
); );
Ok(()) Ok(())
} }
@@ -378,7 +378,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.delete_object(bucket, object, Default::default()).await { match self.ecstore.delete_object(bucket, object, Default::default()).await {
Ok(_) => { Ok(_) => {
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_IO, event = EVENT_HEAL_STORAGE_OBJECT_IO,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -387,7 +387,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket, bucket,
object, object,
result = "ok", result = "ok",
"Heal storage request finished" "Heal storage object delete completed"
); );
Ok(()) Ok(())
} }
@@ -447,7 +447,7 @@ impl HealStorageAPI for ECStoreHealStorage {
let mut stream = reader.stream; let mut stream = reader.stream;
match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await { match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await {
Ok(_) => { Ok(_) => {
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -455,7 +455,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket, bucket,
object, object,
state = "ok", state = "ok",
"Heal storage object verification finished" "Heal storage object verified"
); );
Ok(true) Ok(true)
} }
@@ -544,7 +544,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// After healing, try to read the object data // After healing, try to read the object data
match self.get_object_data(bucket, object).await? { match self.get_object_data(bucket, object).await? {
Some(data) => { Some(data) => {
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP, event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -554,7 +554,7 @@ impl HealStorageAPI for ECStoreHealStorage {
object, object,
bytes = data.len(), bytes = data.len(),
state = "ok", state = "ok",
"Heal storage repair finished" "Heal storage EC decode rebuild completed"
); );
Ok(data) Ok(data)
} }
@@ -608,7 +608,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// TODO: implement disk status check using ecstore // TODO: implement disk status check using ecstore
// For now, return Ok status // For now, return Ok status
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP, event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -617,7 +617,7 @@ impl HealStorageAPI for ECStoreHealStorage {
endpoint = ?endpoint, endpoint = ?endpoint,
result = "ok", result = "ok",
disk_status = "ok", disk_status = "ok",
"Heal storage admin operation finished" "Heal storage disk status resolved"
); );
Ok(DiskStatus::Ok) Ok(DiskStatus::Ok)
} }
@@ -640,7 +640,7 @@ impl HealStorageAPI for ECStoreHealStorage {
if error.is_some() { if error.is_some() {
return Err(Error::other(format!("Format failed: {error:?}"))); return Err(Error::other(format!("Format failed: {error:?}")));
} }
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP, event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -648,7 +648,7 @@ impl HealStorageAPI for ECStoreHealStorage {
operation = "format_disk", operation = "format_disk",
endpoint = ?endpoint, endpoint = ?endpoint,
result = "ok", result = "ok",
"Heal storage admin operation finished" "Heal storage disk format completed"
); );
Ok(()) Ok(())
} }
@@ -726,7 +726,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.heal_bucket(bucket, &heal_opts).await { match self.heal_bucket(bucket, &heal_opts).await {
Ok(_) => { Ok(_) => {
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP, event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -734,7 +734,7 @@ impl HealStorageAPI for ECStoreHealStorage {
operation = "heal_bucket_metadata", operation = "heal_bucket_metadata",
bucket, bucket,
result = "ok", result = "ok",
"Heal storage repair finished" "Heal storage bucket metadata repaired"
); );
Ok(()) Ok(())
} }
@@ -816,7 +816,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket, bucket,
object, object,
result = "not_found", result = "not_found",
"Heal storage request finished" "Heal storage object absence confirmed"
); );
Ok(false) Ok(false)
} else if is_transient_object_exists_error(&e) { } else if is_transient_object_exists_error(&e) {
@@ -923,7 +923,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_object(bucket, object, version_id_str, opts).await { match self.ecstore.heal_object(bucket, object, version_id_str, opts).await {
Ok((result, ecstore_error)) => { Ok((result, ecstore_error)) => {
let error = ecstore_error.map(Error::other); let error = ecstore_error.map(Error::other);
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP, event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -935,7 +935,7 @@ impl HealStorageAPI for ECStoreHealStorage {
drives_after = result.after.drives.len(), drives_after = result.after.drives.len(),
has_error = error.is_some(), has_error = error.is_some(),
result = "ok", result = "ok",
"Heal storage repair finished" "Heal storage object repair completed"
); );
Ok((result, error)) Ok((result, error))
} }
@@ -974,7 +974,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_bucket(bucket, opts).await { match self.ecstore.heal_bucket(bucket, opts).await {
Ok(result) => { Ok(result) => {
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP, event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -983,7 +983,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket, bucket,
drives_after = result.after.drives.len(), drives_after = result.after.drives.len(),
result = "ok", result = "ok",
"Heal storage repair finished" "Heal storage bucket repair completed"
); );
Ok(result) Ok(result)
} }
@@ -1019,7 +1019,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_format(dry_run).await { match self.ecstore.heal_format(dry_run).await {
Ok((result, ecstore_error)) => { Ok((result, ecstore_error)) => {
let error = ecstore_error.map(Error::other); let error = ecstore_error.map(Error::other);
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP, event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1028,7 +1028,7 @@ impl HealStorageAPI for ECStoreHealStorage {
drives_after = result.after.drives.len(), drives_after = result.after.drives.len(),
has_error = error.is_some(), has_error = error.is_some(),
result = "ok", result = "ok",
"Heal storage repair finished" "Heal storage format repair completed"
); );
Ok((result, error)) Ok((result, error))
} }
@@ -1103,7 +1103,7 @@ impl HealStorageAPI for ECStoreHealStorage {
} }
} }
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP, event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1113,7 +1113,7 @@ impl HealStorageAPI for ECStoreHealStorage {
prefix, prefix,
object_count = all_objects.len(), object_count = all_objects.len(),
result = "ok", result = "ok",
"Heal storage admin operation finished" "Heal storage object listing completed"
); );
Ok(all_objects) Ok(all_objects)
} }
@@ -1175,12 +1175,12 @@ impl HealStorageAPI for ECStoreHealStorage {
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_STORAGE, subsystem = LOG_SUBSYSTEM_STORAGE,
operation = "list_objects_for_heal_page", operation = "list_objects_for_heal_page",
bucket, bucket,
prefix, prefix,
object_count = page_count, object_count = page_count,
is_truncated = list_info.is_truncated, is_truncated = list_info.is_truncated,
state = "page_loaded", state = "page_loaded",
"Heal storage admin operation finished" "Heal storage object listing page loaded"
); );
Ok((page_objects, list_info.next_continuation_token, list_info.is_truncated)) Ok((page_objects, list_info.next_continuation_token, list_info.is_truncated))
@@ -1210,7 +1210,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// Find the first available disk // Find the first available disk
if let Some(disk_store) = disks.into_iter().flatten().next() { if let Some(disk_store) = disks.into_iter().flatten().next() {
info!( debug!(
target: "rustfs::heal::storage", target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP, event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1219,7 +1219,7 @@ impl HealStorageAPI for ECStoreHealStorage {
set_disk_id, set_disk_id,
result = "ok", result = "ok",
disk = ?disk_store, disk = ?disk_store,
"Heal storage admin operation finished" "Heal storage resume disk resolved"
); );
return Ok(disk_store); return Ok(disk_store);
} }
crates/heal/src/heal/task.rs
+51 -51
View File
@@ -445,7 +445,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(), heal_type = self.heal_type.log_kind(),
state = "started", state = "started",
queue_delay = ?queue_delay, queue_delay = ?queue_delay,
"Heal task state updated" "Heal task started"
); );
let result = match &self.heal_type { let result = match &self.heal_type {
@@ -484,7 +484,7 @@ impl HealTask {
task_id = %self.id, task_id = %self.id,
heal_type = self.heal_type.log_kind(), heal_type = self.heal_type.log_kind(),
state = "completed", state = "completed",
"Heal task state updated" "Heal task completed"
); );
} }
Err(Error::TaskCancelled) => { Err(Error::TaskCancelled) => {
@@ -498,7 +498,7 @@ impl HealTask {
task_id = %self.id, task_id = %self.id,
heal_type = self.heal_type.log_kind(), heal_type = self.heal_type.log_kind(),
state = "cancelled", state = "cancelled",
"Heal task state updated" "Heal task cancelled"
); );
} }
Err(Error::TaskTimeout) => { Err(Error::TaskTimeout) => {
@@ -512,7 +512,7 @@ impl HealTask {
task_id = %self.id, task_id = %self.id,
heal_type = self.heal_type.log_kind(), heal_type = self.heal_type.log_kind(),
state = "timed_out", state = "timed_out",
"Heal task state updated" "Heal task timed out"
); );
} }
Err(e) => { Err(e) => {
@@ -527,7 +527,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(), heal_type = self.heal_type.log_kind(),
state = "failed", state = "failed",
error = %e, error = %e,
"Heal task state updated" "Heal task failed"
); );
} }
} }
@@ -539,7 +539,7 @@ impl HealTask {
self.cancel_token.cancel(); self.cancel_token.cancel();
let mut status = self.status.write().await; let mut status = self.status.write().await;
*status = HealTaskStatus::Cancelled; *status = HealTaskStatus::Cancelled;
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_TASK_STATE, event = EVENT_HEAL_TASK_STATE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -548,7 +548,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(), heal_type = self.heal_type.log_kind(),
state = "cancelled", state = "cancelled",
source = "manual", source = "manual",
"Heal task state updated" "Heal task cancellation requested"
); );
Ok(()) Ok(())
} }
@@ -572,7 +572,7 @@ impl HealTask {
// specific heal implementation method // specific heal implementation method
#[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))] #[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))]
async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE, event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -582,7 +582,7 @@ impl HealTask {
object, object,
version_id = ?version_id, version_id = ?version_id,
stage = "start", stage = "start",
"Heal object workflow started" "Heal object started"
); );
// update progress // update progress
@@ -625,7 +625,7 @@ impl HealTask {
"Heal target object is missing" "Heal target object is missing"
); );
if self.options.recreate_missing { if self.options.recreate_missing {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE, event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -634,7 +634,7 @@ impl HealTask {
bucket, bucket,
object, object,
stage = "recreate_missing", stage = "recreate_missing",
"Heal object recovery started" "Heal object recreate requested"
); );
return self.recreate_missing_object(bucket, object, version_id).await; return self.recreate_missing_object(bucket, object, version_id).await;
} else { } else {
@@ -690,7 +690,7 @@ impl HealTask {
// Check if this is a "File not found" error during delete operations // Check if this is a "File not found" error during delete operations
let error_msg = format!("{e}"); let error_msg = format!("{e}");
if error_msg.contains("File not found") || error_msg.contains("not found") { if error_msg.contains("File not found") || error_msg.contains("not found") {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT, event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -699,7 +699,7 @@ impl HealTask {
bucket, bucket,
object, object,
result = "treated_as_deleted", result = "treated_as_deleted",
"Heal object finished after target disappeared during repair" "Heal missing object treated as deleted"
); );
{ {
let mut progress = self.progress.write().await; let mut progress = self.progress.write().await;
@@ -723,7 +723,7 @@ impl HealTask {
// If heal failed and remove_corrupted is enabled, delete the corrupted object // If heal failed and remove_corrupted is enabled, delete the corrupted object
if self.options.remove_corrupted { if self.options.remove_corrupted {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP, event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -737,7 +737,7 @@ impl HealTask {
); );
if !self.options.dry_run { if !self.options.dry_run {
self.await_with_control(self.storage.delete_object(bucket, object)).await?; self.await_with_control(self.storage.delete_object(bucket, object)).await?;
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP, event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -747,7 +747,7 @@ impl HealTask {
object, object,
action = "delete_corrupted_object", action = "delete_corrupted_object",
result = "deleted", result = "deleted",
"Heal object cleanup completed" "Heal corrupted object deleted"
); );
} }
} }
@@ -775,7 +775,7 @@ impl HealTask {
"Heal object stage entered" "Heal object stage entered"
); );
let object_size = result.object_size as u64; let object_size = result.object_size as u64;
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT, event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -786,7 +786,7 @@ impl HealTask {
object_size = object_size, object_size = object_size,
drives_healed = result.after.drives.len(), drives_healed = result.after.drives.len(),
result = "ok", result = "ok",
"Heal object completed" "Heal object repaired"
); );
{ {
@@ -806,7 +806,7 @@ impl HealTask {
// Check if this is a "File not found" error during delete operations // Check if this is a "File not found" error during delete operations
let error_msg = format!("{e}"); let error_msg = format!("{e}");
if error_msg.contains("File not found") || error_msg.contains("not found") { if error_msg.contains("File not found") || error_msg.contains("not found") {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT, event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -815,7 +815,7 @@ impl HealTask {
bucket, bucket,
object, object,
result = "treated_as_deleted", result = "treated_as_deleted",
"Heal object finished after target disappeared during repair" "Heal missing object treated as deleted"
); );
{ {
let mut progress = self.progress.write().await; let mut progress = self.progress.write().await;
@@ -839,7 +839,7 @@ impl HealTask {
// If heal failed and remove_corrupted is enabled, delete the corrupted object // If heal failed and remove_corrupted is enabled, delete the corrupted object
if self.options.remove_corrupted { if self.options.remove_corrupted {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP, event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -853,7 +853,7 @@ impl HealTask {
); );
if !self.options.dry_run { if !self.options.dry_run {
self.await_with_control(self.storage.delete_object(bucket, object)).await?; self.await_with_control(self.storage.delete_object(bucket, object)).await?;
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP, event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -863,7 +863,7 @@ impl HealTask {
object, object,
action = "delete_corrupted_object", action = "delete_corrupted_object",
result = "deleted", result = "deleted",
"Heal object cleanup completed" "Heal corrupted object deleted"
); );
} }
} }
@@ -882,7 +882,7 @@ impl HealTask {
/// Recreate missing object (for EC decode scenarios) /// Recreate missing object (for EC decode scenarios)
async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE, event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -892,7 +892,7 @@ impl HealTask {
object, object,
version_id = ?version_id, version_id = ?version_id,
stage = "recreate_missing", stage = "recreate_missing",
"Heal object recovery started" "Heal object recreate started"
); );
// Use ecstore's heal_object with recreate option // Use ecstore's heal_object with recreate option
@@ -932,7 +932,7 @@ impl HealTask {
} }
let object_size = result.object_size as u64; let object_size = result.object_size as u64;
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT, event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -942,7 +942,7 @@ impl HealTask {
object, object,
object_size, object_size,
result = "recreated", result = "recreated",
"Heal object recovery finished" "Heal object recreated"
); );
{ {
@@ -975,7 +975,7 @@ impl HealTask {
} }
async fn heal_bucket(&self, bucket: &str) -> Result<()> { async fn heal_bucket(&self, bucket: &str) -> Result<()> {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_STAGE, event = EVENT_HEAL_BUCKET_STAGE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -984,7 +984,7 @@ impl HealTask {
bucket, bucket,
stage = "start", stage = "start",
recursive = self.options.recursive, recursive = self.options.recursive,
"Heal bucket workflow started" "Heal bucket started"
); );
// update progress // update progress
@@ -1060,7 +1060,7 @@ impl HealTask {
match heal_result { match heal_result {
Ok(result) => { Ok(result) => {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT, event = EVENT_HEAL_BUCKET_RESULT,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1269,7 +1269,7 @@ impl HealTask {
}); });
} }
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT, event = EVENT_HEAL_BUCKET_RESULT,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1287,7 +1287,7 @@ impl HealTask {
} }
async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> { async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_METADATA_STAGE, event = EVENT_HEAL_METADATA_STAGE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1296,7 +1296,7 @@ impl HealTask {
bucket, bucket,
object, object,
stage = "start", stage = "start",
"Heal metadata workflow started" "Heal metadata started"
); );
// update progress // update progress
@@ -1400,7 +1400,7 @@ impl HealTask {
}); });
} }
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_METADATA_RESULT, event = EVENT_HEAL_METADATA_RESULT,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1410,7 +1410,7 @@ impl HealTask {
object, object,
drives_healed = result.after.drives.len(), drives_healed = result.after.drives.len(),
result = "ok", result = "ok",
"Heal metadata completed" "Heal metadata repaired"
); );
{ {
@@ -1447,7 +1447,7 @@ impl HealTask {
} }
async fn heal_mrf(&self, meta_path: &str) -> Result<()> { async fn heal_mrf(&self, meta_path: &str) -> Result<()> {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_MRF_STAGE, event = EVENT_HEAL_MRF_STAGE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1455,7 +1455,7 @@ impl HealTask {
task_id = %self.id, task_id = %self.id,
meta_path, meta_path,
stage = "start", stage = "start",
"Heal MRF workflow started" "Heal MRF started"
); );
// update progress // update progress
@@ -1530,7 +1530,7 @@ impl HealTask {
}); });
} }
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_MRF_RESULT, event = EVENT_HEAL_MRF_RESULT,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1541,7 +1541,7 @@ impl HealTask {
object = %object, object = %object,
drives_healed = result.after.drives.len(), drives_healed = result.after.drives.len(),
result = "ok", result = "ok",
"Heal MRF completed" "Heal MRF repaired"
); );
{ {
@@ -1579,7 +1579,7 @@ impl HealTask {
} }
async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_EC_DECODE_STAGE, event = EVENT_HEAL_EC_DECODE_STAGE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1589,7 +1589,7 @@ impl HealTask {
object, object,
version_id = ?version_id, version_id = ?version_id,
stage = "start", stage = "start",
"Heal EC decode workflow started" "Heal EC decode started"
); );
// update progress // update progress
@@ -1694,7 +1694,7 @@ impl HealTask {
} }
let object_size = result.object_size as u64; let object_size = result.object_size as u64;
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_EC_DECODE_RESULT, event = EVENT_HEAL_EC_DECODE_RESULT,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1705,7 +1705,7 @@ impl HealTask {
object_size, object_size,
drives_healed = result.after.drives.len(), drives_healed = result.after.drives.len(),
result = "ok", result = "ok",
"Heal EC decode completed" "Heal EC decode repaired"
); );
{ {
@@ -1742,7 +1742,7 @@ impl HealTask {
} }
async fn heal_erasure_set(&self, buckets: Vec<String>, set_disk_id: String) -> Result<()> { async fn heal_erasure_set(&self, buckets: Vec<String>, set_disk_id: String) -> Result<()> {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_STAGE, event = EVENT_HEAL_ERASURE_SET_STAGE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1751,7 +1751,7 @@ impl HealTask {
set_disk_id, set_disk_id,
bucket_count = buckets.len(), bucket_count = buckets.len(),
stage = "start", stage = "start",
"Heal erasure set workflow started" "Heal erasure set started"
); );
// update progress // update progress
@@ -1762,7 +1762,7 @@ impl HealTask {
} }
let buckets = if buckets.is_empty() { let buckets = if buckets.is_empty() {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_STAGE, event = EVENT_HEAL_ERASURE_SET_STAGE,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1770,7 +1770,7 @@ impl HealTask {
task_id = %self.id, task_id = %self.id,
set_disk_id, set_disk_id,
stage = "list_buckets", stage = "list_buckets",
"Heal erasure set resolved bucket list from storage" "Heal erasure set bucket list resolved"
); );
let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?; let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
bucket_infos.into_iter().map(|info| info.name).collect() bucket_infos.into_iter().map(|info| info.name).collect()
@@ -1814,7 +1814,7 @@ impl HealTask {
}); });
} }
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_RESULT, event = EVENT_HEAL_ERASURE_SET_RESULT,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1823,7 +1823,7 @@ impl HealTask {
set_disk_id, set_disk_id,
drives_healed = result.after.drives.len(), drives_healed = result.after.drives.len(),
result = "format_ok", result = "format_ok",
"Heal erasure set format repair completed" "Heal erasure set format repaired"
); );
} }
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled), Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
@@ -1970,7 +1970,7 @@ impl HealTask {
match result { match result {
Ok(_) => { Ok(_) => {
info!( debug!(
target: "rustfs::heal::task", target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_RESULT, event = EVENT_HEAL_ERASURE_SET_RESULT,
component = LOG_COMPONENT_HEAL, component = LOG_COMPONENT_HEAL,
@@ -1979,7 +1979,7 @@ impl HealTask {
set_disk_id, set_disk_id,
bucket_count = buckets.len(), bucket_count = buckets.len(),
result = "ok", result = "ok",
"Heal erasure set completed" "Heal erasure set repaired"
); );
Ok(()) Ok(())
} }
crates/iam/src/lib.rs
+79 -11
View File
@@ -19,7 +19,13 @@ use rustfs_ecstore::store::ECStore;
use std::sync::{Arc, OnceLock}; use std::sync::{Arc, OnceLock};
use store::object::ObjectStore; use store::object::ObjectStore;
use sys::IamSys; use sys::IamSys;
use tracing::{error, info, instrument, warn}; use tracing::{debug, error, info, instrument, warn};
const LOG_COMPONENT_IAM: &str = "iam";
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
const LOG_SUBSYSTEM_OIDC: &str = "oidc";
const EVENT_IAM_STATE: &str = "iam_state";
const EVENT_OIDC_STATE: &str = "oidc_state";
pub mod cache; pub mod cache;
pub mod error; pub mod error;
@@ -37,11 +43,23 @@ static OIDC_SYS: OnceLock<Arc<OidcSys>> = OnceLock::new();
#[instrument(skip(ecstore))] #[instrument(skip(ecstore))]
pub async fn init_iam_sys(ecstore: Arc<ECStore>) -> Result<()> { pub async fn init_iam_sys(ecstore: Arc<ECStore>) -> Result<()> {
if IAM_SYS.get().is_some() { if IAM_SYS.get().is_some() {
info!("IAM system already initialized, skipping."); info!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "already_initialized",
"IAM runtime already initialized"
);
return Ok(()); return Ok(());
} }
info!("Starting IAM system initialization sequence..."); info!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "starting",
"IAM runtime starting"
);
// 1. Create the persistent storage adapter // 1. Create the persistent storage adapter
let storage_adapter = ObjectStore::new(ecstore); let storage_adapter = ObjectStore::new(ecstore);
@@ -55,11 +73,23 @@ pub async fn init_iam_sys(ecstore: Arc<ECStore>) -> Result<()> {
// 4. Securely set the global singleton // 4. Securely set the global singleton
if IAM_SYS.set(iam_instance).is_err() { if IAM_SYS.set(iam_instance).is_err() {
error!("Critical: Race condition detected during IAM initialization!"); error!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "singleton_set_failed",
"IAM runtime singleton set failed"
);
return Err(Error::IamSysAlreadyInitialized); return Err(Error::IamSysAlreadyInitialized);
} }
info!("IAM system initialization completed successfully."); info!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "ready",
"IAM runtime ready"
);
Ok(()) Ok(())
} }
@@ -84,29 +114,67 @@ pub fn get_global_iam_sys() -> Option<Arc<IamSys<ObjectStore>>> {
/// Initialize the global OIDC system. Non-fatal if no OIDC providers are configured. /// Initialize the global OIDC system. Non-fatal if no OIDC providers are configured.
pub async fn init_oidc_sys() -> Result<()> { pub async fn init_oidc_sys() -> Result<()> {
if OIDC_SYS.get().is_some() { if OIDC_SYS.get().is_some() {
info!("OIDC system already initialized, skipping."); debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "already_initialized",
"OIDC runtime already initialized"
);
return Ok(()); return Ok(());
} }
info!("Starting OIDC system initialization..."); debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "starting",
"OIDC runtime starting"
);
let oidc_sys = match OidcSys::new().await { let oidc_sys = match OidcSys::new().await {
Ok(sys) => { Ok(sys) => {
if sys.has_providers() { if sys.has_providers() {
info!("OIDC system initialized with {} provider(s)", sys.list_providers().len()); debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
provider_count = sys.list_providers().len(),
state = "ready",
"OIDC runtime ready"
);
} else { } else {
info!("No OIDC providers configured"); debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "empty",
"OIDC runtime has no providers"
);
} }
sys sys
} }
Err(e) => { Err(e) => {
warn!("OIDC initialization failed (non-fatal): {}", e); warn!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "init_failed_non_fatal",
error = %e,
"OIDC runtime initialization failed"
);
OidcSys::empty().map_err(Error::StringError)? OidcSys::empty().map_err(Error::StringError)?
} }
}; };
if OIDC_SYS.set(Arc::new(oidc_sys)).is_err() { if OIDC_SYS.set(Arc::new(oidc_sys)).is_err() {
warn!("Race condition during OIDC initialization (non-fatal)"); warn!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "singleton_set_race",
"OIDC runtime singleton set raced"
);
} }
Ok(()) Ok(())
crates/iam/src/manager.rs
+14 -10
View File
@@ -54,7 +54,7 @@ use tokio::{
}, },
}; };
use tracing::warn; use tracing::warn;
use tracing::{error, info}; use tracing::{debug, error};
const IAM_FORMAT_FILE: &str = "format.json"; const IAM_FORMAT_FILE: &str = "format.json";
const IAM_FORMAT_VERSION_1: i32 = 1; const IAM_FORMAT_VERSION_1: i32 = 1;
@@ -153,10 +153,14 @@ where
if let Err(e) = self.clone().load().await { if let Err(e) = self.clone().load().await {
if attempt == MAX_RETRIES - 1 { if attempt == MAX_RETRIES - 1 {
self.state.store(IamState::Error as u8, Ordering::SeqCst); self.state.store(IamState::Error as u8, Ordering::SeqCst);
warn!("IAM failed to load initial data after {} attempts: {:?}", MAX_RETRIES, e); warn!(
attempts = MAX_RETRIES,
error = ?e,
"IAM initial load failed"
);
load_error = Some(e); load_error = Some(e);
} else { } else {
warn!("IAM load failed, retrying... attempt {}", attempt + 1); warn!(attempt = attempt + 1, max_attempts = MAX_RETRIES, "IAM load retry scheduled");
tokio::time::sleep(INITIAL_LOAD_RETRY_DELAY).await; tokio::time::sleep(INITIAL_LOAD_RETRY_DELAY).await;
} }
} else { } else {
@@ -169,7 +173,7 @@ where
} }
self.state.store(IamState::Ready as u8, Ordering::SeqCst); self.state.store(IamState::Ready as u8, Ordering::SeqCst);
info!("IAM System successfully initialized and marked as READY"); debug!(state = "ready", "IAM manager ready");
// Background ticker for synchronization // Background ticker for synchronization
// Check if environment variable is set // Check if environment variable is set
@@ -185,20 +189,20 @@ where
loop { loop {
select! { select! {
_ = ticker.tick() => { _ = ticker.tick() => {
info!("iam load ticker"); debug!(source = "ticker", "IAM reload tick");
if let Err(err) =s.clone().load().await{ if let Err(err) =s.clone().load().await{
warn!("iam load err {:?}", err); warn!(source = "ticker", error = ?err, "IAM reload failed");
} }
}, },
i = receiver.recv() => { i = receiver.recv() => {
info!("iam load receiver"); debug!(source = "receiver", "IAM reload signal received");
match i { match i {
Some(t) => { Some(t) => {
let last = s.last_timestamp.load(Ordering::Relaxed); let last = s.last_timestamp.load(Ordering::Relaxed);
if last <= t { if last <= t {
info!("iam load receiver load"); debug!(source = "receiver", "IAM reload accepted");
if let Err(err) =s.clone().load().await{ if let Err(err) =s.clone().load().await{
warn!("iam load err {:?}", err); warn!(source = "receiver", error = ?err, "IAM reload failed");
} }
ticker.reset(); ticker.reset();
} }
@@ -1294,7 +1298,7 @@ where
let cache = self.cache.snapshot(); let cache = self.cache.snapshot();
let users = Arc::clone(&cache.users); let users = Arc::clone(&cache.users);
if let Some(x) = users.get(access_key) { if let Some(x) = users.get(access_key) {
warn!("user already exists: {:?}", x); warn!(error = ?x, "IAM user already exists");
if x.credentials.is_temp() { if x.credentials.is_temp() {
return Err(Error::IAMActionNotAllowed); return Err(Error::IAMActionNotAllowed);
} }
crates/iam/src/oidc.rs
+5 -5
View File
@@ -40,7 +40,7 @@ use std::pin::Pin;
use std::sync::{LazyLock, Mutex, MutexGuard, RwLock}; use std::sync::{LazyLock, Mutex, MutexGuard, RwLock};
use std::time::{Duration as StdDuration, Instant}; use std::time::{Duration as StdDuration, Instant};
use tokio::time::sleep; use tokio::time::sleep;
use tracing::{error, info, warn}; use tracing::{debug, error, warn};
use url::Url; use url::Url;
const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60); const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
@@ -76,7 +76,7 @@ fn lock_oidc_plugin_authn_metrics<'a, T>(mutex: &'a Mutex<T>, metric: &'static s
match mutex.lock() { match mutex.lock() {
Ok(guard) => guard, Ok(guard) => guard,
Err(err) => { Err(err) => {
warn!("recovering poisoned OIDC plugin authn metrics lock: {}", metric); warn!(metric, "Recovering poisoned OIDC authn metrics lock");
err.into_inner() err.into_inner()
} }
} }
@@ -417,18 +417,18 @@ impl OidcSys {
for sourced_config in parsed_configs { for sourced_config in parsed_configs {
let config = sourced_config.config; let config = sourced_config.config;
if !config.enabled { if !config.enabled {
info!("OIDC provider '{}' is disabled, skipping", config.id); debug!(provider = %config.id, "OIDC provider disabled");
continue; continue;
} }
match Self::discover_provider(&config, &http_client).await { match Self::discover_provider(&config, &http_client).await {
Ok(state) => { Ok(state) => {
info!("OIDC provider '{}' discovered successfully", config.id); debug!(provider = %config.id, "OIDC provider discovered");
provider_states.insert(config.id.clone(), state); provider_states.insert(config.id.clone(), state);
configs.insert(config.id.clone(), config); configs.insert(config.id.clone(), config);
} }
Err(e) => { Err(e) => {
error!("Failed to discover OIDC provider '{}': {}", config.id, e); error!(provider = %config.id, error = %e, "OIDC provider discovery failed");
} }
} }
} }
crates/iam/src/store/object.rs
+31 -33
View File
@@ -41,7 +41,7 @@ use std::time::{Duration, Instant};
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use tokio::sync::mpsc::{self, Sender}; use tokio::sync::mpsc::{self, Sender};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn}; use tracing::{debug, error, warn};
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam")); pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam"));
pub static IAM_CONFIG_USERS_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam/users/")); pub static IAM_CONFIG_USERS_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam/users/"));
@@ -307,11 +307,11 @@ impl ObjectStore {
} }
Err(StorageError::PreconditionFailed) => { Err(StorageError::PreconditionFailed) => {
Self::complete_lazy_rewrite(path.as_str(), false); Self::complete_lazy_rewrite(path.as_str(), false);
debug!("iam lazy rewrite skipped due to stale etag, path: {}", path); debug!(path = %path, state = "stale_etag", "IAM lazy rewrite skipped");
} }
Err(err) => { Err(err) => {
Self::complete_lazy_rewrite(path.as_str(), false); Self::complete_lazy_rewrite(path.as_str(), false);
warn!("iam lazy rewrite failed, path: {}, err: {}", path, err); warn!(path = %path, error = %err, state = "rewrite_failed", "IAM lazy rewrite failed");
} }
} }
}); });
@@ -377,7 +377,7 @@ impl ObjectStore {
bucket = Self::BUCKET_NAME, bucket = Self::BUCKET_NAME,
prefix = %path, prefix = %path,
error = %err, error = %err,
"system path walk failed" "IAM config walk failed"
); );
let _ = sender_on_error let _ = sender_on_error
.send(StringOrErr { .send(StringOrErr {
@@ -643,7 +643,7 @@ impl Store for ObjectStore {
let outcome = match Self::decrypt_data_with_source(&data) { let outcome = match Self::decrypt_data_with_source(&data) {
Ok(v) => v, Ok(v) => v,
Err(err) => { Err(err) => {
warn!("config decrypt failed, keeping file: {}, path: {}", err, path_ref); warn!(path = %path_ref, error = %err, "IAM config decrypt failed; keeping file");
// keep the config file when decrypt failed - do not delete // keep the config file when decrypt failed - do not delete
return Err(Error::ConfigNotFound); return Err(Error::ConfigNotFound);
} }
@@ -692,7 +692,7 @@ impl Store for ObjectStore {
tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await; tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await;
} }
Err(e) => { Err(e) => {
error!("Final failure saving IAM config to {}: {:?}", path_ref, e); error!(path = %path_ref, error = ?e, "IAM config save failed");
return Err(e.into()); return Err(e.into());
} }
} }
@@ -717,7 +717,7 @@ impl Store for ObjectStore {
debug!("Saving IAM identity to path: {}", path); debug!("Saving IAM identity to path: {}", path);
self.save_iam_config(user_identity, path).await.map_err(|e| { self.save_iam_config(user_identity, path).await.map_err(|e| {
error!("ObjectStore save failure for {}: {:?}", name, e); error!(name, error = ?e, "IAM identity save failed");
e e
}) })
} }
@@ -739,10 +739,10 @@ impl Store for ObjectStore {
.await .await
.map_err(|err| { .map_err(|err| {
if is_err_config_not_found(&err) { if is_err_config_not_found(&err) {
warn!("load_user_identity failed: no such user, name: {name}, user_type: {user_type:?}"); warn!(name, user_type = ?user_type, "IAM user identity missing");
Error::NoSuchUser(name.to_owned()) Error::NoSuchUser(name.to_owned())
} else { } else {
warn!("load_user_identity failed: {err:?}, name: {name}, user_type: {user_type:?}"); warn!(name, user_type = ?user_type, error = ?err, "IAM user identity load failed");
err err
} }
})?; })?;
@@ -750,9 +750,7 @@ impl Store for ObjectStore {
if u.credentials.is_expired() { if u.credentials.is_expired() {
let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await; let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await; let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
warn!( warn!(name, user_type = ?user_type, "IAM user identity expired and was removed");
"load_user_identity failed: user is expired, delete the user and mapped policy, name: {name}, user_type: {user_type:?}"
);
return Err(Error::NoSuchUser(name.to_owned())); return Err(Error::NoSuchUser(name.to_owned()));
} }
@@ -776,7 +774,7 @@ impl Store for ObjectStore {
let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await; let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await; let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
} }
warn!("extract_jwt_claims failed: {err:?}, name: {name}, user_type: {user_type:?}"); warn!(name, user_type = ?user_type, error = ?err, "IAM JWT claim extraction failed");
return Err(Error::NoSuchUser(name.to_owned())); return Err(Error::NoSuchUser(name.to_owned()));
} }
} }
@@ -804,7 +802,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await { while let Some(v) = rx.recv().await {
if let Some(err) = v.err { if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err); warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel(); let _ = ctx.cancel();
return Err(err); return Err(err);
@@ -866,7 +864,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await { while let Some(v) = rx.recv().await {
if let Some(err) = v.err { if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err); warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel(); let _ = ctx.cancel();
return Err(err); return Err(err);
@@ -934,7 +932,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await { while let Some(v) = rx.recv().await {
if let Some(err) = v.err { if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err); warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel(); let _ = ctx.cancel();
return Err(err); return Err(err);
@@ -1009,7 +1007,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await { while let Some(v) = rx.recv().await {
if let Some(err) = v.err { if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err); warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel(); let _ = ctx.cancel();
return Err(err); return Err(err);
@@ -1044,7 +1042,7 @@ impl Store for ObjectStore {
let policy_name = rustfs_utils::path::dir(&policies_list[idx]); let policy_name = rustfs_utils::path::dir(&policies_list[idx]);
info!("load policy: {}", policy_name); debug!(policy = %policy_name, "IAM policy loaded");
policy_docs_cache.insert(policy_name, p); policy_docs_cache.insert(policy_name, p);
} }
@@ -1059,7 +1057,7 @@ impl Store for ObjectStore {
} }
let policy_name = rustfs_utils::path::dir(&policies_list[idx]); let policy_name = rustfs_utils::path::dir(&policies_list[idx]);
info!("load policy: {}", policy_name); debug!(policy = %policy_name, "IAM policy loaded");
policy_docs_cache.insert(policy_name, p); policy_docs_cache.insert(policy_name, p);
} }
@@ -1083,7 +1081,7 @@ impl Store for ObjectStore {
} }
let name = rustfs_utils::path::dir(&item_name_list[idx]); let name = rustfs_utils::path::dir(&item_name_list[idx]);
info!("load reg user: {}", name); debug!(user = %name, "IAM regular user loaded");
user_items_cache.insert(name, p); user_items_cache.insert(name, p);
} }
break; break;
@@ -1097,7 +1095,7 @@ impl Store for ObjectStore {
} }
let name = rustfs_utils::path::dir(&item_name_list[idx]); let name = rustfs_utils::path::dir(&item_name_list[idx]);
info!("load reg user: {}", name); debug!(user = %name, "IAM regular user loaded");
user_items_cache.insert(name, p); user_items_cache.insert(name, p);
} }
@@ -1112,7 +1110,7 @@ impl Store for ObjectStore {
for item in item_name_list.iter() { for item in item_name_list.iter() {
let name = rustfs_utils::path::dir(item); let name = rustfs_utils::path::dir(item);
info!("load group: {}", name); debug!(group = %name, "IAM group loaded");
if let Err(err) = self.load_group(&name, &mut items_cache).await { if let Err(err) = self.load_group(&name, &mut items_cache).await {
return Err(Error::other(format!("load group failed: {err}"))); return Err(Error::other(format!("load group failed: {err}")));
}; };
@@ -1140,7 +1138,7 @@ impl Store for ObjectStore {
} }
let name = item_name_list[idx].trim_end_matches(".json").to_owned(); let name = item_name_list[idx].trim_end_matches(".json").to_owned();
info!("load user policy: {}", name); debug!(user = %name, "IAM user policy loaded");
items_cache.insert(name, p); items_cache.insert(name, p);
} }
break; break;
@@ -1156,7 +1154,7 @@ impl Store for ObjectStore {
} }
let name = item_name_list[idx].trim_end_matches(".json").to_owned(); let name = item_name_list[idx].trim_end_matches(".json").to_owned();
info!("load user policy: {}", name); debug!(user = %name, "IAM user policy loaded");
items_cache.insert(name, p); items_cache.insert(name, p);
} }
@@ -1174,7 +1172,7 @@ impl Store for ObjectStore {
for item in item_name_list.iter() { for item in item_name_list.iter() {
let name = item.trim_end_matches(".json"); let name = item.trim_end_matches(".json");
info!("load group policy: {}", name); debug!(group = %name, "IAM group policy loaded");
if let Err(err) = self.load_mapped_policy(name, UserType::Reg, true, &mut items_cache).await if let Err(err) = self.load_mapped_policy(name, UserType::Reg, true, &mut items_cache).await
&& !is_err_no_such_policy(&err) && !is_err_no_such_policy(&err)
{ {
@@ -1193,7 +1191,7 @@ impl Store for ObjectStore {
for item in item_name_list.iter() { for item in item_name_list.iter() {
let name = rustfs_utils::path::dir(item); let name = rustfs_utils::path::dir(item);
info!("load svc user: {}", name); debug!(user = %name, "IAM service user loaded");
if let Err(err) = self.load_user(&name, UserType::Svc, &mut items_cache).await if let Err(err) = self.load_user(&name, UserType::Svc, &mut items_cache).await
&& !is_err_no_such_user(&err) && !is_err_no_such_user(&err)
{ {
@@ -1204,7 +1202,7 @@ impl Store for ObjectStore {
for (_, v) in items_cache.iter() { for (_, v) in items_cache.iter() {
let parent = v.credentials.parent_user.clone(); let parent = v.credentials.parent_user.clone();
if !user_items_cache.contains_key(&parent) { if !user_items_cache.contains_key(&parent) {
info!("load sts user policy: {}", parent); debug!(user = %parent, "IAM STS parent policy loaded");
if let Err(err) = self if let Err(err) = self
.load_mapped_policy(&parent, UserType::Sts, false, &mut sts_policies_cache) .load_mapped_policy(&parent, UserType::Sts, false, &mut sts_policies_cache)
.await .await
@@ -1223,12 +1221,12 @@ impl Store for ObjectStore {
// sts users // sts users
if let Some(item_name_list) = listed_config_items.get(STS_LIST_KEY) { if let Some(item_name_list) = listed_config_items.get(STS_LIST_KEY) {
for item in item_name_list.iter() { for item in item_name_list.iter() {
info!("load sts user path: {}", item); debug!(path = %item, "IAM STS user path discovered");
let name = rustfs_utils::path::dir(item); let name = rustfs_utils::path::dir(item);
info!("load sts user: {}", name); debug!(user = %name, "IAM STS user loaded");
if let Err(err) = self.load_user(&name, UserType::Sts, &mut sts_items_cache).await { if let Err(err) = self.load_user(&name, UserType::Sts, &mut sts_items_cache).await {
info!("load sts user failed: {}", err); debug!(user = %name, error = %err, "IAM STS user load failed");
}; };
} }
} }
@@ -1237,12 +1235,12 @@ impl Store for ObjectStore {
if let Some(item_name_list) = listed_config_items.get(POLICY_DB_STS_USERS_LIST_KEY) { if let Some(item_name_list) = listed_config_items.get(POLICY_DB_STS_USERS_LIST_KEY) {
for item in item_name_list.iter() { for item in item_name_list.iter() {
let name = item.trim_end_matches(".json"); let name = item.trim_end_matches(".json");
info!("load sts user policy: {}", name); debug!(user = %name, "IAM STS user policy loaded");
if let Err(err) = self if let Err(err) = self
.load_mapped_policy(name, UserType::Sts, false, &mut sts_policies_cache) .load_mapped_policy(name, UserType::Sts, false, &mut sts_policies_cache)
.await .await
{ {
info!("load sts user policy failed: {}", err); debug!(user = %name, error = %err, "IAM STS user policy load failed");
}; };
} }
} }
@@ -1264,7 +1262,7 @@ impl Store for ObjectStore {
cache.replace_sts_policies(sts_policies_cache); cache.replace_sts_policies(sts_policies_cache);
cache.build_user_group_memberships(); cache.build_user_group_memberships();
} else { } else {
warn!("skip IAM full reload cache commit because one or more IAM caches changed during reload"); warn!("IAM full reload cache commit skipped due to concurrent cache changes");
} }
}); });
crates/kms/src/backends/local.rs
+12 -12
View File
@@ -34,7 +34,7 @@ use std::path::PathBuf;
use std::time::Duration; use std::time::Duration;
use tokio::fs; use tokio::fs;
use tokio::sync::RwLock; use tokio::sync::RwLock;
use tracing::{debug, info, warn}; use tracing::{debug, warn};
/// Local KMS client that stores keys in local files /// Local KMS client that stores keys in local files
pub struct LocalKmsClient { pub struct LocalKmsClient {
@@ -74,7 +74,7 @@ impl LocalKmsClient {
// Create key directory if it doesn't exist // Create key directory if it doesn't exist
if !config.key_dir.exists() { if !config.key_dir.exists() {
fs::create_dir_all(&config.key_dir).await?; fs::create_dir_all(&config.key_dir).await?;
info!("Created KMS key directory: {:?}", config.key_dir); debug!(path = ?config.key_dir, "KMS key directory created");
} }
// Initialize master cipher if master key is provided // Initialize master cipher if master key is provided
@@ -219,7 +219,7 @@ impl LocalKmsClient {
fs::rename(&temp_path, &key_path).await?; fs::rename(&temp_path, &key_path).await?;
info!("Saved master key {} to {:?}", master_key.key_id, key_path); debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
Ok(()) Ok(())
} }
@@ -278,7 +278,7 @@ impl KmsClient for LocalKmsClient {
let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone()); let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
info!("Generated data key for master key: {}", request.master_key_id); debug!(key_id = %request.master_key_id, "Local KMS data key generated");
Ok(data_key) Ok(data_key)
} }
@@ -334,7 +334,7 @@ impl KmsClient for LocalKmsClient {
.decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce) .decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce)
.await?; .await?;
info!("Successfully decrypted data"); debug!("Local KMS data decrypted");
Ok(plaintext) Ok(plaintext)
} }
@@ -367,7 +367,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await; let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone()); cache.insert(key_id.to_string(), master_key.clone());
info!("Created master key: {}", key_id); debug!(key_id, "Local KMS master key created");
Ok(master_key) Ok(master_key)
} }
@@ -453,7 +453,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await; let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key); cache.insert(key_id.to_string(), master_key);
info!("Enabled key: {}", key_id); debug!(key_id, "Local KMS key enabled");
Ok(()) Ok(())
} }
@@ -470,7 +470,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await; let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key); cache.insert(key_id.to_string(), master_key);
info!("Disabled key: {}", key_id); debug!(key_id, "Local KMS key disabled");
Ok(()) Ok(())
} }
@@ -492,7 +492,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await; let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key); cache.insert(key_id.to_string(), master_key);
warn!("Scheduled key deletion: {}", key_id); debug!(key_id, "Local KMS key deletion scheduled");
Ok(()) Ok(())
} }
@@ -509,7 +509,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await; let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key); cache.insert(key_id.to_string(), master_key);
info!("Canceled deletion for key: {}", key_id); debug!(key_id, "Local KMS key deletion canceled");
Ok(()) Ok(())
} }
@@ -528,7 +528,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await; let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone()); cache.insert(key_id.to_string(), master_key.clone());
info!("Rotated key: {}", key_id); debug!(key_id, "Local KMS key rotated");
Ok(master_key) Ok(master_key)
} }
@@ -721,7 +721,7 @@ impl KmsBackend for LocalKmsBackend {
let mut cache = self.client.key_cache.write().await; let mut cache = self.client.key_cache.write().await;
cache.remove(key_id); cache.remove(key_id);
info!("Immediately deleted key: {}", key_id); debug!(key_id, "Local KMS key deleted immediately");
// Return success response for immediate deletion // Return success response for immediate deletion
let key_metadata = KeyMetadata { let key_metadata = KeyMetadata {
crates/kms/src/backends/vault.rs
+13 -13
View File
@@ -98,7 +98,7 @@ impl VaultKmsClient {
let client = let client =
VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}")))?; VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}")))?;
info!("Successfully connected to Vault at {}", config.address); info!(address = %config.address, "Vault KMS backend connected");
Ok(Self { Ok(Self {
client, client,
@@ -136,7 +136,7 @@ impl VaultKmsClient {
// If encrypted_key_material is empty, generate and store it (fix for old keys) // If encrypted_key_material is empty, generate and store it (fix for old keys)
if key_data.encrypted_key_material.is_empty() { if key_data.encrypted_key_material.is_empty() {
warn!("Key {} has empty encrypted_key_material, generating and storing new key material", key_id); warn!(key_id, "Vault KMS key material missing; regenerating");
let key_material = generate_key_material(&key_data.algorithm)?; let key_material = generate_key_material(&key_data.algorithm)?;
key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?; key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
// Store the updated key data back to Vault // Store the updated key data back to Vault
@@ -147,7 +147,7 @@ impl VaultKmsClient {
let key_material = match self.decrypt_key_material(&key_data.encrypted_key_material).await { let key_material = match self.decrypt_key_material(&key_data.encrypted_key_material).await {
Ok(km) => km, Ok(km) => km,
Err(e) => { Err(e) => {
warn!("Failed to decrypt key material for key {}: {}, generating new key material", key_id, e); warn!(key_id, error = %e, "Vault KMS key material decrypt failed; regenerating");
let new_key_material = generate_key_material(&key_data.algorithm)?; let new_key_material = generate_key_material(&key_data.algorithm)?;
key_data.encrypted_key_material = self.encrypt_key_material(&new_key_material).await?; key_data.encrypted_key_material = self.encrypt_key_material(&new_key_material).await?;
// Store the updated key data back to Vault // Store the updated key data back to Vault
@@ -210,7 +210,7 @@ impl VaultKmsClient {
// If encrypted_key_material is empty, generate it (this handles the case where // If encrypted_key_material is empty, generate it (this handles the case where
// an old key was created without proper key material) // an old key was created without proper key material)
if existing_key_data.encrypted_key_material.is_empty() { if existing_key_data.encrypted_key_material.is_empty() {
warn!("Key {} has empty encrypted_key_material, generating new key material", key_id); warn!(key_id, "Vault KMS key metadata missing encrypted key material");
let key_material = generate_key_material(&existing_key_data.algorithm)?; let key_material = generate_key_material(&existing_key_data.algorithm)?;
existing_key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?; existing_key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
} }
@@ -316,7 +316,7 @@ impl KmsClient for VaultKmsClient {
let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone()); let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
info!("Generated data key for master key: {}", request.master_key_id); debug!(key_id = %request.master_key_id, "Vault KMS data key generated");
Ok(data_key) Ok(data_key)
} }
@@ -373,7 +373,7 @@ impl KmsClient for VaultKmsClient {
.decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce) .decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce)
.await?; .await?;
info!("Successfully decrypted data"); debug!("Vault KMS data decrypted");
Ok(plaintext) Ok(plaintext)
} }
@@ -418,7 +418,7 @@ impl KmsClient for VaultKmsClient {
created_by: None, created_by: None,
}; };
info!("Successfully created master key: {}", key_id); debug!(key_id, "Vault KMS master key created");
Ok(master_key) Ok(master_key)
} }
@@ -486,7 +486,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Active; key_data.status = KeyStatus::Active;
self.store_key_data(key_id, &key_data).await?; self.store_key_data(key_id, &key_data).await?;
info!("Enabled key: {}", key_id); debug!(key_id, "Vault KMS key enabled");
Ok(()) Ok(())
} }
@@ -497,7 +497,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Disabled; key_data.status = KeyStatus::Disabled;
self.store_key_data(key_id, &key_data).await?; self.store_key_data(key_id, &key_data).await?;
info!("Disabled key: {}", key_id); debug!(key_id, "Vault KMS key disabled");
Ok(()) Ok(())
} }
@@ -513,7 +513,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::PendingDeletion; key_data.status = KeyStatus::PendingDeletion;
self.store_key_data(key_id, &key_data).await?; self.store_key_data(key_id, &key_data).await?;
info!("Scheduled key deletion: {}", key_id); debug!(key_id, "Vault KMS key deletion scheduled");
Ok(()) Ok(())
} }
@@ -524,7 +524,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Active; key_data.status = KeyStatus::Active;
self.store_key_data(key_id, &key_data).await?; self.store_key_data(key_id, &key_data).await?;
info!("Canceled key deletion: {}", key_id); debug!(key_id, "Vault KMS key deletion canceled");
Ok(()) Ok(())
} }
@@ -553,7 +553,7 @@ impl KmsClient for VaultKmsClient {
created_by: None, created_by: None,
}; };
info!("Successfully rotated key: {}", key_id); debug!(key_id, "Vault KMS key rotated");
Ok(master_key) Ok(master_key)
} }
@@ -573,7 +573,7 @@ impl KmsClient for VaultKmsClient {
debug!("Vault health check passed - 404 error is expected when no keys exist yet"); debug!("Vault health check passed - 404 error is expected when no keys exist yet");
Ok(()) Ok(())
} else { } else {
warn!("Vault health check failed: {}", e); warn!(error = %e, "Vault KMS health check failed");
Err(e) Err(e)
} }
} }
crates/kms/src/service.rs
+19 -7
View File
@@ -24,7 +24,7 @@ use rand::random;
use std::collections::HashMap; use std::collections::HashMap;
use std::io::Cursor; use std::io::Cursor;
use tokio::io::{AsyncRead, AsyncReadExt}; use tokio::io::{AsyncRead, AsyncReadExt};
use tracing::{debug, info}; use tracing::debug;
use zeroize::Zeroize; use zeroize::Zeroize;
/// Data key for object encryption /// Data key for object encryption
@@ -209,7 +209,7 @@ impl ObjectEncryptionService {
// Generate a unique random nonce for this data key // Generate a unique random nonce for this data key
// This ensures each object/part gets a unique base nonce for streaming encryption // This ensures each object/part gets a unique base nonce for streaming encryption
let nonce: [u8; 12] = random(); let nonce: [u8; 12] = random();
tracing::info!("Generated random nonce for data key: {:02x?}", nonce); tracing::debug!("Generated random nonce for data key");
let data_key = DataKey { let data_key = DataKey {
plaintext_key: data_key_response plaintext_key: data_key_response
@@ -302,7 +302,7 @@ impl ObjectEncryptionService {
key_id: actual_key_id.to_string(), key_id: actual_key_id.to_string(),
}; };
if let Err(KmsError::KeyNotFound { .. }) = self.kms_manager.describe_key(describe_req).await { if let Err(KmsError::KeyNotFound { .. }) = self.kms_manager.describe_key(describe_req).await {
info!("Auto-creating SSE-S3 key: {}", actual_key_id); debug!(key_id = %actual_key_id, "Auto-creating SSE-S3 key");
let create_req = CreateKeyRequest { let create_req = CreateKeyRequest {
key_name: Some(actual_key_id.to_string()), key_name: Some(actual_key_id.to_string()),
key_usage: KeyUsage::EncryptDecrypt, key_usage: KeyUsage::EncryptDecrypt,
@@ -364,7 +364,13 @@ impl ObjectEncryptionService {
encrypted_data_key: data_key.ciphertext_blob, encrypted_data_key: data_key.ciphertext_blob,
}; };
info!("Successfully encrypted object {}/{} ({} bytes)", bucket, object_key, original_size); debug!(
bucket,
object = object_key,
original_size,
algorithm = %algorithm.as_str(),
"Object encrypted"
);
Ok(EncryptionResult { ciphertext, metadata }) Ok(EncryptionResult { ciphertext, metadata })
} }
@@ -429,7 +435,13 @@ impl ObjectEncryptionService {
// Decrypt the data // Decrypt the data
let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?; let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?;
info!("Successfully decrypted object {}/{} ({} bytes)", bucket, object_key, plaintext.len()); debug!(
bucket,
object = object_key,
plaintext_len = plaintext.len(),
algorithm = %metadata.algorithm,
"Object decrypted"
);
Ok(Box::new(Cursor::new(plaintext))) Ok(Box::new(Cursor::new(plaintext)))
} }
@@ -507,7 +519,7 @@ impl ObjectEncryptionService {
encrypted_data_key: Vec::new(), // Empty for SSE-C encrypted_data_key: Vec::new(), // Empty for SSE-C
}; };
info!( debug!(
"Successfully encrypted object {}/{} with SSE-C ({} bytes)", "Successfully encrypted object {}/{} with SSE-C ({} bytes)",
bucket, object_key, original_size bucket, object_key, original_size
); );
@@ -568,7 +580,7 @@ impl ObjectEncryptionService {
// Decrypt the data // Decrypt the data
let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?; let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?;
info!( debug!(
"Successfully decrypted SSE-C object {}/{} ({} bytes)", "Successfully decrypted SSE-C object {}/{} ({} bytes)",
bucket, bucket,
object_key, object_key,
crates/kms/src/service_manager.rs
+61 -11
View File
@@ -25,7 +25,11 @@ use std::sync::{
atomic::{AtomicU64, Ordering}, atomic::{AtomicU64, Ordering},
}; };
use tokio::sync::{Mutex, RwLock}; use tokio::sync::{Mutex, RwLock};
use tracing::{error, info, warn}; use tracing::{debug, error, info, warn};
const LOG_COMPONENT_KMS: &str = "kms";
const LOG_SUBSYSTEM_SERVICE: &str = "service";
const EVENT_KMS_SERVICE_STATE: &str = "kms_service_state";
/// KMS service status /// KMS service status
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
@@ -106,7 +110,13 @@ impl KmsServiceManager {
*status = KmsServiceStatus::Configured; *status = KmsServiceStatus::Configured;
} }
info!("KMS configuration updated successfully"); debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "configured",
"KMS service configured"
);
Ok(()) Ok(())
} }
@@ -132,7 +142,14 @@ impl KmsServiceManager {
} }
}; };
info!("Starting KMS service with backend: {:?}", config.backend); info!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
backend = ?config.backend,
state = "starting",
"KMS service starting"
);
match self.create_service_version(&config).await { match self.create_service_version(&config).await {
Ok(service_version) => { Ok(service_version) => {
@@ -146,7 +163,13 @@ impl KmsServiceManager {
*status = KmsServiceStatus::Running; *status = KmsServiceStatus::Running;
} }
info!("KMS service started successfully"); debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "running",
"KMS service running"
);
Ok(()) Ok(())
} }
Err(e) => { Err(e) => {
@@ -170,7 +193,13 @@ impl KmsServiceManager {
/// Internal stop implementation (called within lifecycle mutex) /// Internal stop implementation (called within lifecycle mutex)
async fn stop_internal(&self) -> Result<()> { async fn stop_internal(&self) -> Result<()> {
info!("Stopping KMS service"); debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "stopping",
"KMS service stopping"
);
// Atomically clear current service version (lock-free, instant) // Atomically clear current service version (lock-free, instant)
// Note: Existing Arc references will keep the service alive until operations complete // Note: Existing Arc references will keep the service alive until operations complete
@@ -184,7 +213,13 @@ impl KmsServiceManager {
} }
} }
info!("KMS service stopped successfully (existing operations may continue)"); debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "configured",
"KMS service stopped"
);
Ok(()) Ok(())
} }
@@ -201,7 +236,13 @@ impl KmsServiceManager {
pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> { pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> {
let _guard = self.lifecycle_mutex.lock().await; let _guard = self.lifecycle_mutex.lock().await;
info!("Reconfiguring KMS service (zero-downtime)"); debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "reconfiguring",
"KMS service reconfiguring"
);
new_config.validate()?; new_config.validate()?;
// Configure with new config // Configure with new config
@@ -230,13 +271,22 @@ impl KmsServiceManager {
if let Some(old_ver) = old_version { if let Some(old_ver) = old_version {
info!( info!(
"KMS service reconfigured successfully: version {} -> {} (old service will be cleaned up when operations complete)", event = EVENT_KMS_SERVICE_STATE,
old_ver, new_service_version.version component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
old_version = old_ver,
new_version = new_service_version.version,
state = "running",
"KMS service reconfigured"
); );
} else { } else {
info!( info!(
"KMS service reconfigured successfully: version {} (service started)", event = EVENT_KMS_SERVICE_STATE,
new_service_version.version component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
new_version = new_service_version.version,
state = "running",
"KMS service started from reconfigure"
); );
} }
Ok(()) Ok(())
crates/notify/src/bucket_config_manager.rs
+4 -4
View File
@@ -69,7 +69,7 @@ impl NotifyBucketConfigManager {
bucket = %bucket, bucket = %bucket,
region = %cfg.region, region = %cfg.region,
available_arn_count = arn_list.len(), available_arn_count = arn_list.len(),
"Loaded available notify target ARNs for bucket config validation" "notify bucket config validation"
); );
if let Err(e) = cfg.validate(&cfg.region, &arn_list) { if let Err(e) = cfg.validate(&cfg.region, &arn_list) {
@@ -81,7 +81,7 @@ impl NotifyBucketConfigManager {
region = %cfg.region, region = %cfg.region,
error = %e, error = %e,
result = "validation_failed", result = "validation_failed",
"Bucket notification config validation failed" "notify bucket config validation"
); );
if !matches!(e, ParseConfigError::ArnNotFound(_)) { if !matches!(e, ParseConfigError::ArnNotFound(_)) {
return Err(NotificationError::BucketNotification(e.to_string())); return Err(NotificationError::BucketNotification(e.to_string()));
@@ -94,7 +94,7 @@ impl NotifyBucketConfigManager {
region = %cfg.region, region = %cfg.region,
error = %e, error = %e,
result = "missing_target_arn", result = "missing_target_arn",
"Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules" "notify bucket config validation"
); );
} }
@@ -107,7 +107,7 @@ impl NotifyBucketConfigManager {
bucket = %bucket, bucket = %bucket,
region = %cfg.region, region = %cfg.region,
rule_count = cfg.get_rules_map().inner().len(), rule_count = cfg.get_rules_map().inner().len(),
"Loaded bucket notification config" "notify bucket config state"
); );
Ok(()) Ok(())
} }
crates/notify/src/config_manager.rs
+15 -15
View File
@@ -91,7 +91,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY, component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG, subsystem = LOG_SUBSYSTEM_CONFIG,
state = "initializing", state = "initializing",
"Initializing notification system" "notify runtime lifecycle"
); );
let config = { let config = {
@@ -111,7 +111,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG, subsystem = LOG_SUBSYSTEM_CONFIG,
state = "targets_created", state = "targets_created",
target_count = targets.len(), target_count = targets.len(),
"Created notification targets" "notify runtime lifecycle"
); );
if targets.is_empty() { if targets.is_empty() {
debug!( debug!(
@@ -121,7 +121,7 @@ impl NotifyConfigManager {
state = "idle", state = "idle",
reason = "no_targets_configured", reason = "no_targets_configured",
hint = %notify_configuration_hint(), hint = %notify_configuration_hint(),
"Notification runtime has no configured targets" "notify runtime lifecycle"
); );
} }
@@ -132,7 +132,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY, component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG, subsystem = LOG_SUBSYSTEM_CONFIG,
state = "initialized", state = "initialized",
"Initialized notification system" "notify runtime lifecycle"
); );
Ok(()) Ok(())
} }
@@ -156,13 +156,13 @@ impl NotifyConfigManager {
if let Some(targets_of_type) = config.0.get_mut(&ttype) { if let Some(targets_of_type) = config.0.get_mut(&ttype) {
if targets_of_type.remove(&tname).is_some() { if targets_of_type.remove(&tname).is_some() {
info!( info!(
event = EVENT_NOTIFY_CONFIG_UPDATE, event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY, component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG, subsystem = LOG_SUBSYSTEM_CONFIG,
action = "remove_target", action = "remove_target",
target_id = %target_id, target_id = %target_id,
result = "removed", result = "removed",
"Removed notification target from configuration" "notify config update"
); );
changed = true; changed = true;
} }
@@ -178,7 +178,7 @@ impl NotifyConfigManager {
action = "remove_target", action = "remove_target",
target_id = %target_id, target_id = %target_id,
result = "not_found", result = "not_found",
"Notification target not found in configuration" "notify config update"
); );
} }
changed changed
@@ -246,7 +246,7 @@ impl NotifyConfigManager {
target_type = %target_type, target_type = %target_type,
target_name = %target_name, target_name = %target_name,
result = "not_found", result = "not_found",
"Notification target configuration not found" "notify config update"
); );
} }
debug!( debug!(
@@ -264,7 +264,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY, component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG, subsystem = LOG_SUBSYSTEM_CONFIG,
state = "reloading", state = "reloading",
"Reloading notification configuration" "notify runtime lifecycle"
); );
self.update_config(new_config.clone()).await; self.update_config(new_config.clone()).await;
@@ -281,7 +281,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG, subsystem = LOG_SUBSYSTEM_CONFIG,
state = "targets_created", state = "targets_created",
target_count = targets.len(), target_count = targets.len(),
"Created notification targets from reloaded configuration" "notify runtime lifecycle"
); );
if targets.is_empty() { if targets.is_empty() {
debug!( debug!(
@@ -291,7 +291,7 @@ impl NotifyConfigManager {
state = "idle", state = "idle",
reason = "no_targets_configured", reason = "no_targets_configured",
hint = %notify_configuration_hint(), hint = %notify_configuration_hint(),
"Notification runtime has no configured targets after reload" "notify runtime lifecycle"
); );
} }
@@ -302,7 +302,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY, component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG, subsystem = LOG_SUBSYSTEM_CONFIG,
state = "reloaded", state = "reloaded",
"Reloaded notification configuration" "notify runtime lifecycle"
); );
Ok(()) Ok(())
} }
@@ -333,7 +333,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG, subsystem = LOG_SUBSYSTEM_CONFIG,
action = "reload_if_changed", action = "reload_if_changed",
result = "unchanged", result = "unchanged",
"Notification configuration unchanged; skipping reload" "notify config update"
); );
return Ok(()); return Ok(());
} }
@@ -348,7 +348,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG, subsystem = LOG_SUBSYSTEM_CONFIG,
action = "reload_if_changed", action = "reload_if_changed",
result = "updated", result = "updated",
"Notification configuration updated; reloading runtime" "notify config update"
); );
self.reload_config(new_config).await self.reload_config(new_config).await
} }
crates/notify/src/global.rs
+10 -1
View File
@@ -23,6 +23,9 @@ use std::sync::{Arc, OnceLock};
use tracing::error; use tracing::error;
static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new(); static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new();
const LOG_COMPONENT_NOTIFY: &str = "notify";
const LOG_SUBSYSTEM_GLOBAL: &str = "global";
const EVENT_NOTIFY_GLOBAL_STATE: &str = "notify_global_state";
/// Initialize the global notification system with the given configuration. /// Initialize the global notification system with the given configuration.
/// This function should only be called once throughout the application life cycle. /// This function should only be called once throughout the application life cycle.
@@ -102,7 +105,13 @@ pub mod notifier_global {
// If the notification system itself cannot be retrieved, it will be returned directly // If the notification system itself cannot be retrieved, it will be returned directly
Some(sys) => sys, Some(sys) => sys,
None => { None => {
error!("Notification system is not initialized."); error!(
event = EVENT_NOTIFY_GLOBAL_STATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_GLOBAL,
state = "uninitialized",
"notify global state"
);
return; return;
} }
}; };
crates/notify/src/integration.rs
+3 -3
View File
@@ -372,7 +372,7 @@ impl Drop for NotificationSystem {
component = LOG_COMPONENT_NOTIFY, component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_INTEGRATION, subsystem = LOG_SUBSYSTEM_INTEGRATION,
state = "dropping", state = "dropping",
"Notification system instance is being dropped" "notify system integration state"
); );
let snapshot = self.snapshot_metrics(); let snapshot = self.snapshot_metrics();
@@ -394,7 +394,7 @@ impl Drop for NotificationSystem {
metric_name = name, metric_name = name,
metric_value = value, metric_value = value,
metric_kind = if is_gauge { "gauge" } else { "counter" }, metric_kind = if is_gauge { "gauge" } else { "counter" },
"Notification shutdown metric snapshot" "notify system integration state"
); );
} }
@@ -406,7 +406,7 @@ impl Drop for NotificationSystem {
subsystem = LOG_SUBSYSTEM_INTEGRATION, subsystem = LOG_SUBSYSTEM_INTEGRATION,
status_key = %key, status_key = %key,
status_value = %value, status_value = %value,
"Notification system status snapshot" "notify system integration state"
); );
} }
} }
crates/notify/src/notifier.rs
+6 -6
View File
@@ -96,7 +96,7 @@ impl EventNotifier {
component = LOG_COMPONENT_NOTIFY, component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH, subsystem = LOG_SUBSYSTEM_DISPATCH,
state = "targets_cleared", state = "targets_cleared",
"Removed all notify targets" "notify runtime lifecycle"
); );
} }
@@ -136,7 +136,7 @@ impl EventNotifier {
bucket = %bucket_name, bucket = %bucket_name,
object = %object_key, object = %object_key,
target_count = target_ids_len, target_count = target_ids_len,
"Dispatching notify event" "notify dispatch"
); );
for target_id in target_ids { for target_id in target_ids {
// `get` now returns Option<Arc<dyn Target + Send + Sync>> // `get` now returns Option<Arc<dyn Target + Send + Sync>>
@@ -166,7 +166,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH, subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_for_task.id(), target_id = %target_for_task.id(),
deferred = is_deferred, deferred = is_deferred,
"Prepared notify target dispatch" "notify dispatch"
); );
// Use cloned data in closures to avoid borrowing conflicts // Use cloned data in closures to avoid borrowing conflicts
// Create an EntityTarget from the event // Create an EntityTarget from the event
@@ -217,7 +217,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH, subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_name_for_task, target_id = %target_name_for_task,
deferred = is_deferred, deferred = is_deferred,
"Completed notify target dispatch" "notify dispatch"
); );
} }
}); });
@@ -256,7 +256,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH, subsystem = LOG_SUBSYSTEM_DISPATCH,
bucket = %bucket_name, bucket = %bucket_name,
target_count = target_ids_len, target_count = target_ids_len,
"Finished notify dispatch fan-out" "notify dispatch"
); );
} }
@@ -284,7 +284,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH, subsystem = LOG_SUBSYSTEM_DISPATCH,
state = "targets_initialized", state = "targets_initialized",
target_count = target_list_guard.len(), target_count = target_list_guard.len(),
"Initialized notify runtime targets" "notify runtime lifecycle"
); );
Ok(()) Ok(())
} }
crates/notify/src/rule_engine.rs
+2 -2
View File
@@ -69,7 +69,7 @@ impl NotifyRuleEngine {
bucket = %bucket, bucket = %bucket,
state = "updated", state = "updated",
event_count, event_count,
"Updated bucket notification rules" "notify bucket rules state"
); );
} }
@@ -85,7 +85,7 @@ impl NotifyRuleEngine {
subsystem = LOG_SUBSYSTEM_RULE_ENGINE, subsystem = LOG_SUBSYSTEM_RULE_ENGINE,
bucket = %bucket, bucket = %bucket,
state = "removed", state = "removed",
"Removed bucket notification rules" "notify bucket rules state"
); );
} }
} }
crates/notify/src/runtime_facade.rs
+5 -5
View File
@@ -66,7 +66,7 @@ impl NotifyRuntimeFacade {
subsystem = LOG_SUBSYSTEM_RUNTIME, subsystem = LOG_SUBSYSTEM_RUNTIME,
target_id = %target_id, target_id = %target_id,
state = "replay_started", state = "replay_started",
"Started notify replay worker" "notify runtime lifecycle"
); );
} else { } else {
debug!( debug!(
@@ -76,7 +76,7 @@ impl NotifyRuntimeFacade {
target_id = %target_id, target_id = %target_id,
state = "replay_skipped", state = "replay_skipped",
reason = "no_store_configured", reason = "no_store_configured",
"Skipped notify replay worker startup" "notify runtime lifecycle"
); );
} }
}), }),
@@ -122,7 +122,7 @@ impl NotifyRuntimeFacade {
component = LOG_COMPONENT_NOTIFY, component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_RUNTIME, subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "stopping", state = "stopping",
"Stopping notification runtime" "notify runtime lifecycle"
); );
let active_targets = self.replay_workers.read().await.len(); let active_targets = self.replay_workers.read().await.len();
@@ -132,7 +132,7 @@ impl NotifyRuntimeFacade {
subsystem = LOG_SUBSYSTEM_RUNTIME, subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "replay_stopping", state = "replay_stopping",
active_targets, active_targets,
"Stopping notify replay workers" "notify runtime lifecycle"
); );
{ {
@@ -160,7 +160,7 @@ impl NotifyRuntimeFacade {
component = LOG_COMPONENT_NOTIFY, component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_RUNTIME, subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "stopped", state = "stopped",
"Stopped notification runtime" "notify runtime lifecycle"
); );
} }
} }
crates/obs/src/telemetry/dial9.rs
+2 -2
View File
@@ -203,7 +203,7 @@ impl Dial9SessionGuard {
component = LOG_COMPONENT_OBS, component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_DIAL9, subsystem = LOG_SUBSYSTEM_DIAL9,
state = "shutdown_requested", state = "shutdown_requested",
"dial9 state changed" "dial9 state"
); );
// TelemetryGuard handles flushing automatically when dropped // TelemetryGuard handles flushing automatically when dropped
} }
@@ -219,7 +219,7 @@ impl Drop for Dial9SessionGuard {
component = LOG_COMPONENT_OBS, component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_DIAL9, subsystem = LOG_SUBSYSTEM_DIAL9,
state = "flushed", state = "flushed",
"dial9 state changed" "dial9 state"
); );
} }
} }
crates/obs/src/telemetry/local.rs
+3 -3
View File
@@ -175,7 +175,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
output_format = "json", output_format = "json",
logger_level, logger_level,
is_production, is_production,
"local logging state changed" "local logging state"
); );
OtelGuard { OtelGuard {
@@ -293,7 +293,7 @@ fn init_file_logging_internal(
stdout_mirror_enabled = stdout_guard.is_some(), stdout_mirror_enabled = stdout_guard.is_some(),
is_production, is_production,
logger_level, logger_level,
"local logging state changed" "local logging state"
); );
Ok(OtelGuard { Ok(OtelGuard {
@@ -482,7 +482,7 @@ pub fn spawn_cleanup_task(
zstd_level, zstd_level,
zstd_fallback_to_gzip, zstd_fallback_to_gzip,
zstd_workers, zstd_workers,
"log cleaner state changed" "log cleaner state"
); );
tokio::spawn(async move { tokio::spawn(async move {
crates/protocols/src/ftps/driver.rs
+5 -5
View File
@@ -655,7 +655,7 @@ where
state = "requested", state = "requested",
username = %masked_username, username = %masked_username,
path = %path_str, path = %path_str,
"ftps object delete state changed" "FTPS delete requested"
); );
let (bucket, key) = self let (bucket, key) = self
@@ -749,7 +749,7 @@ where
username = %masked_username, username = %masked_username,
path = %path_str, path = %path_str,
bucket = %bucket, bucket = %bucket,
"ftps directory state changed" "FTPS directory created"
); );
Ok(()) Ok(())
} }
@@ -793,7 +793,7 @@ where
state = "removed", state = "removed",
path = %path_str, path = %path_str,
bucket = %bucket, bucket = %bucket,
"ftps directory state changed" "FTPS directory removed"
); );
Ok(()) Ok(())
} }
@@ -807,7 +807,7 @@ where
subsystem = LOG_SUBSYSTEM_FTPS_DRIVER, subsystem = LOG_SUBSYSTEM_FTPS_DRIVER,
state = "already_removed", state = "already_removed",
bucket = %bucket, bucket = %bucket,
"ftps directory state changed" "FTPS directory already removed"
); );
Ok(()) Ok(())
} else { } else {
@@ -877,7 +877,7 @@ where
username = %MaskedAccessKey(&user.username), username = %MaskedAccessKey(&user.username),
from = %from_str, from = %from_str,
to = %to_str, to = %to_str,
"ftps rename state changed" "FTPS rename unsupported"
); );
Err(Error::new( Err(Error::new(
crates/protocols/src/ftps/server.rs
+24 -24
View File
@@ -107,7 +107,7 @@ where
bind_addr = %self.config.bind_addr, bind_addr = %self.config.bind_addr,
tls_enabled = self.config.tls_enabled, tls_enabled = self.config.tls_enabled,
ftps_required = self.config.ftps_required, ftps_required = self.config.ftps_required,
"ftps server state changed" "FTPS server starting"
); );
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false); let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
@@ -121,14 +121,14 @@ where
// Configure passive ports for data connections // Configure passive ports for data connections
if let Some(passive_ports) = &self.config.passive_ports { if let Some(passive_ports) = &self.config.passive_ports {
let range = self.config.parse_passive_ports()?; let range = self.config.parse_passive_ports()?;
info!( debug!(
event = EVENT_FTPS_CONFIG_STATE, event = EVENT_FTPS_CONFIG_STATE,
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER, subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "passive_ports_configured", state = "passive_ports_configured",
passive_ports = %passive_ports, passive_ports = %passive_ports,
passive_port_range = ?range, passive_port_range = ?range,
"ftps config state changed" "FTPS passive ports configured"
); );
server_builder = server_builder.passive_ports(range); server_builder = server_builder.passive_ports(range);
} else { } else {
@@ -137,19 +137,19 @@ where
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER, subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
result = "system_assigned_passive_ports", result = "system_assigned_passive_ports",
"ftps config state changed" "FTPS passive ports defaulted"
); );
} }
// Configure external IP address for passive mode // Configure external IP address for passive mode
if let Some(ref external_ip) = self.config.external_ip { if let Some(ref external_ip) = self.config.external_ip {
info!( debug!(
event = EVENT_FTPS_CONFIG_STATE, event = EVENT_FTPS_CONFIG_STATE,
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER, subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "external_ip_configured", state = "external_ip_configured",
external_ip = %external_ip, external_ip = %external_ip,
"ftps config state changed" "FTPS external IP configured"
); );
server_builder = server_builder.passive_host(external_ip.as_str()); server_builder = server_builder.passive_host(external_ip.as_str());
} }
@@ -157,13 +157,13 @@ where
// Configure both active and passive mode support // Configure both active and passive mode support
use libunftp::options::ActivePassiveMode; use libunftp::options::ActivePassiveMode;
server_builder = server_builder.active_passive_mode(ActivePassiveMode::ActiveAndPassive); server_builder = server_builder.active_passive_mode(ActivePassiveMode::ActiveAndPassive);
info!( debug!(
event = EVENT_FTPS_CONFIG_STATE, event = EVENT_FTPS_CONFIG_STATE,
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER, subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "active_passive_mode_enabled", state = "active_passive_mode_enabled",
mode = "active_and_passive", mode = "active_and_passive",
"ftps config state changed" "FTPS active/passive mode configured"
); );
// Configure FTPS / TLS // Configure FTPS / TLS
@@ -175,7 +175,7 @@ where
subsystem = LOG_SUBSYSTEM_FTPS_SERVER, subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "enabled", state = "enabled",
cert_dir = %cert_dir, cert_dir = %cert_dir,
"ftps tls state changed" "FTPS TLS enabled"
); );
let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir) let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
@@ -197,12 +197,12 @@ where
server_builder = server_builder.ftps_manual::<std::path::PathBuf>(Arc::new(server_config)); server_builder = server_builder.ftps_manual::<std::path::PathBuf>(Arc::new(server_config));
if self.config.ftps_required { if self.config.ftps_required {
info!( debug!(
event = EVENT_FTPS_TLS_STATE, event = EVENT_FTPS_TLS_STATE,
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER, subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "required", state = "required",
"ftps tls state changed" "FTPS TLS required"
); );
server_builder = server_builder.ftps_required(FtpsRequired::All, FtpsRequired::All); server_builder = server_builder.ftps_required(FtpsRequired::All, FtpsRequired::All);
} }
@@ -218,7 +218,7 @@ where
subsystem = LOG_SUBSYSTEM_FTPS_SERVER, subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "disabled", state = "disabled",
mode = "plain_ftp", mode = "plain_ftp",
"ftps tls state changed" "FTPS TLS disabled"
); );
} }
@@ -248,13 +248,13 @@ where
let _ = reload_shutdown_tx.send(true); let _ = reload_shutdown_tx.send(true);
match result { match result {
Ok(Ok(())) => { Ok(Ok(())) => {
info!( debug!(
event = EVENT_FTPS_SERVER_STATE, event = EVENT_FTPS_SERVER_STATE,
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER, subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "stopped", state = "stopped",
result = "ok", result = "ok",
"ftps server state changed" "FTPS server stopped"
); );
Ok(()) Ok(())
} }
@@ -288,7 +288,7 @@ where
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER, subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "shutdown_requested", state = "shutdown_requested",
"ftps server state changed" "FTPS shutdown requested"
); );
let _ = reload_shutdown_tx.send(true); let _ = reload_shutdown_tx.send(true);
// libunftp listen() is not easily cancellable gracefully without dropping the future. // libunftp listen() is not easily cancellable gracefully without dropping the future.
@@ -330,7 +330,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
result = "iam_unavailable", result = "iam_unavailable",
phase = "user_detail", phase = "user_detail",
error = %e, error = %e,
"ftps auth state changed" "FTPS user-detail IAM unavailable"
); );
UserDetailError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e))) UserDetailError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e)))
})?; })?;
@@ -344,7 +344,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
phase = "user_detail", phase = "user_detail",
username = %masked_username, username = %masked_username,
error = %e, error = %e,
"ftps auth state changed" "FTPS user-detail key check failed"
); );
UserDetailError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e))) UserDetailError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e)))
})?; })?;
@@ -357,7 +357,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
result = "identity_missing", result = "identity_missing",
phase = "user_detail", phase = "user_detail",
username = %masked_username, username = %masked_username,
"ftps auth state changed" "FTPS user-detail identity missing"
); );
UserDetailError::UserNotFound { UserDetailError::UserNotFound {
username: principal.username.clone(), username: principal.username.clone(),
@@ -406,7 +406,7 @@ impl Authenticator for FtpsAuthenticator {
result = "iam_unavailable", result = "iam_unavailable",
phase = "authenticate", phase = "authenticate",
error = %e, error = %e,
"ftps auth state changed" "FTPS auth IAM unavailable"
); );
AuthenticationError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e))) AuthenticationError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e)))
})?; })?;
@@ -433,7 +433,7 @@ impl Authenticator for FtpsAuthenticator {
phase = "authenticate", phase = "authenticate",
username = %masked_username, username = %masked_username,
error = %e, error = %e,
"ftps auth state changed" "FTPS auth key check failed"
); );
AuthenticationError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e))) AuthenticationError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e)))
})?; })?;
@@ -446,7 +446,7 @@ impl Authenticator for FtpsAuthenticator {
result = "invalid_access_key", result = "invalid_access_key",
phase = "authenticate", phase = "authenticate",
username = %masked_username, username = %masked_username,
"ftps auth state changed" "FTPS auth rejected access key"
); );
return Err(AuthenticationError::BadUser); return Err(AuthenticationError::BadUser);
} }
@@ -459,7 +459,7 @@ impl Authenticator for FtpsAuthenticator {
result = "identity_missing", result = "identity_missing",
phase = "authenticate", phase = "authenticate",
username = %masked_username, username = %masked_username,
"ftps auth state changed" "FTPS auth identity missing"
); );
AuthenticationError::BadUser AuthenticationError::BadUser
})?; })?;
@@ -472,7 +472,7 @@ impl Authenticator for FtpsAuthenticator {
result = "invalid_secret_key", result = "invalid_secret_key",
phase = "authenticate", phase = "authenticate",
username = %masked_username, username = %masked_username,
"ftps auth state changed" "FTPS auth rejected secret key"
); );
return Err(AuthenticationError::BadPassword); return Err(AuthenticationError::BadPassword);
} }
@@ -484,7 +484,7 @@ impl Authenticator for FtpsAuthenticator {
result = "authenticated", result = "authenticated",
phase = "authenticate", phase = "authenticate",
username = %masked_username, username = %masked_username,
"ftps auth state changed" "FTPS auth accepted"
); );
Ok(Principal { Ok(Principal {
username: username.to_string(), username: username.to_string(),
crates/protocols/src/sftp/driver.rs
+2 -2
View File
@@ -188,7 +188,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
peer = %self.session_context.source_ip, peer = %self.session_context.source_ip,
user = %MaskedAccessKey(&self.session_context.principal.user_identity.credentials.access_key), user = %MaskedAccessKey(&self.session_context.principal.user_identity.credentials.access_key),
result = "read_only_rejected", result = "read_only_rejected",
"sftp driver state changed" "SFTP write rejected by read-only mode"
); );
return Err(SftpError::code(StatusCode::PermissionDenied)); return Err(SftpError::code(StatusCode::PermissionDenied));
} }
@@ -250,7 +250,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
op = op, op = op,
timeout_secs = self.backend_op_timeout_secs, timeout_secs = self.backend_op_timeout_secs,
result = "timeout", result = "timeout",
"sftp backend state changed" "SFTP backend operation timed out"
); );
Err(SftpError::code(StatusCode::Failure)) Err(SftpError::code(StatusCode::Failure))
} }
crates/protocols/src/sftp/server.rs
+8 -8
View File
@@ -255,7 +255,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
state = "enabled", state = "enabled",
host_key_dir = %config.host_key_dir.display(), host_key_dir = %config.host_key_dir.display(),
interval_secs, interval_secs,
"sftp host key reload state changed" "SFTP host-key reload enabled"
); );
tokio::spawn(async move { tokio::spawn(async move {
@@ -265,13 +265,13 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
loop { loop {
tokio::select! { tokio::select! {
_ = shutdown_token.cancelled() => { _ = shutdown_token.cancelled() => {
info!( debug!(
event = EVENT_SFTP_HOST_KEY_RELOAD_STATE, event = EVENT_SFTP_HOST_KEY_RELOAD_STATE,
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_SERVER, subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
state = "stopped", state = "stopped",
host_key_dir = %config.host_key_dir.display(), host_key_dir = %config.host_key_dir.display(),
"sftp host key reload state changed" "SFTP host-key reload stopped"
); );
break; break;
} }
@@ -280,14 +280,14 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
match holder.reload_from_config(&config).await { match holder.reload_from_config(&config).await {
Ok(Some(host_key_count)) => { Ok(Some(host_key_count)) => {
info!( debug!(
event = EVENT_SFTP_HOST_KEY_RELOAD_STATE, event = EVENT_SFTP_HOST_KEY_RELOAD_STATE,
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_SERVER, subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
result = "reloaded", result = "reloaded",
host_key_dir = %config.host_key_dir.display(), host_key_dir = %config.host_key_dir.display(),
host_key_count, host_key_count,
"sftp host key reload state changed" "SFTP host keys reloaded"
); );
} }
Ok(None) => { Ok(None) => {
@@ -297,7 +297,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
subsystem = LOG_SUBSYSTEM_SFTP_SERVER, subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
result = "unchanged", result = "unchanged",
host_key_dir = %config.host_key_dir.display(), host_key_dir = %config.host_key_dir.display(),
"sftp host key reload state changed" "SFTP host keys unchanged"
); );
} }
Err(err) => { Err(err) => {
@@ -308,7 +308,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
result = "reload_failed", result = "reload_failed",
host_key_dir = %config.host_key_dir.display(), host_key_dir = %config.host_key_dir.display(),
err = %err, err = %err,
"sftp host key reload state changed" "SFTP host-key reload failed"
); );
} }
} }
@@ -398,7 +398,7 @@ where
state = "listening", state = "listening",
bind_addr = %self.config.bind_addr, bind_addr = %self.config.bind_addr,
read_only = self.config.read_only, read_only = self.config.read_only,
"sftp server state changed" "SFTP server listening"
); );
let mut sessions: JoinSet<()> = JoinSet::new(); let mut sessions: JoinSet<()> = JoinSet::new();
crates/protocols/src/sftp/write.rs
+15 -15
View File
@@ -75,7 +75,7 @@ pub(super) fn write_dispatch_byte_count(phase: &WritePhase, part_size: u64, satu
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE, subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "byte_count_overflow", result = "byte_count_overflow",
"sftp write state changed" "SFTP write byte-count overflow"
); );
Err(SftpError::code(StatusCode::Failure)) Err(SftpError::code(StatusCode::Failure))
} }
@@ -90,7 +90,7 @@ pub(super) fn write_dispatch_byte_count(phase: &WritePhase, part_size: u64, satu
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE, subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "handle_poisoned", result = "handle_poisoned",
"sftp write state changed" "SFTP write handle poisoned"
); );
Err(SftpError::code(StatusCode::Failure)) Err(SftpError::code(StatusCode::Failure))
} }
@@ -115,7 +115,7 @@ pub(super) fn write_dispatch_append_bytes(phase: &mut WritePhase, data: &[u8]) -
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE, subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "append_on_failed_handle", result = "append_on_failed_handle",
"sftp write state changed" "SFTP write append rejected on failed handle"
); );
Err(SftpError::code(StatusCode::Failure)) Err(SftpError::code(StatusCode::Failure))
} }
@@ -368,7 +368,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
key = %sanitise_control_bytes(key), key = %sanitise_control_bytes(key),
attempt = attempt, attempt = attempt,
state = "retrying_put_object", state = "retrying_put_object",
"sftp write state changed", "SFTP put_object retry scheduled",
); );
} }
@@ -414,7 +414,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
bucket = %sanitise_control_bytes(bucket), bucket = %sanitise_control_bytes(bucket),
key = %sanitise_control_bytes(key), key = %sanitise_control_bytes(key),
result = "retry_loop_fell_through", result = "retry_loop_fell_through",
"sftp write state changed", "SFTP put_object retry loop fell through",
); );
Err(SftpError::code(StatusCode::Failure)) Err(SftpError::code(StatusCode::Failure))
} }
@@ -459,7 +459,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
upload_id = %upload_id, upload_id = %upload_id,
part_number = part_number, part_number = part_number,
result = "etag_missing", result = "etag_missing",
"sftp multipart state changed" "SFTP multipart part missing etag"
); );
SftpError::code(StatusCode::Failure) SftpError::code(StatusCode::Failure)
})?; })?;
@@ -543,7 +543,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
bucket = %sanitise_control_bytes(bucket), bucket = %sanitise_control_bytes(bucket),
key = %sanitise_control_bytes(key), key = %sanitise_control_bytes(key),
result = "upload_id_missing", result = "upload_id_missing",
"sftp multipart state changed" "SFTP multipart upload missing upload_id"
); );
SftpError::code(StatusCode::Failure) SftpError::code(StatusCode::Failure)
})?; })?;
@@ -708,7 +708,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE, subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "buffering_phase_lost", result = "buffering_phase_lost",
"sftp multipart state changed" "SFTP multipart buffering phase lost"
); );
return Err(SftpError::code(StatusCode::Failure)); return Err(SftpError::code(StatusCode::Failure));
} }
@@ -772,7 +772,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
key = %key, key = %key,
limit = S3_MAX_MULTIPART_PARTS, limit = S3_MAX_MULTIPART_PARTS,
result = "parts_limit_exceeded", result = "parts_limit_exceeded",
"sftp multipart state changed", "SFTP multipart parts limit exceeded",
); );
let upload_id_for_fail = upload_id.clone(); let upload_id_for_fail = upload_id.clone();
let abort_authorized_for_fail = *abort_authorized; let abort_authorized_for_fail = *abort_authorized;
@@ -791,7 +791,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE, subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "flush_without_streaming", result = "flush_without_streaming",
"sftp multipart state changed" "SFTP multipart flush requested without streaming state"
); );
return Err(SftpError::code(StatusCode::Failure)); return Err(SftpError::code(StatusCode::Failure));
} }
@@ -817,7 +817,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE, subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "post_upload_phase_missing", result = "post_upload_phase_missing",
"sftp multipart state changed" "SFTP multipart post-upload phase missing"
); );
Err(SftpError::code(StatusCode::Failure)) Err(SftpError::code(StatusCode::Failure))
} }
@@ -887,7 +887,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
err = ?abort_err, err = ?abort_err,
context = context, context = context,
result = "abort_failed", result = "abort_failed",
"sftp abort state changed", "SFTP multipart abort failed",
); );
} }
} else { } else {
@@ -901,7 +901,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
access_key = %MaskedAccessKey(self.access_key()), access_key = %MaskedAccessKey(self.access_key()),
context = context, context = context,
result = "abort_skipped_unauthorized", result = "abort_skipped_unauthorized",
"sftp abort state changed", "SFTP multipart abort skipped by policy",
); );
} }
} }
@@ -959,7 +959,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
upload_id = %upload_id, upload_id = %upload_id,
limit = S3_MAX_MULTIPART_PARTS, limit = S3_MAX_MULTIPART_PARTS,
result = "final_part_limit_exceeded", result = "final_part_limit_exceeded",
"sftp multipart state changed", "SFTP multipart final-part limit exceeded",
); );
self.close_abort_or_skip(bucket, key, &upload_id, abort_authorized, "parts-limit breach") self.close_abort_or_skip(bucket, key, &upload_id, abort_authorized, "parts-limit breach")
.await; .await;
@@ -1080,7 +1080,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
upload_id = %mp.upload_id, upload_id = %mp.upload_id,
part_number = part_number, part_number = part_number,
result = "etag_missing", result = "etag_missing",
"sftp multipart copy state changed" "SFTP multipart copy part missing etag"
); );
SftpError::code(StatusCode::Failure) SftpError::code(StatusCode::Failure)
})?; })?;
crates/protocols/src/webdav/driver.rs
+14 -14
View File
@@ -423,7 +423,7 @@ where
bucket = %bucket, bucket = %bucket,
object = %key, object = %key,
file_size, file_size,
"webdav object write state changed" "WebDAV object flush completed"
); );
// Buffer already cleared by std::mem::take above // Buffer already cleared by std::mem::take above
Ok(()) Ok(())
@@ -438,7 +438,7 @@ where
object = %key, object = %key,
file_size, file_size,
error = %e, error = %e,
"webdav object write state changed" "WebDAV object flush failed"
); );
Err(FsError::GeneralFailure) Err(FsError::GeneralFailure)
} }
@@ -1219,7 +1219,7 @@ where
result = "not_found", result = "not_found",
bucket = %bucket, bucket = %bucket,
error = %e, error = %e,
"webdav bucket metadata state changed" "WebDAV bucket metadata listed"
); );
Err(FsError::NotFound) Err(FsError::NotFound)
} }
@@ -1279,7 +1279,7 @@ where
state = "created", state = "created",
bucket = %bucket, bucket = %bucket,
object = %dir_key, object = %dir_key,
"webdav directory state changed" "WebDAV directory marker created"
); );
return Ok(()); return Ok(());
} }
@@ -1292,7 +1292,7 @@ where
bucket = %bucket, bucket = %bucket,
object = %dir_key, object = %dir_key,
error = %e, error = %e,
"webdav directory state changed" "WebDAV directory marker create failed"
); );
return Err(FsError::GeneralFailure); return Err(FsError::GeneralFailure);
} }
@@ -1320,7 +1320,7 @@ where
subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER, subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
state = "bucket_created", state = "bucket_created",
bucket = %bucket, bucket = %bucket,
"webdav directory state changed" "WebDAV bucket created"
); );
Ok(()) Ok(())
} }
@@ -1332,7 +1332,7 @@ where
state = "bucket_create_failed", state = "bucket_create_failed",
bucket = %bucket, bucket = %bucket,
error = %e, error = %e,
"webdav directory state changed" "WebDAV bucket create failed"
); );
Err(FsError::GeneralFailure) Err(FsError::GeneralFailure)
} }
@@ -1465,7 +1465,7 @@ where
state = "deleted", state = "deleted",
bucket = %bucket, bucket = %bucket,
object = %key, object = %key,
"webdav object delete state changed" "WebDAV object deleted"
); );
Ok(()) Ok(())
} }
@@ -1478,7 +1478,7 @@ where
bucket = %bucket, bucket = %bucket,
object = %key, object = %key,
error = %e, error = %e,
"webdav object delete state changed" "WebDAV object delete failed"
); );
Err(FsError::GeneralFailure) Err(FsError::GeneralFailure)
} }
@@ -1529,7 +1529,7 @@ where
dst_bucket = %dst_bucket, dst_bucket = %dst_bucket,
dst_object = %dst_key, dst_object = %dst_key,
error = %e, error = %e,
"webdav rename state changed" "WebDAV rename source delete failed"
); );
FsError::GeneralFailure FsError::GeneralFailure
})?; })?;
@@ -1543,7 +1543,7 @@ where
src_object = %src_key, src_object = %src_key,
dst_bucket = %dst_bucket, dst_bucket = %dst_bucket,
dst_object = %dst_key, dst_object = %dst_key,
"webdav rename state changed" "WebDAV file renamed"
); );
return Ok(()); return Ok(());
} }
@@ -1603,7 +1603,7 @@ where
dst_bucket = %dst_bucket, dst_bucket = %dst_bucket,
dst_prefix = %dst_prefix, dst_prefix = %dst_prefix,
error = %e, error = %e,
"webdav rename state changed" "WebDAV rename directory listing failed"
); );
FsError::GeneralFailure FsError::GeneralFailure
})?; })?;
@@ -1652,7 +1652,7 @@ where
src_object = %src_key, src_object = %src_key,
dst_bucket = %dst_bucket, dst_bucket = %dst_bucket,
dst_object = %dst_key, dst_object = %dst_key,
"webdav rename state changed" "WebDAV rename source not found"
); );
return Err(FsError::NotFound); return Err(FsError::NotFound);
} }
@@ -1666,7 +1666,7 @@ where
src_object = %src_key, src_object = %src_key,
dst_bucket = %dst_bucket, dst_bucket = %dst_bucket,
dst_object = %dst_key, dst_object = %dst_key,
"webdav rename state changed" "WebDAV directory renamed"
); );
Ok(()) Ok(())
} }
crates/protocols/src/webdav/server.rs
+11 -11
View File
@@ -86,7 +86,7 @@ where
bind_addr = %self.config.bind_addr, bind_addr = %self.config.bind_addr,
tls_enabled = self.config.tls_enabled, tls_enabled = self.config.tls_enabled,
max_body_size = self.config.max_body_size, max_body_size = self.config.max_body_size,
"webdav server state changed" "WebDAV server starting"
); );
let listener = TcpListener::bind(self.config.bind_addr).await?; let listener = TcpListener::bind(self.config.bind_addr).await?;
@@ -96,7 +96,7 @@ where
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER, subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "listening", state = "listening",
bind_addr = %self.config.bind_addr, bind_addr = %self.config.bind_addr,
"webdav server state changed" "WebDAV server listening"
); );
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false); let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
@@ -109,7 +109,7 @@ where
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER, subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "enabled", state = "enabled",
cert_dir = %cert_dir, cert_dir = %cert_dir,
"webdav tls state changed" "WebDAV TLS enabled"
); );
let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir) let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
@@ -211,7 +211,7 @@ where
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER, subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "shutdown_requested", state = "shutdown_requested",
"webdav server state changed" "WebDAV shutdown requested"
); );
let _ = reload_shutdown_tx.send(true); let _ = reload_shutdown_tx.send(true);
break; break;
@@ -225,7 +225,7 @@ where
component = LOG_COMPONENT_PROTOCOLS, component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER, subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "stopped", state = "stopped",
"webdav server state changed" "WebDAV server stopped"
); );
Ok(()) Ok(())
} }
@@ -376,7 +376,7 @@ where
result = "iam_unavailable", result = "iam_unavailable",
source_ip = %source_ip, source_ip = %source_ip,
error = %e, error = %e,
"webdav auth state changed" "WebDAV auth IAM unavailable"
); );
WebDavInitError::Server("Internal authentication service unavailable".to_string()) WebDavInitError::Server("Internal authentication service unavailable".to_string())
})?; })?;
@@ -403,7 +403,7 @@ where
source_ip = %source_ip, source_ip = %source_ip,
access_key = %masked_access_key, access_key = %masked_access_key,
error = %e, error = %e,
"webdav auth state changed" "WebDAV auth key check failed"
); );
WebDavInitError::Server("Authentication verification failed".to_string()) WebDavInitError::Server("Authentication verification failed".to_string())
})?; })?;
@@ -416,7 +416,7 @@ where
result = "invalid_access_key", result = "invalid_access_key",
source_ip = %source_ip, source_ip = %source_ip,
access_key = %masked_access_key, access_key = %masked_access_key,
"webdav auth state changed" "WebDAV auth rejected access key"
); );
return Err(WebDavInitError::Server("Invalid credentials".to_string())); return Err(WebDavInitError::Server("Invalid credentials".to_string()));
} }
@@ -429,7 +429,7 @@ where
result = "identity_missing", result = "identity_missing",
source_ip = %source_ip, source_ip = %source_ip,
access_key = %masked_access_key, access_key = %masked_access_key,
"webdav auth state changed" "WebDAV auth identity missing"
); );
WebDavInitError::Server("User not found".to_string()) WebDavInitError::Server("User not found".to_string())
})?; })?;
@@ -442,7 +442,7 @@ where
result = "invalid_secret_key", result = "invalid_secret_key",
source_ip = %source_ip, source_ip = %source_ip,
access_key = %masked_access_key, access_key = %masked_access_key,
"webdav auth state changed" "WebDAV auth rejected secret key"
); );
return Err(WebDavInitError::Server("Invalid credentials".to_string())); return Err(WebDavInitError::Server("Invalid credentials".to_string()));
} }
@@ -454,7 +454,7 @@ where
result = "authenticated", result = "authenticated",
source_ip = %source_ip, source_ip = %source_ip,
access_key = %masked_access_key, access_key = %masked_access_key,
"webdav auth state changed" "WebDAV auth accepted"
); );
Ok(SessionContext::new( Ok(SessionContext::new(
crates/scanner/src/scanner.rs
+24 -24
View File
@@ -97,7 +97,7 @@ fn resolve_scanner_runtime_config() -> crate::runtime_config::ScannerRuntimeConf
subsystem = LOG_SUBSYSTEM_RUNTIME, subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "resolve_failed", state = "resolve_failed",
error = %err, error = %err,
"Scanner runtime config resolution failed; using last applied config" "Scanner runtime config fallback applied"
); );
current_scanner_runtime_config() current_scanner_runtime_config()
} }
@@ -179,7 +179,7 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "startup_inspect_failed", state = "startup_inspect_failed",
error = %err, error = %err,
"Scanner startup cache inspection failed; keeping configured startup delay" "Scanner startup cache inspection failed"
); );
return false; return false;
} }
@@ -198,7 +198,7 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "startup_decode_failed", state = "startup_decode_failed",
error = %err, error = %err,
"Scanner startup cache decode failed; skipping startup delay" "Scanner startup cache decode failed"
); );
true true
} }
@@ -222,7 +222,7 @@ async fn initial_scanner_startup_usage_state(storeapi: &Arc<ECStore>) -> (bool,
subsystem = LOG_SUBSYSTEM_RUNTIME, subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "startup_bucket_inspect_failed", state = "startup_bucket_inspect_failed",
error = %err, error = %err,
"Scanner startup bucket inspection failed; keeping configured startup delay" "Scanner startup bucket inspection failed"
); );
false false
} }
@@ -418,7 +418,7 @@ async fn configure_scanner_defaults(storeapi: &Arc<ECStore>) {
replication_active = features.replication, replication_active = features.replication,
feature_inspection_failed = features.inspection_failed, feature_inspection_failed = features.inspection_failed,
state = "single_disk_defaults_applied", state = "single_disk_defaults_applied",
"Scanner defaults updated for single-disk deployment" "Scanner defaults applied"
); );
} else { } else {
set_scanner_default_speed(ScannerSpeed::Default); set_scanner_default_speed(ScannerSpeed::Default);
@@ -562,7 +562,7 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
path = %&*BACKGROUND_HEAL_INFO_PATH, path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "decode_failed", state = "decode_failed",
error = %e, error = %e,
"Scanner background heal state decode failed" "Scanner background heal decode failed"
); );
BackgroundHealInfo::default() BackgroundHealInfo::default()
}), }),
@@ -577,7 +577,7 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
path = %&*BACKGROUND_HEAL_INFO_PATH, path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "read_failed", state = "read_failed",
error = %e, error = %e,
"Scanner background heal state read failed" "Scanner background heal read failed"
); );
} }
BackgroundHealInfo::default() BackgroundHealInfo::default()
@@ -605,7 +605,7 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
path = %&*BACKGROUND_HEAL_INFO_PATH, path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "encode_failed", state = "encode_failed",
error = %e, error = %e,
"Scanner background heal state encode failed" "Scanner background heal encode failed"
); );
return; return;
} }
@@ -621,7 +621,7 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
path = %&*BACKGROUND_HEAL_INFO_PATH, path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "save_failed", state = "save_failed",
error = %e, error = %e,
"Scanner background heal state save failed" "Scanner background heal save failed"
); );
} }
} }
@@ -650,7 +650,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
subsystem = LOG_SUBSYSTEM_RUNTIME, subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "refresh_failed", state = "refresh_failed",
error = %err, error = %err,
"Scanner runtime config refresh failed; using last applied config" "Scanner runtime config refresh failed"
); );
} }
let configured_cycle_interval = scanner_cycle_interval(); let configured_cycle_interval = scanner_cycle_interval();
@@ -685,7 +685,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
cycle = cycle_info.current, cycle = cycle_info.current,
scan_mode = ?scan_mode, scan_mode = ?scan_mode,
state = "started", state = "started",
"Scanner cycle state updated" "Scanner cycle started"
); );
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode); let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
if let Some(new_heal_info) = background_heal_info_for_scan_start( if let Some(new_heal_info) = background_heal_info_for_scan_start(
@@ -730,7 +730,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
max_objects = ?cycle_budget.max_objects(), max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(), max_directories = ?cycle_budget.max_directories(),
state = "budget_reached", state = "budget_reached",
"Scanner cycle stopped after reaching budget" "Scanner cycle budget reached"
); );
let budget_reason = cycle_budget.reason(); let budget_reason = cycle_budget.reason();
emit_scan_cycle_partial_with_source( emit_scan_cycle_partial_with_source(
@@ -751,7 +751,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
state = "failed", state = "failed",
duration = ?now.elapsed(), duration = ?now.elapsed(),
error = %e, error = %e,
"Scanner cycle state updated" "Scanner cycle failed"
); );
emit_scan_cycle_complete(false, cycle_start.elapsed()); emit_scan_cycle_complete(false, cycle_start.elapsed());
if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) { if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) {
@@ -772,7 +772,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
max_objects = ?cycle_budget.max_objects(), max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(), max_directories = ?cycle_budget.max_directories(),
state = "budget_reached", state = "budget_reached",
"Scanner cycle stopped after reaching budget" "Scanner cycle budget reached"
); );
global_metrics().finish_scan_cycle_work(cycle_work_start); global_metrics().finish_scan_cycle_work(cycle_work_start);
let budget_reason = cycle_budget.reason(); let budget_reason = cycle_budget.reason();
@@ -806,7 +806,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
state = "completed", state = "completed",
duration = ?now.elapsed(), duration = ?now.elapsed(),
cycles_total = cycle_info.cycle_completed.len(), cycles_total = cycle_info.cycle_completed.len(),
"Scanner cycle state updated" "Scanner cycle completed"
); );
retain_recent_cycle_completions(&mut cycle_info.cycle_completed); retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
@@ -830,14 +830,14 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
"Scanner state persistence failed" "Scanner state persistence failed"
); );
} else { } else {
info!( debug!(
target: "rustfs::scanner", target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE, event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER, component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME, subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH, path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "saved", state = "saved",
"Scanner state persisted" "Scanner state saved"
); );
} }
} }
@@ -854,7 +854,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
subsystem = LOG_SUBSYSTEM_RUNTIME, subsystem = LOG_SUBSYSTEM_RUNTIME,
lock_name = "leader.lock", lock_name = "leader.lock",
state = "acquired", state = "acquired",
"Scanner leader lock state updated" "Scanner leader lock acquired"
); );
guard guard
} }
@@ -867,7 +867,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
lock_name = "leader.lock", lock_name = "leader.lock",
state = "contended", state = "contended",
error = ?e, error = ?e,
"Scanner leader lock state updated" "Scanner leader lock contended"
); );
return Ok(()); return Ok(());
} }
@@ -881,7 +881,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
lock_name = "leader.lock", lock_name = "leader.lock",
state = "create_failed", state = "create_failed",
error = %e, error = %e,
"Scanner leader lock state updated" "Scanner leader lock creation failed"
); );
return Ok(()); return Ok(());
} }
@@ -977,7 +977,7 @@ pub async fn store_data_usage_in_backend(
&& let (Some(new_ts), Some(existing_ts)) = (data_usage_info.last_update, existing.last_update) && let (Some(new_ts), Some(existing_ts)) = (data_usage_info.last_update, existing.last_update)
&& new_ts <= existing_ts && new_ts <= existing_ts
{ {
info!( debug!(
target: "rustfs::scanner", target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE, event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER, component = LOG_COMPONENT_SCANNER,
@@ -986,7 +986,7 @@ pub async fn store_data_usage_in_backend(
incoming_last_update = ?new_ts, incoming_last_update = ?new_ts,
existing_last_update = ?existing_ts, existing_last_update = ?existing_ts,
state = "skip_stale_update", state = "skip_stale_update",
"Scanner data usage persistence skipped stale update" "Scanner stale data usage update skipped"
); );
continue; continue;
} }
@@ -1003,7 +1003,7 @@ pub async fn store_data_usage_in_backend(
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "encode_failed", state = "encode_failed",
error = %e, error = %e,
"Scanner data usage persistence encode failed" "Scanner data usage encode failed"
); );
continue; continue;
} }
@@ -1040,7 +1040,7 @@ pub async fn store_data_usage_in_backend(
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "save_failed", state = "save_failed",
error = %e, error = %e,
"Scanner data usage persistence failed" "Scanner data usage save failed"
); );
} else { } else {
rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await; rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
crates/scanner/src/scanner_folder.rs
+6 -6
View File
@@ -66,7 +66,7 @@ use time::OffsetDateTime;
use tokio::select; use tokio::select;
use tokio::sync::mpsc; use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn}; use tracing::{debug, error, warn};
const LOG_COMPONENT_SCANNER: &str = "scanner"; const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_FOLDER: &str = "folder"; const LOG_SUBSYSTEM_FOLDER: &str = "folder";
@@ -639,7 +639,7 @@ impl ScannerItem {
subsystem = LOG_SUBSYSTEM_LIFECYCLE, subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(), object_path = %self.object_path(),
state = "started", state = "started",
"Scanner lifecycle action evaluation started" "Scanner lifecycle evaluation started"
); );
let versioning_config = match BucketVersioningSys::get(&self.bucket).await { let versioning_config = match BucketVersioningSys::get(&self.bucket).await {
@@ -990,7 +990,7 @@ impl ScannerItem {
let age = now - mod_time; let age = now - mod_time;
age.whole_seconds().max(0) age.whole_seconds().max(0)
}); });
info!( debug!(
target: "rustfs::scanner::folder", target: "rustfs::scanner::folder",
event = EVENT_SCANNER_HEAL_ADMISSION, event = EVENT_SCANNER_HEAL_ADMISSION,
component = LOG_COMPONENT_SCANNER, component = LOG_COMPONENT_SCANNER,
@@ -1003,7 +1003,7 @@ impl ScannerItem {
original_scan_mode = %HealScanMode::Deep.as_str(), original_scan_mode = %HealScanMode::Deep.as_str(),
effective_scan_mode = %scan_mode.as_str(), effective_scan_mode = %scan_mode.as_str(),
state = "downgraded_to_normal", state = "downgraded_to_normal",
"Scanner heal admission downgraded deep scan during cooldown" "Scanner heal deep scan downgraded"
); );
} }
@@ -1688,14 +1688,14 @@ impl FolderScanner {
if found_objects && is_erasure().await { if found_objects && is_erasure().await {
// If we found an object in erasure mode, we skip subdirs (only datadirs)... // If we found an object in erasure mode, we skip subdirs (only datadirs)...
info!( debug!(
target: "rustfs::scanner::folder", target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE, event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER, component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER, subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name, folder = %folder.name,
state = "erasure_object_found", state = "erasure_object_found",
"Scanner folder stopped descending after finding erasure object" "Scanner folder descent stopped after erasure object"
); );
break; break;
} }
crates/scanner/src/scanner_io.rs
+14 -14
View File
@@ -900,7 +900,7 @@ impl ScannerIOCache for SetDisks {
subsystem = LOG_SUBSYSTEM_IO, subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name, bucket = %bucket.name,
state = "scan_started", state = "scan_started",
"Scanner disk bucket state updated" "Scanner disk bucket scan started"
); );
let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]); let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]);
@@ -910,13 +910,13 @@ impl ScannerIOCache for SetDisks {
error!( error!(
target: "rustfs::scanner::io", target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE, event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER, component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO, subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name, bucket = %bucket.name,
cache_name = %cache_name, cache_name = %cache_name,
state = "cache_load_failed", state = "cache_load_failed",
error = %e, error = %e,
"Scanner disk bucket state updated" "Scanner disk bucket cache load failed"
); );
} }
@@ -942,7 +942,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name, bucket = %bucket.name,
cache_name = ?cache.info.name, cache_name = ?cache.info.name,
state = "cache_ready", state = "cache_ready",
"Scanner disk bucket state updated" "Scanner disk bucket cache ready"
); );
let (updates_tx, mut updates_rx) = mpsc::channel::<DataUsageEntry>(1); let (updates_tx, mut updates_rx) = mpsc::channel::<DataUsageEntry>(1);
@@ -997,7 +997,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name, bucket = %bucket.name,
state = "cancelled", state = "cancelled",
error = %e, error = %e,
"Scanner disk bucket state updated" "Scanner disk bucket scan cancelled"
); );
} else { } else {
error!( error!(
@@ -1008,7 +1008,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name, bucket = %bucket.name,
state = "scan_failed", state = "scan_failed",
error = %e, error = %e,
"Scanner disk bucket state updated" "Scanner disk bucket scan failed"
); );
} }
@@ -1104,7 +1104,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name, bucket = %bucket.name,
cache_name = %cache.info.name, cache_name = %cache.info.name,
state = "scan_completed", state = "scan_completed",
"Scanner disk bucket state updated" "Scanner disk bucket scan completed"
); );
if let Err(e) = update_fut.await { if let Err(e) = update_fut.await {
@@ -1132,7 +1132,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name, bucket = %bucket.name,
cache_name = %cache.info.name, cache_name = %cache.info.name,
state = "send_root_entry", state = "send_root_entry",
"Scanner data usage stream progress updated" "Scanner root entry publish started"
); );
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await { if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
@@ -1182,7 +1182,7 @@ impl ScannerIOCache for SetDisks {
component = LOG_COMPONENT_SCANNER, component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO, subsystem = LOG_SUBSYSTEM_IO,
state = "set_scan_completed", state = "set_scan_completed",
"Scanner set-level disk scan completed" "Scanner set scan completed"
); );
Ok(()) Ok(())
crates/targets/src/store.rs
+108 -16
View File
@@ -30,6 +30,10 @@ use std::{
use tracing::{debug, warn}; use tracing::{debug, warn};
use uuid::Uuid; use uuid::Uuid;
const LOG_COMPONENT_TARGETS: &str = "targets";
const LOG_SUBSYSTEM_STORE: &str = "store";
const EVENT_TARGET_STORE_STATE: &str = "target_store_state";
fn resolve_queue_store_compression_from_env_value(value: Option<&str>) -> bool { fn resolve_queue_store_compression_from_env_value(value: Option<&str>) -> bool {
value value
.and_then(|value| value.parse::<EnableState>().ok().map(|state| state.is_enabled())) .and_then(|value| value.parse::<EnableState>().ok().map(|state| state.is_enabled()))
@@ -83,7 +87,14 @@ impl std::fmt::Display for Key {
/// Parses a string into a Key /// Parses a string into a Key
pub fn parse_key(s: &str) -> Key { pub fn parse_key(s: &str) -> Key {
debug!("Parsing key: {}", s); debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "parse_key",
key = %s,
"target store state"
);
let mut name = s.to_string(); let mut name = s.to_string();
let mut extension = String::new(); let mut extension = String::new();
@@ -111,8 +122,16 @@ pub fn parse_key(s: &str) -> Key {
} }
debug!( debug!(
"Parsed key - name: {}, extension: {}, item_count: {}, compress: {}", event = EVENT_TARGET_STORE_STATE,
name, extension, item_count, compress component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "parse_key",
result = "parsed",
key_name = %name,
extension = %extension,
item_count,
compressed = compress,
"target store state"
); );
Key { Key {
@@ -276,7 +295,15 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
.read() .read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?; .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let path = self.file_path(key); let path = self.file_path(key);
debug!("Reading file for key: {},path: {}", key, path.display()); debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "read_file",
key = %key,
path = %path.display(),
"target store state"
);
let data = std::fs::read(&path).map_err(|e| { let data = std::fs::read(&path).map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound { if e.kind() == std::io::ErrorKind::NotFound {
StoreError::NotFound StoreError::NotFound
@@ -342,7 +369,14 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
std::fs::write(&path, data).map_err(StoreError::Io)?; std::fs::write(&path, data).map_err(StoreError::Io)?;
} }
let modified = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64; let modified = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64;
debug!("Wrote event to store: {}", key); debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "write_file",
key = %key,
"target store state"
);
Ok(modified) Ok(modified)
} }
@@ -409,7 +443,15 @@ where
} }
} }
debug!("Opened store at: {:?}", self.directory); debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
state = "opened",
store_dir = ?self.directory,
entry_count = entries_map.len(),
"target store state"
);
Ok(()) Ok(())
} }
@@ -471,7 +513,14 @@ where
} }
fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error> { fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error> {
debug!("Reading items from store for key: {}", key); debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "read_batch",
key = %key,
"target store state"
);
let data = self.get_raw(key)?; let data = self.get_raw(key)?;
if data.is_empty() { if data.is_empty() {
return Err(StoreError::Deserialization("Cannot deserialize empty data".to_string())); return Err(StoreError::Deserialization("Cannot deserialize empty data".to_string()));
@@ -500,10 +549,15 @@ where
if items.len() < key.item_count && !items.is_empty() { if items.len() < key.item_count && !items.is_empty() {
// Partial read // Partial read
warn!( warn!(
"Expected {} items for key {}, but only found {}. Possible data corruption or incorrect item_count.", event = EVENT_TARGET_STORE_STATE,
key.item_count, component = LOG_COMPONENT_TARGETS,
key, subsystem = LOG_SUBSYSTEM_STORE,
items.len() action = "read_batch",
key = %key,
expected_items = key.item_count,
actual_items = items.len(),
reason = "partial_batch_read",
"target store state"
); );
// Depending on strictness, this could be an error. // Depending on strictness, this could be an error.
} else if items.is_empty() { } else if items.is_empty() {
@@ -538,7 +592,15 @@ where
Ok(()) => {} Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => { Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// File already gone — still clean up the entries map to avoid stale keys. // File already gone — still clean up the entries map to avoid stale keys.
warn!("File not found for key {} during del, cleaning up entries map.", key); warn!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "delete",
key = %key,
result = "file_missing",
"target store state"
);
} }
Err(e) => return Err(StoreError::Io(e)), Err(e) => return Err(StoreError::Io(e)),
} }
@@ -550,9 +612,25 @@ where
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?; .map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
if entries.remove(&key.to_key_string()).is_none() { if entries.remove(&key.to_key_string()).is_none() {
debug!("Key {} not found in entries map during del, might have been already removed.", key); debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "delete",
key = %key,
result = "entry_missing",
"target store state"
);
} }
debug!("Deleted event from store: {}", key.to_string()); debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "delete",
key = %key,
result = "deleted",
"target store state"
);
Ok(()) Ok(())
} }
@@ -580,7 +658,14 @@ where
let entries = match self.entries.read() { let entries = match self.entries.read() {
Ok(entries) => entries, Ok(entries) => entries,
Err(_) => { Err(_) => {
debug!("Failed to acquire read lock on entries for listing"); debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "list",
result = "lock_unavailable",
"target store state"
);
return Vec::new(); return Vec::new();
} }
}; };
@@ -597,7 +682,14 @@ where
match self.entries.read() { match self.entries.read() {
Ok(entries) => entries.len(), Ok(entries) => entries.len(),
Err(_) => { Err(_) => {
debug!("Failed to acquire read lock on entries for len"); debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "len",
result = "lock_unavailable",
"target store state"
);
0 0
} }
} }
crates/targets/src/target/mqtt.rs
+464 -56
View File
@@ -60,6 +60,10 @@ const DEFAULT_MQTT_TLS_PORT: u16 = 8883;
const DEFAULT_MQTT_WSS_PORT: u16 = 443; const DEFAULT_MQTT_WSS_PORT: u16 = 443;
const MAX_MQTT_PACKET_SIZE_BYTES: u32 = 100 * 1024 * 1024; const MAX_MQTT_PACKET_SIZE_BYTES: u32 = 100 * 1024 * 1024;
const DEFAULT_MQTT_WS_PATH_ALLOWLIST: &[&str] = &["/", "/mqtt"]; const DEFAULT_MQTT_WS_PATH_ALLOWLIST: &[&str] = &["/", "/mqtt"];
const LOG_COMPONENT_TARGETS: &str = "targets";
const LOG_SUBSYSTEM_MQTT: &str = "mqtt";
const EVENT_MQTT_TARGET_STATE: &str = "mqtt_target_state";
const EVENT_MQTT_DELIVERY_STATE: &str = "mqtt_delivery_state";
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MQTTTlsPolicy { pub enum MQTTTlsPolicy {
@@ -574,7 +578,14 @@ where
Some(MAX_MQTT_PACKET_SIZE_BYTES), Some(MAX_MQTT_PACKET_SIZE_BYTES),
)?; )?;
info!(target_id = %target_id, "MQTT target created"); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "created",
"mqtt target state"
);
Ok(MQTTTarget::<E> { Ok(MQTTTarget::<E> {
id: target_id, id: target_id,
args, args,
@@ -593,7 +604,14 @@ where
#[instrument(skip(self), fields(target_id = %self.id))] #[instrument(skip(self), fields(target_id = %self.id))]
async fn init(&self) -> Result<(), TargetError> { async fn init(&self) -> Result<(), TargetError> {
if self.connected.load(Ordering::SeqCst) { if self.connected.load(Ordering::SeqCst) {
debug!(target_id = %self.id, "Already connected."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "already_connected",
"mqtt target state"
);
return Ok(()); return Ok(());
} }
@@ -607,7 +625,14 @@ where
let _ = bg_task_manager let _ = bg_task_manager
.init_cell .init_cell
.get_or_try_init(|| async { .get_or_try_init(|| async {
debug!(target_id = %target_id_clone, "Initializing MQTT background task."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "background_task_initializing",
"mqtt target state"
);
// Use the latest MqttOptions (may have been updated by TLS reload coordinator). // Use the latest MqttOptions (may have been updated by TLS reload coordinator).
let mqtt_options: MqttOptions = (**pending_mqtt_options.load()).clone(); let mqtt_options: MqttOptions = (**pending_mqtt_options.load()).clone();
@@ -615,30 +640,67 @@ where
let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build(); let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build();
if let Err(e) = new_client.subscribe(&args_clone.topic, args_clone.qos).await { if let Err(e) = new_client.subscribe(&args_clone.topic, args_clone.qos).await {
error!(target_id = %target_id_clone, error = %e, "Failed to subscribe to MQTT topic during init"); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "subscribe_failed",
error = %e,
"mqtt target state"
);
return Err(TargetError::Network(format!("MQTT subscribe failed: {e}"))); return Err(TargetError::Network(format!("MQTT subscribe failed: {e}")));
} }
let mut rx_guard = bg_task_manager.initial_cancel_rx.lock().await; let mut rx_guard = bg_task_manager.initial_cancel_rx.lock().await;
let cancel_rx = rx_guard.take().ok_or_else(|| { let cancel_rx = rx_guard.take().ok_or_else(|| {
error!(target_id = %target_id_clone, "MQTT cancel receiver already taken for task."); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "cancel_receiver_unavailable",
"mqtt target state"
);
TargetError::Configuration("MQTT cancel receiver already taken for task".to_string()) TargetError::Configuration("MQTT cancel receiver already taken for task".to_string())
})?; })?;
drop(rx_guard); drop(rx_guard);
*client_arc.lock().await = Some(new_client.clone()); *client_arc.lock().await = Some(new_client.clone());
info!(target_id = %target_id_clone, "Spawning MQTT event loop task."); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "event_loop_spawning",
"mqtt target state"
);
let task_handle = let task_handle =
tokio::spawn(run_mqtt_event_loop(eventloop, connected_arc.clone(), target_id_clone.clone(), cancel_rx)); tokio::spawn(run_mqtt_event_loop(eventloop, connected_arc.clone(), target_id_clone.clone(), cancel_rx));
Ok(task_handle) Ok(task_handle)
}) })
.await .await
.map_err(|e: TargetError| { .map_err(|e: TargetError| {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT background task"); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "background_task_init_failed",
error = %e,
"mqtt target state"
);
e e
})?; })?;
debug!(target_id = %self.id, "MQTT background task initialized successfully."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "background_task_initialized",
"mqtt target state"
);
match tokio::time::timeout(DEFAULT_CONNECTION_TIMEOUT, async { match tokio::time::timeout(DEFAULT_CONNECTION_TIMEOUT, async {
while !self.connected.load(Ordering::SeqCst) { while !self.connected.load(Ordering::SeqCst) {
@@ -646,23 +708,51 @@ where
&& handle.is_finished() && handle.is_finished()
&& !self.connected.load(Ordering::SeqCst) && !self.connected.load(Ordering::SeqCst)
{ {
error!(target_id = %self.id, "MQTT background task exited prematurely before connection was established."); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "background_task_exited_before_connect",
"mqtt target state"
);
return Err(TargetError::Network("MQTT background task exited prematurely".to_string())); return Err(TargetError::Network("MQTT background task exited prematurely".to_string()));
} }
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
} }
debug!(target_id = %self.id, "MQTT target connected successfully."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "connected",
"mqtt target state"
);
Ok(()) Ok(())
}) })
.await .await
{ {
Ok(Ok(_)) => { Ok(Ok(_)) => {
info!(target_id = %self.id, "MQTT target initialized and connected."); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "ready",
"mqtt target state"
);
Ok(()) Ok(())
} }
Ok(Err(e)) => Err(e), Ok(Err(e)) => Err(e),
Err(_) => { Err(_) => {
error!(target_id = %self.id, "Timeout waiting for MQTT connection after task spawn."); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "connect_timeout",
"mqtt target state"
);
Err(TargetError::Network("Timeout waiting for MQTT connection".to_string())) Err(TargetError::Network("Timeout waiting for MQTT connection".to_string()))
} }
} }
@@ -680,12 +770,16 @@ where
.ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?; .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
debug!( debug!(
target = %self.id, event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
bucket = %meta.bucket_name, bucket = %meta.bucket_name,
object = %meta.object_name, object = %meta.object_name,
event = %meta.event_name, event = %meta.event_name,
payload_len = body.len(), payload_len = body.len(),
"Sending MQTT payload" state = "publishing",
"mqtt delivery state"
); );
client client
@@ -693,7 +787,16 @@ where
.await .await
.map_err(|e| { .map_err(|e| {
if e.to_string().contains("Connection") || e.to_string().contains("Timeout") { if e.to_string().contains("Connection") || e.to_string().contains("Timeout") {
warn!(target_id = %self.id, error = %e, "Publish failed due to connection issue, marking as not connected."); warn!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "publish_failed",
reason = "connectivity_error",
error = %e,
"mqtt delivery state"
);
let err = TargetError::NotConnected; let err = TargetError::NotConnected;
mark_target_disconnected_on_connectivity_error(&self.connected, &err); mark_target_disconnected_on_connectivity_error(&self.connected, &err);
err err
@@ -702,7 +805,15 @@ where
} }
})?; })?;
debug!(target_id = %self.id, topic = %self.args.topic, "Event published to MQTT topic"); debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
topic = %self.args.topic,
state = "published",
"mqtt delivery state"
);
self.delivery_counters.record_success(); self.delivery_counters.record_success();
Ok(()) Ok(())
} }
@@ -781,14 +892,28 @@ async fn run_mqtt_event_loop(
target_id: TargetID, target_id: TargetID,
mut cancel_rx: mpsc::Receiver<()>, mut cancel_rx: mpsc::Receiver<()>,
) { ) {
info!(target_id = %target_id, "MQTT event loop task started."); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "event_loop_started",
"mqtt target state"
);
let mut initial_connection_established = false; let mut initial_connection_established = false;
loop { loop {
tokio::select! { tokio::select! {
biased; biased;
_ = cancel_rx.recv() => { _ = cancel_rx.recv() => {
info!(target_id = %target_id, "MQTT event loop task received cancellation signal. Shutting down."); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "cancellation_received",
"mqtt target state"
);
break; break;
} }
polled_event_result = async { polled_event_result = async {
@@ -796,7 +921,14 @@ async fn run_mqtt_event_loop(
match tokio::time::timeout(EVENT_LOOP_POLL_TIMEOUT, eventloop.poll()).await { match tokio::time::timeout(EVENT_LOOP_POLL_TIMEOUT, eventloop.poll()).await {
Ok(result) => Some(result), Ok(result) => Some(result),
Err(_) => { Err(_) => {
debug!(target_id = %target_id, "MQTT poll timed out (EVENT_LOOP_POLL_TIMEOUT) while not connected or status pending."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "poll_timeout",
"mqtt target state"
);
connected_status.store(false, Ordering::SeqCst); connected_status.store(false, Ordering::SeqCst);
None None
} }
@@ -810,15 +942,38 @@ async fn run_mqtt_event_loop(
trace!(target_id = %target_id, event = ?notification, "Received MQTT event"); trace!(target_id = %target_id, event = ?notification, "Received MQTT event");
match notification { match notification {
rumqttc::Event::Incoming(Incoming::ConnAck(_conn_ack)) => { rumqttc::Event::Incoming(Incoming::ConnAck(_conn_ack)) => {
info!(target_id = %target_id, "MQTT connected (ConnAck)."); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "connack_received",
"mqtt target state"
);
connected_status.store(true, Ordering::SeqCst); connected_status.store(true, Ordering::SeqCst);
initial_connection_established = true; initial_connection_established = true;
} }
rumqttc::Event::Incoming(Incoming::Publish(publish)) => { rumqttc::Event::Incoming(Incoming::Publish(publish)) => {
debug!(target_id = %target_id, topic = ?publish.topic, payload_len = publish.payload.len(), "Received message on subscribed topic."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "publish_received",
topic = ?publish.topic,
payload_len = publish.payload.len(),
"mqtt target state"
);
} }
rumqttc::Event::Incoming(Incoming::Disconnect(_)) => { rumqttc::Event::Incoming(Incoming::Disconnect(_)) => {
info!(target_id = %target_id, "Received Disconnect packet from broker. MQTT connection lost."); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "broker_disconnected",
"mqtt target state"
);
connected_status.store(false, Ordering::SeqCst); connected_status.store(false, Ordering::SeqCst);
} }
rumqttc::Event::Incoming(Incoming::PingResp(_)) => { rumqttc::Event::Incoming(Incoming::PingResp(_)) => {
@@ -832,7 +987,14 @@ async fn run_mqtt_event_loop(
} }
// Process other incoming packet types as needed (PubRec, PubRel, PubComp, UnsubAck) // Process other incoming packet types as needed (PubRec, PubRel, PubComp, UnsubAck)
rumqttc::Event::Outgoing(Outgoing::Disconnect) => { rumqttc::Event::Outgoing(Outgoing::Disconnect) => {
info!(target_id = %target_id, "MQTT outgoing disconnect initiated by client."); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "client_disconnect_requested",
"mqtt target state"
);
connected_status.store(false, Ordering::SeqCst); connected_status.store(false, Ordering::SeqCst);
} }
rumqttc::Event::Outgoing(Outgoing::PingReq) => { rumqttc::Event::Outgoing(Outgoing::PingReq) => {
@@ -848,7 +1010,15 @@ async fn run_mqtt_event_loop(
} }
Some(Err(e)) => { Some(Err(e)) => {
connected_status.store(false, Ordering::SeqCst); connected_status.store(false, Ordering::SeqCst);
error!(target_id = %target_id, error = %e, "Error from MQTT event loop poll"); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "poll_failed",
error = %e,
"mqtt target state"
);
if matches!(e, if matches!(e,
ConnectionError::Io(_) | ConnectionError::Io(_) |
@@ -856,12 +1026,28 @@ async fn run_mqtt_event_loop(
ConnectionError::ConnectionRefused(_) | ConnectionError::ConnectionRefused(_) |
ConnectionError::Tls(_) ConnectionError::Tls(_)
) { ) {
warn!(target_id = %target_id, error = %e, "MQTT connection error. Relying on rumqttc for reconnection if applicable."); warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "reconnect_pending",
error = %e,
"mqtt target state"
);
} }
// Here you can decide whether to break loops based on the error type. // Here you can decide whether to break loops based on the error type.
// For example, for some unrecoverable errors. // For example, for some unrecoverable errors.
if is_fatal_mqtt_error(&e) { if is_fatal_mqtt_error(&e) {
error!(target_id = %target_id, error = %e, "Fatal MQTT error, terminating event loop."); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "fatal_error",
error = %e,
"mqtt target state"
);
break; break;
} }
// rumqttc's eventloop.poll() may return Err and terminate after some errors, // rumqttc's eventloop.poll() may return Err and terminate after some errors,
@@ -871,7 +1057,14 @@ async fn run_mqtt_event_loop(
tokio::time::sleep(Duration::from_secs(1)).await; tokio::time::sleep(Duration::from_secs(1)).await;
} }
None => { None => {
warn!(target_id = %target_id, "Timeout during initial poll or pending state, will retry."); warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "poll_retry_scheduled",
"mqtt target state"
);
continue; continue;
} }
} }
@@ -879,7 +1072,14 @@ async fn run_mqtt_event_loop(
} }
} }
connected_status.store(false, Ordering::SeqCst); connected_status.store(false, Ordering::SeqCst);
info!(target_id = %target_id, "MQTT event loop task finished."); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "event_loop_finished",
"mqtt target state"
);
} }
/// Check whether the given MQTT connection error should be considered a fatal error, /// Check whether the given MQTT connection error should be considered a fatal error,
@@ -938,26 +1138,61 @@ where
#[instrument(skip(self), fields(target_id = %self.id))] #[instrument(skip(self), fields(target_id = %self.id))]
async fn is_active(&self) -> Result<bool, TargetError> { async fn is_active(&self) -> Result<bool, TargetError> {
debug!(target_id = %self.id, "Checking if MQTT target is active."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "activity_check",
"mqtt target state"
);
if self.client.lock().await.is_none() && !self.connected.load(Ordering::SeqCst) { if self.client.lock().await.is_none() && !self.connected.load(Ordering::SeqCst) {
// Check if the background task is running and has not panicked // Check if the background task is running and has not panicked
if let Some(handle) = self.bg_task_manager.init_cell.get() if let Some(handle) = self.bg_task_manager.init_cell.get()
&& handle.is_finished() && handle.is_finished()
{ {
error!(target_id = %self.id, "MQTT background task has finished, possibly due to an error. Target is not active."); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "inactive_background_task_finished",
"mqtt target state"
);
return Err(TargetError::Network("MQTT background task terminated".to_string())); return Err(TargetError::Network("MQTT background task terminated".to_string()));
} }
debug!(target_id = %self.id, "MQTT client not yet initialized or task not running/connected."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "inactive_client_unavailable",
"mqtt target state"
);
return Err(TargetError::Configuration( return Err(TargetError::Configuration(
"MQTT client not available or not initialized/connected".to_string(), "MQTT client not available or not initialized/connected".to_string(),
)); ));
} }
if self.connected.load(Ordering::SeqCst) { if self.connected.load(Ordering::SeqCst) {
debug!(target_id = %self.id, "MQTT target is active (connected flag is true)."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "active",
"mqtt target state"
);
Ok(true) Ok(true)
} else { } else {
debug!(target_id = %self.id, "MQTT target is not connected (connected flag is false)."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "inactive_not_connected",
"mqtt target state"
);
Err(TargetError::NotConnected) Err(TargetError::NotConnected)
} }
} }
@@ -973,14 +1208,36 @@ where
}; };
if let Some(store) = &self.store { if let Some(store) = &self.store {
debug!(target_id = %self.id, "Event saved to store start"); debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_enqueue_started",
"mqtt delivery state"
);
match persist_queued_payload_to_store(store.as_ref(), &queued) { match persist_queued_payload_to_store(store.as_ref(), &queued) {
Ok(_) => { Ok(_) => {
debug!(target_id = %self.id, "Event saved to store for MQTT target successfully."); debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_enqueued",
"mqtt delivery state"
);
Ok(()) Ok(())
} }
Err(e) => { Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to save event to store"); error!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_enqueue_failed",
error = %e,
"mqtt delivery state"
);
self.delivery_counters.record_final_failure(); self.delivery_counters.record_final_failure();
Err(e) Err(e)
} }
@@ -991,18 +1248,47 @@ where
} }
if !self.connected.load(Ordering::SeqCst) { if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Attempting to send directly but not connected; trying to init."); warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "direct_send_requires_init",
"mqtt target state"
);
// Call the struct's init method, not the trait's default // Call the struct's init method, not the trait's default
match MQTTTarget::<E>::init(self).await { match MQTTTarget::<E>::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."), Ok(_) => debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed",
"mqtt target state"
),
Err(e) => { Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target."); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_failed",
error = %e,
"mqtt target state"
);
self.delivery_counters.record_final_failure(); self.delivery_counters.record_final_failure();
return Err(TargetError::NotConnected); return Err(TargetError::NotConnected);
} }
} }
if !self.connected.load(Ordering::SeqCst) { if !self.connected.load(Ordering::SeqCst) {
error!(target_id = %self.id, "Cannot save (send directly) as target is not active after init attempt."); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed_not_connected",
"mqtt target state"
);
self.delivery_counters.record_final_failure(); self.delivery_counters.record_final_failure();
return Err(TargetError::NotConnected); return Err(TargetError::NotConnected);
} }
@@ -1017,50 +1303,143 @@ where
#[instrument(skip(self, body, meta), fields(target_id = %self.id))] #[instrument(skip(self, body, meta), fields(target_id = %self.id))]
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> { async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
debug!(target_id = %self.id, ?key, "Attempting to send queued payload from store."); debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_started",
"mqtt delivery state"
);
if !self.is_enabled() { if !self.is_enabled() {
return Err(TargetError::Disabled); return Err(TargetError::Disabled);
} }
if !self.connected.load(Ordering::SeqCst) { if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Not connected; trying to init before sending from store."); warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_replay_requires_init",
"mqtt target state"
);
match MQTTTarget::<E>::init(self).await { match MQTTTarget::<E>::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."), Ok(_) => debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed",
"mqtt target state"
),
Err(e) => { Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target."); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_failed",
error = %e,
"mqtt target state"
);
return Err(TargetError::NotConnected); return Err(TargetError::NotConnected);
} }
} }
if !self.connected.load(Ordering::SeqCst) { if !self.connected.load(Ordering::SeqCst) {
error!(target_id = %self.id, "Cannot send from store as target is not active after init attempt."); error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed_not_connected",
"mqtt target state"
);
return Err(TargetError::NotConnected); return Err(TargetError::NotConnected);
} }
} }
debug!(target_id = %self.id, ?key, "Sending event from store."); debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_publishing",
"mqtt delivery state"
);
if let Err(e) = self.send_body(body, &meta).await { if let Err(e) = self.send_body(body, &meta).await {
if matches!(e, TargetError::NotConnected) { if matches!(e, TargetError::NotConnected) {
warn!(target_id = %self.id, "Failed to send event from store: Not connected. Event remains in store."); warn!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_deferred",
reason = "not_connected",
"mqtt delivery state"
);
return Err(TargetError::NotConnected); return Err(TargetError::NotConnected);
} }
error!(target_id = %self.id, error = %e, "Failed to send event from store with an unexpected error."); error!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_failed",
error = %e,
"mqtt delivery state"
);
return Err(e); return Err(e);
} }
debug!(target_id = %self.id, ?key, "Event sent from store successfully."); debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_published",
"mqtt delivery state"
);
Ok(()) Ok(())
} }
async fn close(&self) -> Result<(), TargetError> { async fn close(&self) -> Result<(), TargetError> {
info!(target_id = %self.id, "Attempting to close MQTT target."); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "closing",
"mqtt target state"
);
if let Err(e) = self.bg_task_manager.cancel_tx.send(()).await { if let Err(e) = self.bg_task_manager.cancel_tx.send(()).await {
warn!(target_id = %self.id, error = %e, "Failed to send cancel signal to MQTT background task. It might have already exited."); warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "cancel_signal_failed",
error = %e,
"mqtt target state"
);
} }
// Wait for the task to finish if it was initialized // Wait for the task to finish if it was initialized
if let Some(_task_handle) = self.bg_task_manager.init_cell.get() { if let Some(_task_handle) = self.bg_task_manager.init_cell.get() {
debug!(target_id = %self.id, "Waiting for MQTT background task to complete..."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "waiting_for_event_loop",
"mqtt target state"
);
// It's tricky to await here if close is called from a sync context or Drop // It's tricky to await here if close is called from a sync context or Drop
// For async close, this is fine. Consider a timeout. // For async close, this is fine. Consider a timeout.
// let _ = tokio::time::timeout(Duration::from_secs(5), task_handle.await).await; // let _ = tokio::time::timeout(Duration::from_secs(5), task_handle.await).await;
@@ -1069,9 +1448,24 @@ where
} }
if let Some(client_instance) = self.client.lock().await.take() { if let Some(client_instance) = self.client.lock().await.take() {
info!(target_id = %self.id, "Disconnecting MQTT client."); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "disconnecting_client",
"mqtt target state"
);
if let Err(e) = client_instance.disconnect().await { if let Err(e) = client_instance.disconnect().await {
warn!(target_id = %self.id, error = %e, "Error during MQTT client disconnect."); warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "disconnect_failed",
error = %e,
"mqtt target state"
);
} }
} }
@@ -1083,7 +1477,14 @@ where
} }
self.connected.store(false, Ordering::SeqCst); self.connected.store(false, Ordering::SeqCst);
info!(target_id = %self.id, "MQTT target close method finished."); info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "closed",
"mqtt target state"
);
Ok(()) Ok(())
} }
@@ -1097,7 +1498,14 @@ where
async fn init(&self) -> Result<(), TargetError> { async fn init(&self) -> Result<(), TargetError> {
if !self.is_enabled() { if !self.is_enabled() {
debug!(target_id = %self.id, "Target is disabled, skipping init."); debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "disabled",
"mqtt target state"
);
return Ok(()); return Ok(());
} }
// Call the internal init logic // Call the internal init logic
crates/targets/src/target/webhook.rs
+120 -22
View File
@@ -45,6 +45,11 @@ use std::{
use tokio::sync::mpsc; use tokio::sync::mpsc;
use tracing::{debug, error, info, instrument, warn}; use tracing::{debug, error, info, instrument, warn};
const LOG_COMPONENT_TARGETS: &str = "targets";
const LOG_SUBSYSTEM_WEBHOOK: &str = "webhook";
const EVENT_WEBHOOK_TARGET_STATE: &str = "webhook_target_state";
const EVENT_WEBHOOK_DELIVERY_STATE: &str = "webhook_delivery_state";
/// Arguments for configuring a Webhook target /// Arguments for configuring a Webhook target
#[derive(Clone)] #[derive(Clone)]
pub struct WebhookArgs { pub struct WebhookArgs {
@@ -190,7 +195,14 @@ where
// Create a cancel channel // Create a cancel channel
let (cancel_sender, _) = mpsc::channel(1); let (cancel_sender, _) = mpsc::channel(1);
info!(target_id = %target_id.id, "Webhook target created"); info!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %target_id.id,
state = "created",
"webhook target state"
);
Ok(WebhookTarget::<E> { Ok(WebhookTarget::<E> {
id: target_id, id: target_id,
args, args,
@@ -220,8 +232,13 @@ where
// DANGEROUS: For testing only, skip all certificate verification // DANGEROUS: For testing only, skip all certificate verification
client_builder = client_builder.danger_accept_invalid_certs(true); client_builder = client_builder.danger_accept_invalid_certs(true);
warn!( warn!(
"Webhook target '{}' is configured to skip TLS verification. This is insecure and should not be used in production.", event = EVENT_WEBHOOK_TARGET_STATE,
args.endpoint component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
endpoint = %args.endpoint,
state = "tls_verification_skipped",
fallback = "danger_accept_invalid_certs",
"webhook target state"
); );
} else if !args.client_ca.is_empty() { } else if !args.client_ca.is_empty() {
// Use user-provided custom CA certificate // Use user-provided custom CA certificate
@@ -301,10 +318,14 @@ where
match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await { match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await {
Ok(Ok(resp)) => { Ok(Ok(resp)) => {
debug!( debug!(
target = %self.id, event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
status = %resp.status(), status = %resp.status(),
health_check_url = %health_check_url, health_check_url = %health_check_url,
"Webhook health check request succeeded" state = "reachability_probe_succeeded",
"webhook target state"
); );
Ok(true) Ok(true)
} }
@@ -337,7 +358,15 @@ where
// behavior matches real delivery while avoiding path-specific false negatives. // behavior matches real delivery while avoiding path-specific false negatives.
match self.probe_reachability().await { match self.probe_reachability().await {
Ok(true) => { Ok(true) => {
debug!("Webhook target {} reachability probe succeeded via {:?}", self.id, self.health_check_url); debug!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
health_check_url = ?self.health_check_url,
state = "reachable",
"webhook target state"
);
} }
Ok(false) => { Ok(false) => {
return Err(TargetError::NotConnected); return Err(TargetError::NotConnected);
@@ -348,7 +377,14 @@ where
} }
self.initialized.store(true, Ordering::SeqCst); self.initialized.store(true, Ordering::SeqCst);
info!("Webhook target {} initialized", self.id); info!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "initialized",
"webhook target state"
);
Ok(()) Ok(())
} }
@@ -357,14 +393,17 @@ where
} }
async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> { async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
info!("Webhook sending queued payload to target: {}", self.id);
debug!( debug!(
target = %self.id, event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
bucket = %meta.bucket_name, bucket = %meta.bucket_name,
object = %meta.object_name, object = %meta.object_name,
event = %meta.event_name, payload_event = %meta.event_name,
payload_len = body.len(), payload_len = body.len(),
"Sending webhook payload" state = "sending",
"webhook delivery state"
); );
// When a TLS reload adapter is attached, it drives client rebuilds in // When a TLS reload adapter is attached, it drives client rebuilds in
@@ -406,7 +445,15 @@ where
let status = resp.status(); let status = resp.status();
if status.is_success() { if status.is_success() {
debug!("Event sent to webhook target: {}", self.id); debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
status = %status,
state = "sent",
"webhook delivery state"
);
self.delivery_counters.record_success(); self.delivery_counters.record_success();
Ok(()) Ok(())
} else if status == StatusCode::FORBIDDEN { } else if status == StatusCode::FORBIDDEN {
@@ -454,13 +501,28 @@ where
self.delivery_counters.record_final_failure(); self.delivery_counters.record_final_failure();
return Err(e); return Err(e);
} }
debug!("Event saved to store for target: {}", self.id); debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "store_enqueued",
"webhook delivery state"
);
Ok(()) Ok(())
} else { } else {
match self.init().await { match self.init().await {
Ok(_) => (), Ok(_) => (),
Err(e) => { Err(e) => {
error!("Failed to initialize Webhook target {}: {}", self.id.id, e); error!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id.id,
state = "init_failed",
error = %e,
"webhook target state"
);
self.delivery_counters.record_final_failure(); self.delivery_counters.record_final_failure();
return Err(TargetError::NotConnected); return Err(TargetError::NotConnected);
} }
@@ -474,13 +536,27 @@ where
} }
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> { async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
debug!("Sending queued payload from store for target: {}, key: {}", self.id, key); debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
key = %key,
state = "store_replay_started",
"webhook delivery state"
);
match self.init().await { match self.init().await {
Ok(_) => { Ok(_) => {}
debug!("Event sent to store for target: {}", self.name());
}
Err(e) => { Err(e) => {
error!("Failed to initialize Webhook target {}: {}", self.id.id, e); error!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id.id,
state = "init_failed",
error = %e,
"webhook target state"
);
return Err(TargetError::NotConnected); return Err(TargetError::NotConnected);
} }
} }
@@ -492,7 +568,15 @@ where
return Err(e); return Err(e);
} }
debug!("Event sent from store and deleted for target: {}", self.id); debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
key = %key,
state = "store_replay_sent",
"webhook delivery state"
);
Ok(()) Ok(())
} }
@@ -500,7 +584,14 @@ where
// Send cancel signal to background tasks // Send cancel signal to background tasks
let _ = self.cancel_sender.try_send(()); let _ = self.cancel_sender.try_send(());
// Adapter cleanup is done by the coordinator; no local state to reset. // Adapter cleanup is done by the coordinator; no local state to reset.
info!("Webhook target closed: {}", self.id); info!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "closed",
"webhook target state"
);
Ok(()) Ok(())
} }
@@ -515,7 +606,14 @@ where
async fn init(&self) -> Result<(), TargetError> { async fn init(&self) -> Result<(), TargetError> {
if !self.is_enabled() { if !self.is_enabled() {
debug!("Webhook target {} is disabled, skipping initialization", self.id); debug!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "disabled",
"webhook target state"
);
return Ok(()); return Ok(());
} }
self.init_inner().await self.init_inner().await
rustfs/src/admin/handlers/bucket_meta.rs
+195 -41
View File
@@ -65,6 +65,10 @@ use time::OffsetDateTime;
use tracing::warn; use tracing::warn;
use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_BUCKET_META: &str = "bucket_meta";
const EVENT_ADMIN_BUCKET_META_STATE: &str = "admin_bucket_meta_state";
#[derive(Debug, Default, serde::Deserialize)] #[derive(Debug, Default, serde::Deserialize)]
pub struct ExportBucketMetadataQuery { pub struct ExportBucketMetadataQuery {
pub bucket: String, pub bucket: String,
@@ -106,7 +110,7 @@ impl Operation for ExportBucketMetadata {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: ExportBucketMetadataQuery = let input: ExportBucketMetadataQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
ExportBucketMetadataQuery::default() ExportBucketMetadataQuery::default()
@@ -114,7 +118,7 @@ impl Operation for ExportBucketMetadata {
}; };
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -131,19 +135,19 @@ impl Operation for ExportBucketMetadata {
.await?; .await?;
let Some(store) = new_object_layer_fn() else { let Some(store) = new_object_layer_fn() else {
return Err(s3_error!(InvalidRequest, "object store not init")); return Err(s3_error!(InternalError, "object store is not initialized"));
}; };
let buckets = if query.bucket.is_empty() { let buckets = if query.bucket.is_empty() {
store store
.list_bucket(&BucketOptions::default()) .list_bucket(&BucketOptions::default())
.await .await
.map_err(|e| s3_error!(InternalError, "list buckets failed: {e}"))? .map_err(|e| s3_error!(InternalError, "failed to list buckets: {e}"))?
} else { } else {
let bucket = store let bucket = store
.get_bucket_info(&query.bucket, &BucketOptions::default()) .get_bucket_info(&query.bucket, &BucketOptions::default())
.await .await
.map_err(|e| s3_error!(InternalError, "get bucket failed: {e}"))?; .map_err(|e| s3_error!(InternalError, "failed to load bucket: {e}"))?;
vec![bucket] vec![bucket]
}; };
@@ -173,17 +177,17 @@ impl Operation for ExportBucketMetadata {
if e == StorageError::ConfigNotFound { if e == StorageError::ConfigNotFound {
continue; continue;
} }
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
} }
}; };
let config_json = let config_json = serde_json::to_vec(&config)
serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; .map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
zip_writer zip_writer
.start_file(conf_path, SimpleFileOptions::default()) .start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?; .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
zip_writer zip_writer
.write_all(&config_json) .write_all(&config_json)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?; .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
} }
BUCKET_NOTIFICATION_CONFIG => { BUCKET_NOTIFICATION_CONFIG => {
let config: s3s::dto::NotificationConfiguration = let config: s3s::dto::NotificationConfiguration =
@@ -215,18 +219,18 @@ impl Operation for ExportBucketMetadata {
if e == StorageError::ConfigNotFound { if e == StorageError::ConfigNotFound {
continue; continue;
} }
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
} }
}; };
let config_xml = let config_xml =
serialize(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; serialize(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
zip_writer zip_writer
.start_file(conf_path, SimpleFileOptions::default()) .start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?; .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
zip_writer zip_writer
.write_all(&config_xml) .write_all(&config_xml)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?; .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
} }
BUCKET_TAGGING_CONFIG => { BUCKET_TAGGING_CONFIG => {
let config: Tagging = match metadata_sys::get_tagging_config(&bucket.name).await { let config: Tagging = match metadata_sys::get_tagging_config(&bucket.name).await {
@@ -235,18 +239,18 @@ impl Operation for ExportBucketMetadata {
if e == StorageError::ConfigNotFound { if e == StorageError::ConfigNotFound {
continue; continue;
} }
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
} }
}; };
let config_xml = let config_xml =
serialize(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; serialize(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
zip_writer zip_writer
.start_file(conf_path, SimpleFileOptions::default()) .start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?; .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
zip_writer zip_writer
.write_all(&config_xml) .write_all(&config_xml)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?; .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
} }
BUCKET_QUOTA_CONFIG_FILE => { BUCKET_QUOTA_CONFIG_FILE => {
let config: BucketQuota = match metadata_sys::get_quota_config(&bucket.name).await { let config: BucketQuota = match metadata_sys::get_quota_config(&bucket.name).await {
@@ -376,7 +380,7 @@ impl Operation for ExportBucketMetadata {
let zip_bytes = zip_writer let zip_bytes = zip_writer
.finish() .finish()
.map_err(|e| s3_error!(InternalError, "finish zip failed: {e}"))?; .map_err(|e| s3_error!(InternalError, "failed to finalize export archive: {e}"))?;
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/zip".parse().unwrap()); header.insert(CONTENT_TYPE, "application/zip".parse().unwrap());
header.insert(CONTENT_DISPOSITION, "attachment; filename=bucket-meta.zip".parse().unwrap()); header.insert(CONTENT_DISPOSITION, "attachment; filename=bucket-meta.zip".parse().unwrap());
@@ -399,7 +403,7 @@ impl Operation for ImportBucketMetadata {
let _query = { let _query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: ImportBucketMetadataQuery = let input: ImportBucketMetadataQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
ImportBucketMetadataQuery::default() ImportBucketMetadataQuery::default()
@@ -407,7 +411,7 @@ impl Operation for ImportBucketMetadata {
}; };
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -427,24 +431,33 @@ impl Operation for ImportBucketMetadata {
let body = match input.store_all_limited(MAX_BUCKET_METADATA_IMPORT_SIZE).await { let body = match input.store_all_limited(MAX_BUCKET_METADATA_IMPORT_SIZE).await {
Ok(b) => b, Ok(b) => b,
Err(e) => { Err(e) => {
warn!("get body failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "body_read_failed",
error = ?e,
"admin bucket meta state"
);
return Err(s3_error!(InvalidRequest, "bucket metadata import body too large or failed to read")); return Err(s3_error!(InvalidRequest, "bucket metadata import body too large or failed to read"));
} }
}; };
let mut zip_reader = ZipArchive::new(Cursor::new(body)).map_err(|e| s3_error!(InternalError, "get body failed: {e}"))?; let mut zip_reader =
ZipArchive::new(Cursor::new(body)).map_err(|e| s3_error!(InternalError, "failed to read import archive: {e}"))?;
// First pass: read all file contents into memory // First pass: read all file contents into memory
let mut file_contents = Vec::new(); let mut file_contents = Vec::new();
for i in 0..zip_reader.len() { for i in 0..zip_reader.len() {
let mut file = zip_reader let mut file = zip_reader
.by_index(i) .by_index(i)
.map_err(|e| s3_error!(InternalError, "get file failed: {e}"))?; .map_err(|e| s3_error!(InternalError, "failed to read archive entry: {e}"))?;
let file_path = file.name().to_string(); let file_path = file.name().to_string();
let mut content = Vec::new(); let mut content = Vec::new();
file.read_to_end(&mut content) file.read_to_end(&mut content)
.map_err(|e| s3_error!(InternalError, "read file failed: {e}"))?; .map_err(|e| s3_error!(InternalError, "failed to read archive entry content: {e}"))?;
file_contents.push((file_path, content)); file_contents.push((file_path, content));
} }
@@ -455,7 +468,15 @@ impl Operation for ImportBucketMetadata {
let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>(); let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>();
if file_path_split.len() < 2 { if file_path_split.len() < 2 {
warn!("file path is invalid: {}", file_path); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "invalid_file_path",
file_path = %file_path,
"admin bucket meta state"
);
continue; continue;
} }
@@ -474,17 +495,33 @@ impl Operation for ImportBucketMetadata {
} }
Err(e) => { Err(e) => {
if e == StorageError::ConfigNotFound { if e == StorageError::ConfigNotFound {
warn!("bucket metadata not found: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "bucket_metadata_missing",
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
warn!("get bucket metadata failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "bucket_metadata_load_failed",
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
}; };
} }
let Some(store) = new_object_layer_fn() else { let Some(store) = new_object_layer_fn() else {
return Err(s3_error!(InvalidRequest, "object store not init")); return Err(s3_error!(InternalError, "object store is not initialized"));
}; };
let update_at = OffsetDateTime::now_utc(); let update_at = OffsetDateTime::now_utc();
@@ -494,7 +531,15 @@ impl Operation for ImportBucketMetadata {
let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>(); let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>();
if file_path_split.len() < 2 { if file_path_split.len() < 2 {
warn!("file path is invalid: {}", file_path); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "invalid_file_path",
file_path = %file_path,
"admin bucket meta state"
);
continue; continue;
} }
@@ -513,7 +558,16 @@ impl Operation for ImportBucketMetadata {
) )
.await .await
{ {
warn!("create bucket failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "bucket_create_failed",
bucket = %bucket_name,
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
@@ -527,7 +581,17 @@ impl Operation for ImportBucketMetadata {
let config: BucketPolicy = match serde_json::from_slice(&content) { let config: BucketPolicy = match serde_json::from_slice(&content) {
Ok(config) => config, Ok(config) => config,
Err(e) => { Err(e) => {
warn!("deserialize config failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
}; };
@@ -542,7 +606,17 @@ impl Operation for ImportBucketMetadata {
} }
BUCKET_NOTIFICATION_CONFIG => { BUCKET_NOTIFICATION_CONFIG => {
if let Err(e) = deserialize::<s3s::dto::NotificationConfiguration>(&content) { if let Err(e) = deserialize::<s3s::dto::NotificationConfiguration>(&content) {
warn!("deserialize config failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
@@ -553,7 +627,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_LIFECYCLE_CONFIG => { BUCKET_LIFECYCLE_CONFIG => {
if let Err(e) = deserialize::<BucketLifecycleConfiguration>(&content) { if let Err(e) = deserialize::<BucketLifecycleConfiguration>(&content) {
warn!("deserialize config failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
@@ -564,7 +648,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_SSECONFIG => { BUCKET_SSECONFIG => {
if let Err(e) = deserialize::<ServerSideEncryptionConfiguration>(&content) { if let Err(e) = deserialize::<ServerSideEncryptionConfiguration>(&content) {
warn!("deserialize config failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
@@ -575,7 +669,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_TAGGING_CONFIG => { BUCKET_TAGGING_CONFIG => {
if let Err(e) = deserialize::<Tagging>(&content) { if let Err(e) = deserialize::<Tagging>(&content) {
warn!("deserialize config failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
@@ -586,7 +690,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_QUOTA_CONFIG_FILE => { BUCKET_QUOTA_CONFIG_FILE => {
if let Err(e) = serde_json::from_slice::<BucketQuota>(&content) { if let Err(e) = serde_json::from_slice::<BucketQuota>(&content) {
warn!("deserialize config failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
@@ -597,7 +711,17 @@ impl Operation for ImportBucketMetadata {
OBJECT_LOCK_CONFIG => { OBJECT_LOCK_CONFIG => {
if let Err(e) = deserialize::<ObjectLockConfiguration>(&content) { if let Err(e) = deserialize::<ObjectLockConfiguration>(&content) {
warn!("deserialize config failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
@@ -608,7 +732,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_VERSIONING_CONFIG => { BUCKET_VERSIONING_CONFIG => {
if let Err(e) = deserialize::<VersioningConfiguration>(&content) { if let Err(e) = deserialize::<VersioningConfiguration>(&content) {
warn!("deserialize config failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
@@ -619,7 +753,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_REPLICATION_CONFIG => { BUCKET_REPLICATION_CONFIG => {
if let Err(e) = deserialize::<ReplicationConfiguration>(&content) { if let Err(e) = deserialize::<ReplicationConfiguration>(&content) {
warn!("deserialize config failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
@@ -630,7 +774,17 @@ impl Operation for ImportBucketMetadata {
BUCKET_TARGETS_FILE => { BUCKET_TARGETS_FILE => {
if let Err(e) = serde_json::from_slice::<BucketTargets>(&content) { if let Err(e) = serde_json::from_slice::<BucketTargets>(&content) {
warn!("deserialize config failed: {e}"); warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue; continue;
} }
rustfs/src/admin/handlers/group.rs
+214 -38
View File
@@ -41,6 +41,10 @@ use serde::Deserialize;
use serde_urlencoded::from_bytes; use serde_urlencoded::from_bytes;
use tracing::warn; use tracing::warn;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_GROUP: &str = "group";
const EVENT_ADMIN_GROUP_STATE: &str = "admin_group_state";
pub fn register_group_management_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> { pub fn register_group_management_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert( r.insert(
Method::GET, Method::GET,
@@ -85,10 +89,17 @@ pub struct ListGroups {}
#[async_trait::async_trait] #[async_trait::async_trait]
impl Operation for ListGroups { impl Operation for ListGroups {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle ListGroups"); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "list_groups",
state = "requested",
"admin group state"
);
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -104,14 +115,24 @@ impl Operation for ListGroups {
) )
.await?; .await?;
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let groups = iam_store.list_groups_load().await.map_err(|e| { let groups = iam_store.list_groups_load().await.map_err(|e| {
warn!("list groups failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "list_groups",
result = "load_failed",
error = ?e,
"admin group state"
);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?; })?;
let body = serde_json::to_vec(&groups).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; let body = serde_json::to_vec(&groups).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?;
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -124,10 +145,17 @@ pub struct GetGroup {}
#[async_trait::async_trait] #[async_trait::async_trait]
impl Operation for GetGroup { impl Operation for GetGroup {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle GetGroup"); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "get_group",
state = "requested",
"admin group state"
);
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -146,20 +174,31 @@ impl Operation for GetGroup {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: GroupQuery = let input: GroupQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
GroupQuery::default() GroupQuery::default()
} }
}; };
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let g = iam_store.get_group_description(&query.group).await.map_err(|e| { let g = iam_store.get_group_description(&query.group).await.map_err(|e| {
warn!("get group failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "get_group",
group = %query.group,
result = "load_failed",
error = ?e,
"admin group state"
);
iam_error_to_s3_error(e) iam_error_to_s3_error(e)
})?; })?;
let body = serde_json::to_vec(&g).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; let body = serde_json::to_vec(&g).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?;
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -189,10 +228,17 @@ pub struct DeleteGroup {}
#[async_trait::async_trait] #[async_trait::async_trait]
impl Operation for DeleteGroup { impl Operation for DeleteGroup {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle DeleteGroup"); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "delete_group",
state = "requested",
"admin group state"
);
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -210,22 +256,33 @@ impl Operation for DeleteGroup {
let group = decode_delete_group_name(&params)?; let group = decode_delete_group_name(&params)?;
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let updated_at = iam_store.remove_users_from_group(&group, vec![]).await.map_err(|e| { let updated_at = iam_store.remove_users_from_group(&group, vec![]).await.map_err(|e| {
warn!("delete group failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "delete_group",
group = %group,
result = "delete_failed",
error = ?e,
"admin group state"
);
match e { match e {
rustfs_iam::error::Error::GroupNotEmpty => { rustfs_iam::error::Error::GroupNotEmpty => {
s3_error!(InvalidRequest, "group is not empty") s3_error!(InvalidRequest, "group is not empty")
} }
rustfs_iam::error::Error::InvalidArgument => { rustfs_iam::error::Error::InvalidArgument => {
s3_error!(InvalidArgument, "{e}") s3_error!(InvalidArgument, "invalid group request: {e}")
} }
_ => { _ => {
if is_err_no_such_group(&e) { if is_err_no_such_group(&e) {
iam_error_to_s3_error(e) iam_error_to_s3_error(e)
} else { } else {
s3_error!(InternalError, "{e}") s3_error!(InternalError, "failed to delete group: {e}")
} }
} }
} }
@@ -248,7 +305,15 @@ impl Operation for DeleteGroup {
}) })
.await .await
{ {
warn!("site replication group delete hook failed, err: {err}"); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "delete_group",
result = "site_replication_hook_failed",
error = %err,
"admin group state"
);
} }
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
@@ -289,10 +354,17 @@ pub struct SetGroupStatus {}
#[async_trait::async_trait] #[async_trait::async_trait]
impl Operation for SetGroupStatus { impl Operation for SetGroupStatus {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle SetGroupStatus"); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "set_group_status",
state = "requested",
"admin group state"
);
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -311,7 +383,7 @@ impl Operation for SetGroupStatus {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: GroupQuery = let input: GroupQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
GroupQuery::default() GroupQuery::default()
@@ -322,16 +394,38 @@ impl Operation for SetGroupStatus {
return Err(s3_error!(InvalidArgument, "group is required")); return Err(s3_error!(InvalidArgument, "group is required"));
} }
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let updated_at = if let Some(status) = query.status.as_deref() { let updated_at = if let Some(status) = query.status.as_deref() {
match status { match status {
"enabled" => iam_store.set_group_status(&query.group, true).await.map_err(|e| { "enabled" => iam_store.set_group_status(&query.group, true).await.map_err(|e| {
warn!("enable group failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "set_group_status",
group = %query.group,
status = "enabled",
result = "update_failed",
error = ?e,
"admin group state"
);
iam_error_to_s3_error(e) iam_error_to_s3_error(e)
})?, })?,
"disabled" => iam_store.set_group_status(&query.group, false).await.map_err(|e| { "disabled" => iam_store.set_group_status(&query.group, false).await.map_err(|e| {
warn!("enable group failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "set_group_status",
group = %query.group,
status = "disabled",
result = "update_failed",
error = ?e,
"admin group state"
);
iam_error_to_s3_error(e) iam_error_to_s3_error(e)
})?, })?,
_ => { _ => {
@@ -363,7 +457,15 @@ impl Operation for SetGroupStatus {
}) })
.await .await
{ {
warn!("site replication group status hook failed, err: {err}"); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "set_group_status",
result = "site_replication_hook_failed",
error = %err,
"admin group state"
);
} }
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
@@ -377,10 +479,17 @@ pub struct UpdateGroupMembers {}
#[async_trait::async_trait] #[async_trait::async_trait]
impl Operation for UpdateGroupMembers { impl Operation for UpdateGroupMembers {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle UpdateGroupMembers"); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "update_group_members",
state = "requested",
"admin group state"
);
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -400,17 +509,37 @@ impl Operation for UpdateGroupMembers {
let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await { let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await {
Ok(b) => b, Ok(b) => b,
Err(e) => { Err(e) => {
warn!("get body failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "update_group_members",
result = "body_read_failed",
error = ?e,
"admin group state"
);
return Err(s3_error!(InvalidRequest, "group configuration body too large or failed to read")); return Err(s3_error!(InvalidRequest, "group configuration body too large or failed to read"));
} }
}; };
let args: GroupAddRemove = serde_json::from_slice(&body) let args: GroupAddRemove = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
warn!("UpdateGroupMembers args {:?}", args); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "update_group_members",
group = %args.group,
member_count = args.members.len(),
remove = args.is_remove,
state = "decoded",
"admin group state"
);
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
for member in args.members.iter() { for member in args.members.iter() {
match iam_store.is_temp_user(member).await { match iam_store.is_temp_user(member).await {
@@ -433,7 +562,10 @@ impl Operation for UpdateGroupMembers {
Ok(()) Ok(())
}) })
.unwrap_or_else(|| { .unwrap_or_else(|| {
Err(S3Error::with_message(S3ErrorCode::InternalError, "get global cred failed".to_string())) Err(S3Error::with_message(
S3ErrorCode::InternalError,
"failed to load global credentials".to_string(),
))
})?; })?;
} }
Err(e) => { Err(e) => {
@@ -445,29 +577,65 @@ impl Operation for UpdateGroupMembers {
} }
let updated_at = if args.is_remove { let updated_at = if args.is_remove {
warn!("remove group members"); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "remove_group_members",
group = %args.group,
member_count = args.members.len(),
state = "requested",
"admin group state"
);
iam_store iam_store
.remove_users_from_group(&args.group, args.members.clone()) .remove_users_from_group(&args.group, args.members.clone())
.await .await
.map_err(|e| { .map_err(|e| {
warn!("remove group members failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "remove_group_members",
group = %args.group,
result = "update_failed",
error = ?e,
"admin group state"
);
iam_error_to_s3_error(e) iam_error_to_s3_error(e)
})? })?
} else { } else {
warn!("add group members"); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "add_group_members",
group = %args.group,
member_count = args.members.len(),
state = "requested",
"admin group state"
);
if let Err(err) = iam_store.get_group_description(&args.group).await if let Err(err) = iam_store.get_group_description(&args.group).await
&& is_err_no_such_group(&err) && is_err_no_such_group(&err)
&& has_space_be(&args.group) && has_space_be(&args.group)
{ {
return Err(s3_error!(InvalidArgument, "not such group")); return Err(s3_error!(InvalidArgument, "group not found"));
} }
iam_store iam_store
.add_users_to_group(&args.group, args.members.clone()) .add_users_to_group(&args.group, args.members.clone())
.await .await
.map_err(|e| { .map_err(|e| {
warn!("add group members failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "add_group_members",
group = %args.group,
result = "update_failed",
error = ?e,
"admin group state"
);
iam_error_to_s3_error(e) iam_error_to_s3_error(e)
})? })?
}; };
@@ -484,7 +652,15 @@ impl Operation for UpdateGroupMembers {
}) })
.await .await
{ {
warn!("site replication group membership hook failed, err: {err}"); warn!(
event = EVENT_ADMIN_GROUP_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_GROUP,
action = "update_group_members",
result = "site_replication_hook_failed",
error = %err,
"admin group state"
);
} }
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
rustfs/src/admin/handlers/kms_dynamic.rs
+96 -36
View File
@@ -36,6 +36,7 @@ use tracing::{error, info, instrument, warn};
const KMS_CONFIG_PATH: &str = "config/kms_config.json"; const KMS_CONFIG_PATH: &str = "config/kms_config.json";
const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_KMS: &str = "kms"; const LOG_SUBSYSTEM_KMS: &str = "kms";
const EVENT_ADMIN_KMS_DYNAMIC_STATE: &str = "admin_kms_dynamic_state";
fn kms_service_manager_from_context() -> std::sync::Arc<rustfs_kms::KmsServiceManager> { fn kms_service_manager_from_context() -> std::sync::Arc<rustfs_kms::KmsServiceManager> {
resolve_kms_runtime_service_manager().unwrap_or_else(|| { resolve_kms_runtime_service_manager().unwrap_or_else(|| {
@@ -43,7 +44,8 @@ fn kms_service_manager_from_context() -> std::sync::Arc<rustfs_kms::KmsServiceMa
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS, subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_manager_fallback", event = "kms_service_manager_fallback",
"KMS service manager fallback initialized" result = "service_manager_fallback_initialized",
"admin kms dynamic state"
); );
rustfs_kms::init_global_kms_service_manager() rustfs_kms::init_global_kms_service_manager()
}) })
@@ -117,7 +119,8 @@ async fn save_kms_config(config: &KmsConfig) -> Result<(), String> {
subsystem = LOG_SUBSYSTEM_KMS, subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_persisted", event = "kms_config_persisted",
storage_path = KMS_CONFIG_PATH, storage_path = KMS_CONFIG_PATH,
"Persisted KMS configuration" state = "config_persisted",
"admin kms dynamic state"
); );
Ok(()) Ok(())
} }
@@ -131,7 +134,8 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
subsystem = LOG_SUBSYSTEM_KMS, subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_load_skipped", event = "kms_config_load_skipped",
reason = "storage_uninitialized", reason = "storage_uninitialized",
"Skipped KMS configuration load" result = "config_load_skipped",
"admin kms dynamic state"
); );
return None; return None;
}; };
@@ -144,7 +148,8 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
subsystem = LOG_SUBSYSTEM_KMS, subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_loaded", event = "kms_config_loaded",
storage_path = KMS_CONFIG_PATH, storage_path = KMS_CONFIG_PATH,
"Loaded persisted KMS configuration" state = "config_loaded",
"admin kms dynamic state"
); );
Some(config) Some(config)
} }
@@ -154,8 +159,9 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
subsystem = LOG_SUBSYSTEM_KMS, subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_deserialize_failed", event = "kms_config_deserialize_failed",
storage_path = KMS_CONFIG_PATH, storage_path = KMS_CONFIG_PATH,
result = "config_deserialize_failed",
error = %e, error = %e,
"Failed to deserialize KMS configuration" "admin kms dynamic state"
); );
None None
} }
@@ -169,7 +175,7 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
event = "kms_config_loaded", event = "kms_config_loaded",
state = "not_found", state = "not_found",
storage_path = KMS_CONFIG_PATH, storage_path = KMS_CONFIG_PATH,
"Persisted KMS configuration not found" "admin kms dynamic state"
); );
} else { } else {
warn!( warn!(
@@ -177,8 +183,9 @@ pub async fn load_kms_config() -> Option<KmsConfig> {
subsystem = LOG_SUBSYSTEM_KMS, subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_load_failed", event = "kms_config_load_failed",
storage_path = KMS_CONFIG_PATH, storage_path = KMS_CONFIG_PATH,
result = "config_load_failed",
error = %e, error = %e,
"Failed to load KMS configuration" "admin kms dynamic state"
); );
} }
None None
@@ -263,8 +270,9 @@ impl Operation for ConfigureKmsHandler {
subsystem = LOG_SUBSYSTEM_KMS, subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_request_decode_failed", event = "kms_request_decode_failed",
operation = "configure", operation = "configure",
result = "request_decode_failed",
error = %e, error = %e,
"Failed to decode KMS admin request" "admin kms dynamic state"
); );
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}"))))); return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}")))));
} }
@@ -277,7 +285,7 @@ impl Operation for ConfigureKmsHandler {
event = "kms_service_state", event = "kms_service_state",
operation = "configure", operation = "configure",
state = "requested", state = "requested",
"KMS service state changed" "admin kms dynamic state"
); );
let service_manager = kms_service_manager_from_context(); let service_manager = kms_service_manager_from_context();
@@ -303,7 +311,7 @@ impl Operation for ConfigureKmsHandler {
operation = "configure", operation = "configure",
state = "persist_failed", state = "persist_failed",
error = %e, error = %e,
"KMS service state changed" "admin kms dynamic state"
); );
let status = service_manager.get_status().await; let status = service_manager.get_status().await;
(false, error_msg, status) (false, error_msg, status)
@@ -316,7 +324,7 @@ impl Operation for ConfigureKmsHandler {
operation = "configure", operation = "configure",
state = "configured", state = "configured",
status = ?status, status = ?status,
"KMS service state changed" "admin kms dynamic state"
); );
(true, "KMS configured successfully".to_string(), status) (true, "KMS configured successfully".to_string(), status)
} }
@@ -330,7 +338,7 @@ impl Operation for ConfigureKmsHandler {
operation = "configure", operation = "configure",
state = "configure_failed", state = "configure_failed",
error = %e, error = %e,
"KMS service state changed" "admin kms dynamic state"
); );
let status = service_manager.get_status().await; let status = service_manager.get_status().await;
(false, error_msg, status) (false, error_msg, status)
@@ -346,7 +354,15 @@ impl Operation for ConfigureKmsHandler {
let json_response = match serde_json::to_string(&response) { let json_response = match serde_json::to_string(&response) {
Ok(json) => json, Ok(json) => json,
Err(e) => { Err(e) => {
error!("Failed to serialize response: {}", e); error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "configure",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new(( return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR, StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()), Body::from("Serialization error".to_string()),
@@ -398,8 +414,9 @@ impl Operation for StartKmsHandler {
subsystem = LOG_SUBSYSTEM_KMS, subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_request_decode_failed", event = "kms_request_decode_failed",
operation = "start", operation = "start",
result = "request_decode_failed",
error = %e, error = %e,
"Failed to decode KMS admin request" "admin kms dynamic state"
); );
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}"))))); return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}")))));
} }
@@ -413,7 +430,7 @@ impl Operation for StartKmsHandler {
operation = "start", operation = "start",
state = "requested", state = "requested",
force = start_request.force.unwrap_or(false), force = start_request.force.unwrap_or(false),
"KMS service state changed" "admin kms dynamic state"
); );
let service_manager = kms_service_manager_from_context(); let service_manager = kms_service_manager_from_context();
@@ -427,7 +444,7 @@ impl Operation for StartKmsHandler {
event = "kms_service_state", event = "kms_service_state",
operation = "start", operation = "start",
state = "already_running", state = "already_running",
"KMS service state changed" "admin kms dynamic state"
); );
let response = StartKmsResponse { let response = StartKmsResponse {
success: false, success: false,
@@ -437,7 +454,15 @@ impl Operation for StartKmsHandler {
let json_response = match serde_json::to_string(&response) { let json_response = match serde_json::to_string(&response) {
Ok(json) => json, Ok(json) => json,
Err(e) => { Err(e) => {
error!("Failed to serialize response: {}", e); error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "start",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new(( return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR, StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()), Body::from("Serialization error".to_string()),
@@ -462,7 +487,7 @@ impl Operation for StartKmsHandler {
operation = "restart", operation = "restart",
state = "running", state = "running",
status = ?status, status = ?status,
"KMS service state changed" "admin kms dynamic state"
); );
(true, "KMS service restarted successfully".to_string(), status) (true, "KMS service restarted successfully".to_string(), status)
} }
@@ -475,7 +500,7 @@ impl Operation for StartKmsHandler {
operation = "restart", operation = "restart",
state = "start_failed", state = "start_failed",
error = %e, error = %e,
"KMS service state changed" "admin kms dynamic state"
); );
let status = service_manager.get_status().await; let status = service_manager.get_status().await;
(false, error_msg, status) (false, error_msg, status)
@@ -490,7 +515,7 @@ impl Operation for StartKmsHandler {
operation = "restart", operation = "restart",
state = "stop_failed", state = "stop_failed",
error = %e, error = %e,
"KMS service state changed" "admin kms dynamic state"
); );
let status = service_manager.get_status().await; let status = service_manager.get_status().await;
(false, error_msg, status) (false, error_msg, status)
@@ -508,7 +533,7 @@ impl Operation for StartKmsHandler {
operation = "start", operation = "start",
state = "running", state = "running",
status = ?status, status = ?status,
"KMS service state changed" "admin kms dynamic state"
); );
(true, "KMS service started successfully".to_string(), status) (true, "KMS service started successfully".to_string(), status)
} }
@@ -521,7 +546,7 @@ impl Operation for StartKmsHandler {
operation = "start", operation = "start",
state = "start_failed", state = "start_failed",
error = %e, error = %e,
"KMS service state changed" "admin kms dynamic state"
); );
let status = service_manager.get_status().await; let status = service_manager.get_status().await;
(false, error_msg, status) (false, error_msg, status)
@@ -538,7 +563,15 @@ impl Operation for StartKmsHandler {
let json_response = match serde_json::to_string(&response) { let json_response = match serde_json::to_string(&response) {
Ok(json) => json, Ok(json) => json,
Err(e) => { Err(e) => {
error!("Failed to serialize response: {}", e); error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "start",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new(( return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR, StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()), Body::from("Serialization error".to_string()),
@@ -579,7 +612,7 @@ impl Operation for StopKmsHandler {
event = "kms_service_state", event = "kms_service_state",
operation = "stop", operation = "stop",
state = "requested", state = "requested",
"KMS service state changed" "admin kms dynamic state"
); );
let service_manager = kms_service_manager_from_context(); let service_manager = kms_service_manager_from_context();
@@ -594,7 +627,7 @@ impl Operation for StopKmsHandler {
operation = "stop", operation = "stop",
state = "stopped", state = "stopped",
status = ?status, status = ?status,
"KMS service state changed" "admin kms dynamic state"
); );
(true, "KMS service stopped successfully".to_string(), status) (true, "KMS service stopped successfully".to_string(), status)
} }
@@ -607,7 +640,7 @@ impl Operation for StopKmsHandler {
operation = "stop", operation = "stop",
state = "stop_failed", state = "stop_failed",
error = %e, error = %e,
"KMS service state changed" "admin kms dynamic state"
); );
let status = service_manager.get_status().await; let status = service_manager.get_status().await;
(false, error_msg, status) (false, error_msg, status)
@@ -623,7 +656,15 @@ impl Operation for StopKmsHandler {
let json_response = match serde_json::to_string(&response) { let json_response = match serde_json::to_string(&response) {
Ok(json) => json, Ok(json) => json,
Err(e) => { Err(e) => {
error!("Failed to serialize response: {}", e); error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "stop",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new(( return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR, StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()), Body::from("Serialization error".to_string()),
@@ -662,7 +703,8 @@ impl Operation for GetKmsStatusHandler {
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS, subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_status_requested", event = "kms_status_requested",
"KMS status requested" state = "status_requested",
"admin kms dynamic state"
); );
let service_manager = kms_service_manager_from_context(); let service_manager = kms_service_manager_from_context();
@@ -699,13 +741,22 @@ impl Operation for GetKmsStatusHandler {
backend_type = ?response.backend_type, backend_type = ?response.backend_type,
healthy = response.healthy, healthy = response.healthy,
has_config_summary = response.config_summary.is_some(), has_config_summary = response.config_summary.is_some(),
"KMS status resolved" state = "status_resolved",
"admin kms dynamic state"
); );
let json_response = match serde_json::to_string(&response) { let json_response = match serde_json::to_string(&response) {
Ok(json) => json, Ok(json) => json,
Err(e) => { Err(e) => {
error!("Failed to serialize response: {}", e); error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "status",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new(( return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR, StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()), Body::from("Serialization error".to_string()),
@@ -760,8 +811,9 @@ impl Operation for ReconfigureKmsHandler {
subsystem = LOG_SUBSYSTEM_KMS, subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_request_decode_failed", event = "kms_request_decode_failed",
operation = "reconfigure", operation = "reconfigure",
result = "request_decode_failed",
error = %e, error = %e,
"Failed to decode KMS admin request" "admin kms dynamic state"
); );
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}"))))); return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}")))));
} }
@@ -774,7 +826,7 @@ impl Operation for ReconfigureKmsHandler {
event = "kms_service_state", event = "kms_service_state",
operation = "reconfigure", operation = "reconfigure",
state = "requested", state = "requested",
"KMS service state changed" "admin kms dynamic state"
); );
let service_manager = kms_service_manager_from_context(); let service_manager = kms_service_manager_from_context();
@@ -800,7 +852,7 @@ impl Operation for ReconfigureKmsHandler {
operation = "reconfigure", operation = "reconfigure",
state = "persist_failed", state = "persist_failed",
error = %e, error = %e,
"KMS service state changed" "admin kms dynamic state"
); );
let status = service_manager.get_status().await; let status = service_manager.get_status().await;
(false, error_msg, status) (false, error_msg, status)
@@ -813,7 +865,7 @@ impl Operation for ReconfigureKmsHandler {
operation = "reconfigure", operation = "reconfigure",
state = "reconfigured", state = "reconfigured",
status = ?status, status = ?status,
"KMS service state changed" "admin kms dynamic state"
); );
(true, "KMS reconfigured and restarted successfully".to_string(), status) (true, "KMS reconfigured and restarted successfully".to_string(), status)
} }
@@ -827,7 +879,7 @@ impl Operation for ReconfigureKmsHandler {
operation = "reconfigure", operation = "reconfigure",
state = "reconfigure_failed", state = "reconfigure_failed",
error = %e, error = %e,
"KMS service state changed" "admin kms dynamic state"
); );
let status = service_manager.get_status().await; let status = service_manager.get_status().await;
(false, error_msg, status) (false, error_msg, status)
@@ -843,7 +895,15 @@ impl Operation for ReconfigureKmsHandler {
let json_response = match serde_json::to_string(&response) { let json_response = match serde_json::to_string(&response) {
Ok(json) => json, Ok(json) => json,
Err(e) => { Err(e) => {
error!("Failed to serialize response: {}", e); error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "reconfigure",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new(( return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR, StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()), Body::from("Serialization error".to_string()),
rustfs/src/admin/handlers/kms_keys.rs
+157 -37
View File
@@ -33,6 +33,10 @@ use std::collections::HashMap;
use tracing::{error, info}; use tracing::{error, info};
use urlencoding; use urlencoding;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_KMS_KEYS: &str = "kms_keys";
const EVENT_ADMIN_KMS_KEYS_STATE: &str = "admin_kms_keys_state";
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
pub struct CreateKmsKeyRequest { pub struct CreateKmsKeyRequest {
@@ -227,7 +231,7 @@ impl Operation for CreateKeyHandler {
}; };
let Some(service) = kms_encryption_service_from_context().await else { let Some(service) = kms_encryption_service_from_context().await else {
return Err(s3_error!(InternalError, "KMS service not initialized")); return Err(s3_error!(InternalError, "kms service is not initialized"));
}; };
// Extract key name from tags if provided // Extract key name from tags if provided
@@ -259,7 +263,15 @@ impl Operation for CreateKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
} }
Err(e) => { Err(e) => {
error!("Failed to create KMS key: {}", e); error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "create_key",
result = "failed",
error = %e,
"admin kms keys state"
);
Err(s3_error!(InternalError, "failed to create key: {}", e)) Err(s3_error!(InternalError, "failed to create key: {}", e))
} }
} }
@@ -290,11 +302,11 @@ impl Operation for DescribeKeyHandler {
.await?; .await?;
let Some(key_id) = extract_key_id(&req.uri) else { let Some(key_id) = extract_key_id(&req.uri) else {
return Err(s3_error!(InvalidRequest, "missing keyId parameter")); return Err(s3_error!(InvalidRequest, "missing required parameter: 'keyId'"));
}; };
let Some(service) = kms_encryption_service_from_context().await else { let Some(service) = kms_encryption_service_from_context().await else {
return Err(s3_error!(InternalError, "KMS service not initialized")); return Err(s3_error!(InternalError, "kms service is not initialized"));
}; };
let request = DescribeKeyRequest { key_id: key_id.clone() }; let request = DescribeKeyRequest { key_id: key_id.clone() };
@@ -314,7 +326,16 @@ impl Operation for DescribeKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
} }
Err(e) => { Err(e) => {
error!("Failed to describe KMS key {}: {}", key_id, e); error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "describe_key",
key_id = %key_id,
result = "failed",
error = %e,
"admin kms keys state"
);
Err(s3_error!(InternalError, "failed to describe key: {}", e)) Err(s3_error!(InternalError, "failed to describe key: {}", e))
} }
} }
@@ -451,7 +472,7 @@ impl Operation for ListKeysHandler {
let marker = query_params.get("marker").cloned(); let marker = query_params.get("marker").cloned();
let Some(service) = kms_encryption_service_from_context().await else { let Some(service) = kms_encryption_service_from_context().await else {
return Err(s3_error!(InternalError, "KMS service not initialized")); return Err(s3_error!(InternalError, "kms service is not initialized"));
}; };
let request = ListKeysRequest { let request = ListKeysRequest {
@@ -478,7 +499,15 @@ impl Operation for ListKeysHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
} }
Err(e) => { Err(e) => {
error!("Failed to list KMS keys: {}", e); error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "list_keys",
result = "failed",
error = %e,
"admin kms keys state"
);
Err(s3_error!(InternalError, "failed to list keys: {}", e)) Err(s3_error!(InternalError, "failed to list keys: {}", e))
} }
} }
@@ -544,7 +573,15 @@ impl Operation for GenerateDataKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
} }
Err(e) => { Err(e) => {
error!("Failed to generate data key: {}", e); error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "generate_data_key",
result = "failed",
error = %e,
"admin kms keys state"
);
Err(s3_error!(InternalError, "failed to generate data key: {}", e)) Err(s3_error!(InternalError, "failed to generate data key: {}", e))
} }
} }
@@ -593,7 +630,7 @@ impl Operation for CreateKmsKeyHandler {
let Some(service_manager) = kms_service_manager_from_context() else { let Some(service_manager) = kms_service_manager_from_context() else {
let response = CreateKmsKeyResponse { let response = CreateKmsKeyResponse {
success: false, success: false,
message: "KMS service manager not initialized".to_string(), message: "kms service manager is not initialized".to_string(),
key_id: "".to_string(), key_id: "".to_string(),
key_metadata: None, key_metadata: None,
}; };
@@ -607,7 +644,7 @@ impl Operation for CreateKmsKeyHandler {
let Some(manager) = service_manager.get_manager().await else { let Some(manager) = service_manager.get_manager().await else {
let response = CreateKmsKeyResponse { let response = CreateKmsKeyResponse {
success: false, success: false,
message: "KMS service not running".to_string(), message: "kms service is not running".to_string(),
key_id: "".to_string(), key_id: "".to_string(),
key_metadata: None, key_metadata: None,
}; };
@@ -633,10 +670,18 @@ impl Operation for CreateKmsKeyHandler {
match manager.create_key(kms_request).await { match manager.create_key(kms_request).await {
Ok(kms_response) => { Ok(kms_response) => {
info!("Created KMS key: {}", kms_response.key_id); info!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "create_key",
key_id = %kms_response.key_id,
state = "completed",
"admin kms keys state"
);
let response = CreateKmsKeyResponse { let response = CreateKmsKeyResponse {
success: true, success: true,
message: "Key created successfully".to_string(), message: "key created successfully".to_string(),
key_id: kms_response.key_id, key_id: kms_response.key_id,
key_metadata: Some(kms_response.key_metadata), key_metadata: Some(kms_response.key_metadata),
}; };
@@ -650,10 +695,18 @@ impl Operation for CreateKmsKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
} }
Err(e) => { Err(e) => {
error!("Failed to create KMS key: {}", e); error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "create_key",
result = "failed",
error = %e,
"admin kms keys state"
);
let response = CreateKmsKeyResponse { let response = CreateKmsKeyResponse {
success: false, success: false,
message: format!("Failed to create key: {e}"), message: format!("failed to create key: {e}"),
key_id: "".to_string(), key_id: "".to_string(),
key_metadata: None, key_metadata: None,
}; };
@@ -720,7 +773,7 @@ impl Operation for DeleteKmsKeyHandler {
let Some(key_id) = query_params.get("keyId") else { let Some(key_id) = query_params.get("keyId") else {
let response = DeleteKmsKeyResponse { let response = DeleteKmsKeyResponse {
success: false, success: false,
message: "missing keyId parameter".to_string(), message: "missing required parameter: 'keyId'".to_string(),
key_id: "".to_string(), key_id: "".to_string(),
deletion_date: None, deletion_date: None,
}; };
@@ -747,7 +800,7 @@ impl Operation for DeleteKmsKeyHandler {
let Some(service_manager) = kms_service_manager_from_context() else { let Some(service_manager) = kms_service_manager_from_context() else {
let response = DeleteKmsKeyResponse { let response = DeleteKmsKeyResponse {
success: false, success: false,
message: "KMS service manager not initialized".to_string(), message: "kms service manager is not initialized".to_string(),
key_id: request.key_id, key_id: request.key_id,
deletion_date: None, deletion_date: None,
}; };
@@ -761,7 +814,7 @@ impl Operation for DeleteKmsKeyHandler {
let Some(manager) = service_manager.get_manager().await else { let Some(manager) = service_manager.get_manager().await else {
let response = DeleteKmsKeyResponse { let response = DeleteKmsKeyResponse {
success: false, success: false,
message: "KMS service not running".to_string(), message: "kms service is not running".to_string(),
key_id: request.key_id, key_id: request.key_id,
deletion_date: None, deletion_date: None,
}; };
@@ -780,10 +833,18 @@ impl Operation for DeleteKmsKeyHandler {
match manager.delete_key(kms_request).await { match manager.delete_key(kms_request).await {
Ok(kms_response) => { Ok(kms_response) => {
info!("Successfully deleted KMS key: {}", kms_response.key_id); info!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "delete_key",
key_id = %kms_response.key_id,
state = "completed",
"admin kms keys state"
);
let response = DeleteKmsKeyResponse { let response = DeleteKmsKeyResponse {
success: true, success: true,
message: "Key deleted successfully".to_string(), message: "key deleted successfully".to_string(),
key_id: kms_response.key_id, key_id: kms_response.key_id,
deletion_date: kms_response.deletion_date, deletion_date: kms_response.deletion_date,
}; };
@@ -797,7 +858,16 @@ impl Operation for DeleteKmsKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
} }
Err(e) => { Err(e) => {
error!("Failed to delete KMS key {}: {}", request.key_id, e); error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "delete_key",
key_id = %request.key_id,
result = "failed",
error = %e,
"admin kms keys state"
);
let status = match &e { let status = match &e {
KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND, KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND,
KmsError::InvalidOperation { .. } | KmsError::ValidationError { .. } => StatusCode::BAD_REQUEST, KmsError::InvalidOperation { .. } | KmsError::ValidationError { .. } => StatusCode::BAD_REQUEST,
@@ -870,7 +940,7 @@ impl Operation for CancelKmsKeyDeletionHandler {
let Some(key_id) = query_params.get("keyId") else { let Some(key_id) = query_params.get("keyId") else {
let response = CancelKmsKeyDeletionResponse { let response = CancelKmsKeyDeletionResponse {
success: false, success: false,
message: "missing keyId parameter".to_string(), message: "missing required parameter: 'keyId'".to_string(),
key_id: "".to_string(), key_id: "".to_string(),
key_metadata: None, key_metadata: None,
}; };
@@ -888,7 +958,7 @@ impl Operation for CancelKmsKeyDeletionHandler {
let Some(service_manager) = kms_service_manager_from_context() else { let Some(service_manager) = kms_service_manager_from_context() else {
let response = CancelKmsKeyDeletionResponse { let response = CancelKmsKeyDeletionResponse {
success: false, success: false,
message: "KMS service manager not initialized".to_string(), message: "kms service manager is not initialized".to_string(),
key_id: request.key_id, key_id: request.key_id,
key_metadata: None, key_metadata: None,
}; };
@@ -902,7 +972,7 @@ impl Operation for CancelKmsKeyDeletionHandler {
let Some(manager) = service_manager.get_manager().await else { let Some(manager) = service_manager.get_manager().await else {
let response = CancelKmsKeyDeletionResponse { let response = CancelKmsKeyDeletionResponse {
success: false, success: false,
message: "KMS service not running".to_string(), message: "kms service is not running".to_string(),
key_id: request.key_id, key_id: request.key_id,
key_metadata: None, key_metadata: None,
}; };
@@ -919,10 +989,18 @@ impl Operation for CancelKmsKeyDeletionHandler {
match manager.cancel_key_deletion(kms_request).await { match manager.cancel_key_deletion(kms_request).await {
Ok(kms_response) => { Ok(kms_response) => {
info!("Cancelled deletion for KMS key: {}", kms_response.key_id); info!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "cancel_delete_key",
key_id = %kms_response.key_id,
state = "completed",
"admin kms keys state"
);
let response = CancelKmsKeyDeletionResponse { let response = CancelKmsKeyDeletionResponse {
success: true, success: true,
message: "Key deletion cancelled successfully".to_string(), message: "key deletion cancelled successfully".to_string(),
key_id: kms_response.key_id, key_id: kms_response.key_id,
key_metadata: Some(kms_response.key_metadata), key_metadata: Some(kms_response.key_metadata),
}; };
@@ -936,7 +1014,16 @@ impl Operation for CancelKmsKeyDeletionHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
} }
Err(e) => { Err(e) => {
error!("Failed to cancel deletion for KMS key {}: {}", request.key_id, e); error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "cancel_delete_key",
key_id = %request.key_id,
result = "failed",
error = %e,
"admin kms keys state"
);
let response = CancelKmsKeyDeletionResponse { let response = CancelKmsKeyDeletionResponse {
success: false, success: false,
message: format!("Failed to cancel key deletion: {e}"), message: format!("Failed to cancel key deletion: {e}"),
@@ -995,7 +1082,7 @@ impl Operation for ListKmsKeysHandler {
let Some(service_manager) = kms_service_manager_from_context() else { let Some(service_manager) = kms_service_manager_from_context() else {
let response = ListKmsKeysResponse { let response = ListKmsKeysResponse {
success: false, success: false,
message: "KMS service manager not initialized".to_string(), message: "kms service manager is not initialized".to_string(),
keys: vec![], keys: vec![],
truncated: false, truncated: false,
next_marker: None, next_marker: None,
@@ -1010,7 +1097,7 @@ impl Operation for ListKmsKeysHandler {
let Some(manager) = service_manager.get_manager().await else { let Some(manager) = service_manager.get_manager().await else {
let response = ListKmsKeysResponse { let response = ListKmsKeysResponse {
success: false, success: false,
message: "KMS service not running".to_string(), message: "kms service is not running".to_string(),
keys: vec![], keys: vec![],
truncated: false, truncated: false,
next_marker: None, next_marker: None,
@@ -1031,10 +1118,18 @@ impl Operation for ListKmsKeysHandler {
match manager.list_keys(kms_request).await { match manager.list_keys(kms_request).await {
Ok(kms_response) => { Ok(kms_response) => {
info!("Listed {} KMS keys", kms_response.keys.len()); info!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "list_keys",
state = "completed",
key_count = kms_response.keys.len(),
"admin kms keys state"
);
let response = ListKmsKeysResponse { let response = ListKmsKeysResponse {
success: true, success: true,
message: "Keys listed successfully".to_string(), message: "keys listed successfully".to_string(),
keys: kms_response.keys, keys: kms_response.keys,
truncated: kms_response.truncated, truncated: kms_response.truncated,
next_marker: kms_response.next_marker, next_marker: kms_response.next_marker,
@@ -1049,10 +1144,18 @@ impl Operation for ListKmsKeysHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
} }
Err(e) => { Err(e) => {
error!("Failed to list KMS keys: {}", e); error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "list_keys",
result = "failed",
error = %e,
"admin kms keys state"
);
let response = ListKmsKeysResponse { let response = ListKmsKeysResponse {
success: false, success: false,
message: format!("Failed to list keys: {e}"), message: format!("failed to list keys: {e}"),
keys: vec![], keys: vec![],
truncated: false, truncated: false,
next_marker: None, next_marker: None,
@@ -1103,7 +1206,7 @@ impl Operation for DescribeKmsKeyHandler {
let Some(key_id) = params.get("key_id") else { let Some(key_id) = params.get("key_id") else {
let response = DescribeKmsKeyResponse { let response = DescribeKmsKeyResponse {
success: false, success: false,
message: "missing keyId parameter".to_string(), message: "missing required parameter: 'keyId'".to_string(),
key_metadata: None, key_metadata: None,
}; };
let data = let data =
@@ -1116,7 +1219,7 @@ impl Operation for DescribeKmsKeyHandler {
let Some(service_manager) = kms_service_manager_from_context() else { let Some(service_manager) = kms_service_manager_from_context() else {
let response = DescribeKmsKeyResponse { let response = DescribeKmsKeyResponse {
success: false, success: false,
message: "KMS service manager not initialized".to_string(), message: "kms service manager is not initialized".to_string(),
key_metadata: None, key_metadata: None,
}; };
let data = let data =
@@ -1129,7 +1232,7 @@ impl Operation for DescribeKmsKeyHandler {
let Some(manager) = service_manager.get_manager().await else { let Some(manager) = service_manager.get_manager().await else {
let response = DescribeKmsKeyResponse { let response = DescribeKmsKeyResponse {
success: false, success: false,
message: "KMS service not running".to_string(), message: "kms service is not running".to_string(),
key_metadata: None, key_metadata: None,
}; };
let data = let data =
@@ -1145,7 +1248,15 @@ impl Operation for DescribeKmsKeyHandler {
match manager.describe_key(kms_request).await { match manager.describe_key(kms_request).await {
Ok(kms_response) => { Ok(kms_response) => {
info!("Described KMS key: {}", key_id); info!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "describe_key",
key_id = %key_id,
state = "completed",
"admin kms keys state"
);
let response = DescribeKmsKeyResponse { let response = DescribeKmsKeyResponse {
success: true, success: true,
message: "Key described successfully".to_string(), message: "Key described successfully".to_string(),
@@ -1161,7 +1272,16 @@ impl Operation for DescribeKmsKeyHandler {
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers))
} }
Err(e) => { Err(e) => {
error!("Failed to describe KMS key {}: {}", key_id, e); error!(
event = EVENT_ADMIN_KMS_KEYS_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS_KEYS,
action = "describe_key",
key_id = %key_id,
result = "failed",
error = %e,
"admin kms keys state"
);
let status = match &e { let status = match &e {
KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND, KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND,
KmsError::InvalidOperation { .. } => StatusCode::BAD_REQUEST, KmsError::InvalidOperation { .. } => StatusCode::BAD_REQUEST,
rustfs/src/admin/handlers/oidc.rs
+51 -8
View File
@@ -39,6 +39,10 @@ use time::OffsetDateTime;
use tracing::{error, info, warn}; use tracing::{error, info, warn};
use url::Url; use url::Url;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_OIDC: &str = "oidc";
const EVENT_ADMIN_OIDC_STATE: &str = "admin_oidc_state";
const OIDC_PUBLIC_PROVIDERS_SUFFIX: &str = "/v3/oidc/providers"; const OIDC_PUBLIC_PROVIDERS_SUFFIX: &str = "/v3/oidc/providers";
const OIDC_AUTHORIZE_SUFFIX: &str = "/v3/oidc/authorize/"; const OIDC_AUTHORIZE_SUFFIX: &str = "/v3/oidc/authorize/";
const OIDC_CALLBACK_SUFFIX: &str = "/v3/oidc/callback/"; const OIDC_CALLBACK_SUFFIX: &str = "/v3/oidc/callback/";
@@ -448,7 +452,14 @@ impl Operation for OidcAuthorizeHandler {
.await .await
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("authorize failed: {e}")))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("authorize failed: {e}")))?;
info!("OIDC authorize redirect for provider '{}' to IdP", provider_id); info!(
event = EVENT_ADMIN_OIDC_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
provider_id = %provider_id,
state = "authorize_redirect",
"admin oidc state"
);
// Return 302 redirect // Return 302 redirect
let mut resp = S3Response::new((StatusCode::FOUND, Body::empty())); let mut resp = S3Response::new((StatusCode::FOUND, Body::empty()));
@@ -486,7 +497,15 @@ impl Operation for OidcCallbackHandler {
// Check for error response from IdP // Check for error response from IdP
if let Some(error) = extract_query_param(&req.uri, "error") { if let Some(error) = extract_query_param(&req.uri, "error") {
let desc = extract_query_param(&req.uri, "error_description").unwrap_or_default(); let desc = extract_query_param(&req.uri, "error_description").unwrap_or_default();
warn!("OIDC callback received error from IdP: {} - {}", error, desc); warn!(
event = EVENT_ADMIN_OIDC_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
result = "idp_callback_error",
error_code = %error,
error_description = %desc,
"admin oidc state"
);
return Err(S3Error::with_message( return Err(S3Error::with_message(
S3ErrorCode::AccessDenied, S3ErrorCode::AccessDenied,
format!("OIDC authentication failed: {error} - {desc}"), format!("OIDC authentication failed: {error} - {desc}"),
@@ -500,21 +519,38 @@ impl Operation for OidcCallbackHandler {
// Exchange authorization code for tokens and extract claims // Exchange authorization code for tokens and extract claims
let (claims, actual_provider_id, session, id_token) = let (claims, actual_provider_id, session, id_token) =
oidc_sys.exchange_code(&state, &code, &redirect_uri).await.map_err(|e| { oidc_sys.exchange_code(&state, &code, &redirect_uri).await.map_err(|e| {
error!("OIDC code exchange failed: {}", e); error!(
event = EVENT_ADMIN_OIDC_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
result = "code_exchange_failed",
error = %e,
"admin oidc state"
);
S3Error::with_message(S3ErrorCode::AccessDenied, format!("code exchange failed: {e}")) S3Error::with_message(S3ErrorCode::AccessDenied, format!("code exchange failed: {e}"))
})?; })?;
info!( info!(
"OIDC login successful: username='{}', email='{}', sub='{}' (provider: {})", event = EVENT_ADMIN_OIDC_STATE,
claims.username, claims.email, claims.sub, actual_provider_id component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
provider_id = %actual_provider_id,
state = "authentication_succeeded",
"admin oidc state"
); );
// Map claims to policies and groups // Map claims to policies and groups
let (policies, groups) = oidc_sys.map_claims_to_policies(&actual_provider_id, &claims); let (policies, groups) = oidc_sys.map_claims_to_policies(&actual_provider_id, &claims);
info!( info!(
"OIDC claim mapping: user='{}', policies={:?}, groups={:?}", event = EVENT_ADMIN_OIDC_STATE,
claims.username, policies, groups component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
provider_id = %actual_provider_id,
policy_count = policies.len(),
group_count = groups.len(),
state = "claims_mapped",
"admin oidc state"
); );
// Generate STS credentials using the shared helper. // Generate STS credentials using the shared helper.
@@ -568,7 +604,14 @@ impl Operation for OidcLogoutHandler {
Ok(Some(url)) => url, Ok(Some(url)) => url,
Ok(None) => fallback_location.clone(), Ok(None) => fallback_location.clone(),
Err(err) => { Err(err) => {
warn!("OIDC logout fallback triggered: {}", err); warn!(
event = EVENT_ADMIN_OIDC_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OIDC,
result = "logout_fallback_triggered",
error = %err,
"admin oidc state"
);
fallback_location.clone() fallback_location.clone()
} }
}, },
rustfs/src/admin/handlers/policies.rs
+116 -56
View File
@@ -52,6 +52,7 @@ use url::form_urlencoded;
const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_POLICY: &str = "policy"; const LOG_SUBSYSTEM_POLICY: &str = "policy";
const EVENT_ADMIN_POLICY_STATE: &str = "admin_policy_state";
pub fn register_iam_policy_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> { pub fn register_iam_policy_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert( r.insert(
@@ -117,7 +118,7 @@ pub struct ListCannedPolicies {}
impl Operation for ListCannedPolicies { impl Operation for ListCannedPolicies {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -136,23 +137,26 @@ impl Operation for ListCannedPolicies {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: BucketQuery = let input: BucketQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
BucketQuery::default() BucketQuery::default()
} }
}; };
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let policies = iam_store.list_polices(&query.bucket).await.map_err(|e| { let policies = iam_store.list_polices(&query.bucket).await.map_err(|e| {
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_list_failed", event = EVENT_ADMIN_POLICY_STATE,
bucket = %query.bucket, bucket = %query.bucket,
result = "list_failed",
error = ?e, error = ?e,
"Failed to list canned policies" "admin policy state"
); );
S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?; })?;
@@ -162,7 +166,7 @@ impl Operation for ListCannedPolicies {
.filter(|(_, v)| serde_json::to_string(v).is_ok()) .filter(|(_, v)| serde_json::to_string(v).is_ok())
.collect(); .collect();
let body = serde_json::to_vec(&kvs).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; let body = serde_json::to_vec(&kvs).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?;
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -181,7 +185,7 @@ pub struct AddCannedPolicy {}
impl Operation for AddCannedPolicy { impl Operation for AddCannedPolicy {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -200,7 +204,7 @@ impl Operation for AddCannedPolicy {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: PolicyNameQuery = let input: PolicyNameQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
PolicyNameQuery::default() PolicyNameQuery::default()
@@ -208,11 +212,11 @@ impl Operation for AddCannedPolicy {
}; };
if query.name.is_empty() { if query.name.is_empty() {
return Err(s3_error!(InvalidArgument, "policy name is empty")); return Err(s3_error!(InvalidArgument, "policy name is required"));
} }
if has_space_be(&query.name) { if has_space_be(&query.name) {
return Err(s3_error!(InvalidArgument, "policy name has space")); return Err(s3_error!(InvalidArgument, "policy name contains spaces"));
} }
let mut input = req.input; let mut input = req.input;
@@ -222,10 +226,11 @@ impl Operation for AddCannedPolicy {
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_body_read_failed", event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name, policy = %query.name,
result = "body_read_failed",
error = ?e, error = ?e,
"Failed to read policy request body" "admin policy state"
); );
return Err(s3_error!(InvalidRequest, "policy configuration body too large or failed to read")); return Err(s3_error!(InvalidRequest, "policy configuration body too large or failed to read"));
} }
@@ -235,27 +240,31 @@ impl Operation for AddCannedPolicy {
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_parse_failed", event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name, policy = %query.name,
result = "parse_failed",
error = ?e, error = ?e,
"Failed to parse canned policy" "admin policy state"
); );
S3Error::with_message(S3ErrorCode::InvalidRequest, e.to_string()) S3Error::with_message(S3ErrorCode::InvalidRequest, e.to_string())
})?; })?;
if policy.version.is_empty() { if policy.version.is_empty() {
return Err(s3_error!(InvalidArgument, "policy version is empty")); return Err(s3_error!(InvalidArgument, "policy version is required"));
} }
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let updated_at = iam_store.set_policy(&query.name, policy.clone()).await.map_err(|e| { let updated_at = iam_store.set_policy(&query.name, policy.clone()).await.map_err(|e| {
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_set_failed", event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name, policy = %query.name,
result = "persist_failed",
error = ?e, error = ?e,
"Failed to persist canned policy" "admin policy state"
); );
S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?; })?;
@@ -272,7 +281,16 @@ impl Operation for AddCannedPolicy {
}) })
.await .await
{ {
warn!(policy = %query.name, error = ?err, "site replication policy add hook failed"); warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name,
action = "add_policy",
result = "site_replication_hook_failed",
error = ?err,
"admin policy state"
);
} }
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
@@ -287,7 +305,7 @@ pub struct InfoCannedPolicy {}
impl Operation for InfoCannedPolicy { impl Operation for InfoCannedPolicy {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -306,7 +324,7 @@ impl Operation for InfoCannedPolicy {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: PolicyNameQuery = let input: PolicyNameQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
PolicyNameQuery::default() PolicyNameQuery::default()
@@ -314,7 +332,7 @@ impl Operation for InfoCannedPolicy {
}; };
if query.name.is_empty() { if query.name.is_empty() {
return Err(s3_error!(InvalidArgument, "policy name is empty")); return Err(s3_error!(InvalidArgument, "policy name is required"));
} }
let policies = MappedPolicy::new(&query.name).to_slice(); let policies = MappedPolicy::new(&query.name).to_slice();
@@ -322,21 +340,24 @@ impl Operation for InfoCannedPolicy {
return Err(s3_error!(InvalidArgument, "too many policies")); return Err(s3_error!(InvalidArgument, "too many policies"));
} }
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
let pd = iam_store.info_policy(&query.name).await.map_err(|e| { let pd = iam_store.info_policy(&query.name).await.map_err(|e| {
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_info_failed", event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name, policy = %query.name,
result = "info_failed",
error = ?e, error = ?e,
"Failed to load canned policy" "admin policy state"
); );
S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?; })?;
let body = serde_json::to_vec(&pd).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; let body = serde_json::to_vec(&pd).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?;
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -350,7 +371,7 @@ pub struct RemoveCannedPolicy {}
impl Operation for RemoveCannedPolicy { impl Operation for RemoveCannedPolicy {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -369,7 +390,7 @@ impl Operation for RemoveCannedPolicy {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: PolicyNameQuery = let input: PolicyNameQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
PolicyNameQuery::default() PolicyNameQuery::default()
@@ -377,19 +398,22 @@ impl Operation for RemoveCannedPolicy {
}; };
if query.name.is_empty() { if query.name.is_empty() {
return Err(s3_error!(InvalidArgument, "policy name is empty")); return Err(s3_error!(InvalidArgument, "policy name is required"));
} }
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
iam_store.delete_policy(&query.name, true).await.map_err(|e| { iam_store.delete_policy(&query.name, true).await.map_err(|e| {
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_delete_failed", event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name, policy = %query.name,
result = "delete_failed",
error = ?e, error = ?e,
"Failed to delete canned policy" "admin policy state"
); );
S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?; })?;
@@ -403,7 +427,16 @@ impl Operation for RemoveCannedPolicy {
}) })
.await .await
{ {
warn!(policy = %query.name, error = ?err, "site replication policy delete hook failed"); warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = EVENT_ADMIN_POLICY_STATE,
policy = %query.name,
action = "delete_policy",
result = "site_replication_hook_failed",
error = ?err,
"admin policy state"
);
} }
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
@@ -429,7 +462,7 @@ pub struct SetPolicyForUserOrGroup {}
impl Operation for SetPolicyForUserOrGroup { impl Operation for SetPolicyForUserOrGroup {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -448,7 +481,7 @@ impl Operation for SetPolicyForUserOrGroup {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: SetPolicyForUserOrGroupQuery = let input: SetPolicyForUserOrGroupQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
SetPolicyForUserOrGroupQuery::default() SetPolicyForUserOrGroupQuery::default()
@@ -456,10 +489,12 @@ impl Operation for SetPolicyForUserOrGroup {
}; };
if query.user_or_group.is_empty() { if query.user_or_group.is_empty() {
return Err(s3_error!(InvalidArgument, "user or group is empty")); return Err(s3_error!(InvalidArgument, "user or group is required"));
} }
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InternalError, "iam is not initialized"));
};
if !query.is_group { if !query.is_group {
match iam_store.is_temp_user(&query.user_or_group).await { match iam_store.is_temp_user(&query.user_or_group).await {
@@ -473,11 +508,12 @@ impl Operation for SetPolicyForUserOrGroup {
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_target_validation_failed", event = EVENT_ADMIN_POLICY_STATE,
target = %query.user_or_group, target = %query.user_or_group,
check = "is_temp_user", check = "is_temp_user",
result = "target_validation_failed",
error = ?err, error = ?err,
"Failed to validate policy target" "admin policy state"
); );
return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string())); return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string()));
} }
@@ -485,28 +521,29 @@ impl Operation for SetPolicyForUserOrGroup {
}; };
let Some(sys_cred) = get_global_action_cred() else { let Some(sys_cred) = get_global_action_cred() else {
return Err(s3_error!(InternalError, "get global action cred failed")); return Err(s3_error!(InternalError, "failed to load global credentials"));
}; };
if query.user_or_group == sys_cred.access_key { if query.user_or_group == sys_cred.access_key {
return Err(s3_error!(InvalidArgument, "can't set policy for system user")); return Err(s3_error!(InvalidArgument, "cannot set a policy for the system user"));
} }
} }
if !query.is_group { if !query.is_group {
if iam_store.get_user(&query.user_or_group).await.is_none() { if iam_store.get_user(&query.user_or_group).await.is_none() {
return Err(s3_error!(InvalidArgument, "user not exist")); return Err(s3_error!(InvalidArgument, "user not found"));
} }
} else { } else {
iam_store.get_group_description(&query.user_or_group).await.map_err(|e| { iam_store.get_group_description(&query.user_or_group).await.map_err(|e| {
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_target_validation_failed", event = EVENT_ADMIN_POLICY_STATE,
target = %query.user_or_group, target = %query.user_or_group,
check = "group_description", check = "group_description",
result = "target_validation_failed",
error = ?e, error = ?e,
"Failed to validate policy target" "admin policy state"
); );
iam_error_to_s3_error(e) iam_error_to_s3_error(e)
})?; })?;
@@ -519,12 +556,13 @@ impl Operation for SetPolicyForUserOrGroup {
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_mapping_set_failed", event = EVENT_ADMIN_POLICY_STATE,
target = %query.user_or_group, target = %query.user_or_group,
policy = %query.policy_name, policy = %query.policy_name,
is_group = query.is_group, is_group = query.is_group,
result = "mapping_set_failed",
error = ?e, error = ?e,
"Failed to update policy mapping" "admin policy state"
); );
S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?; })?;
@@ -546,7 +584,16 @@ impl Operation for SetPolicyForUserOrGroup {
}) })
.await .await
{ {
warn!(target = %query.user_or_group, error = ?err, "site replication policy mapping hook failed"); warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = EVENT_ADMIN_POLICY_STATE,
target = %query.user_or_group,
action = "set_policy_mapping",
result = "site_replication_hook_failed",
error = ?err,
"admin policy state"
);
} }
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
@@ -923,11 +970,12 @@ async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_target_validation_failed", event = EVENT_ADMIN_POLICY_STATE,
target = %assoc_req.user, target = %assoc_req.user,
check = "is_temp_user", check = "is_temp_user",
result = "target_validation_failed",
error = ?err, error = ?err,
"Failed to validate policy association target" "admin policy state"
); );
return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string())); return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string()));
} }
@@ -950,10 +998,11 @@ async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_user_info_failed", event = EVENT_ADMIN_POLICY_STATE,
target = %assoc_req.user, target = %assoc_req.user,
result = "user_info_failed",
error = ?e, error = ?e,
"Failed to load policy association target" "admin policy state"
); );
iam_error_to_s3_error(e) iam_error_to_s3_error(e)
})?; })?;
@@ -964,10 +1013,11 @@ async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_group_lookup_failed", event = EVENT_ADMIN_POLICY_STATE,
group = %assoc_req.group, group = %assoc_req.group,
result = "group_lookup_failed",
error = ?e, error = ?e,
"Failed to load policy association target" "admin policy state"
); );
iam_error_to_s3_error(e) iam_error_to_s3_error(e)
})?; })?;
@@ -988,12 +1038,13 @@ async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool
warn!( warn!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY, subsystem = LOG_SUBSYSTEM_POLICY,
event = "policy_mapping_set_failed", event = EVENT_ADMIN_POLICY_STATE,
target = %target_name, target = %target_name,
policy_count = updated_policies.len(), policy_count = updated_policies.len(),
is_group, is_group,
result = "mapping_set_failed",
error = ?e, error = ?e,
"Failed to update policy association" "admin policy state"
); );
S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?; })?;
@@ -1015,7 +1066,16 @@ async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool
}) })
.await .await
{ {
warn!(target = %target_name, error = ?err, "site replication policy association hook failed"); warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_POLICY,
event = EVENT_ADMIN_POLICY_STATE,
target = %target_name,
action = if is_attach { "attach_policy" } else { "detach_policy" },
result = "site_replication_hook_failed",
error = ?err,
"admin policy state"
);
} }
let policies_attached = if is_attach { changed_policies.clone() } else { Vec::new() }; let policies_attached = if is_attach { changed_policies.clone() } else { Vec::new() };
rustfs/src/admin/handlers/quota.rs
+92 -30
View File
@@ -33,6 +33,10 @@ use tokio::sync::RwLock;
use tracing::{debug, info, warn}; use tracing::{debug, info, warn};
use url::form_urlencoded; use url::form_urlencoded;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_QUOTA: &str = "quota";
const EVENT_ADMIN_QUOTA_STATE: &str = "admin_quota_state";
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
pub struct SetBucketQuotaRequest { pub struct SetBucketQuotaRequest {
pub quota: Option<u64>, pub quota: Option<u64>,
@@ -231,7 +235,14 @@ pub fn register_quota_route(r: &mut S3Router<AdminOperation>) -> std::io::Result
impl Operation for SetBucketQuotaHandler { impl Operation for SetBucketQuotaHandler {
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle SetBucketQuota"); warn!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "set_bucket_quota",
state = "requested",
"admin quota state"
);
let Some(ref cred) = req.credentials else { let Some(ref cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required")); return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -276,7 +287,7 @@ impl Operation for SetBucketQuotaHandler {
quota_checker quota_checker
.set_quota_config(&bucket, quota.clone()) .set_quota_config(&bucket, quota.clone())
.await .await
.map_err(|e| s3_error!(InternalError, "Failed to set quota: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to set quota: {}", e))?;
// Get real-time usage from data usage system // Get real-time usage from data usage system
let current_usage = current_usage_from_context(&bucket).await; let current_usage = current_usage_from_context(&bucket).await;
@@ -291,7 +302,7 @@ impl Operation for SetBucketQuotaHandler {
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(), quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
}; };
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))? serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
}; };
Ok(S3Response::new((StatusCode::OK, Body::from(json)))) Ok(S3Response::new((StatusCode::OK, Body::from(json))))
@@ -302,7 +313,14 @@ impl Operation for SetBucketQuotaHandler {
impl Operation for GetBucketQuotaHandler { impl Operation for GetBucketQuotaHandler {
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle GetBucketQuota"); warn!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "get_bucket_quota",
state = "requested",
"admin quota state"
);
let Some(ref cred) = req.credentials else { let Some(ref cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required")); return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -327,21 +345,21 @@ impl Operation for GetBucketQuotaHandler {
) )
.await?; .await?;
let metadata_sys_lock = let metadata_sys_lock = bucket_metadata_from_context()
bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?; .ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?;
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
let (quota, current_usage) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e { let (quota, current_usage) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e {
QuotaError::ConfigNotFound { .. } => { QuotaError::ConfigNotFound { .. } => {
s3_error!(NoSuchBucket, "Bucket not found: {}", bucket) s3_error!(NoSuchBucket, "bucket not found: {}", bucket)
} }
_ => s3_error!(InternalError, "Failed to get quota: {}", e), _ => s3_error!(InternalError, "failed to get quota: {}", e),
})?; })?;
let json = if is_compat_get_bucket_quota_path(req.uri.path()) { let json = if is_compat_get_bucket_quota_path(req.uri.path()) {
serde_json::to_string(&compat_bucket_quota_response(&quota)) serde_json::to_string(&compat_bucket_quota_response(&quota))
.map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))? .map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
} else { } else {
let response = BucketQuotaResponse { let response = BucketQuotaResponse {
bucket, bucket,
@@ -350,7 +368,7 @@ impl Operation for GetBucketQuotaHandler {
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(), quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
}; };
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))? serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
}; };
Ok(S3Response::new((StatusCode::OK, Body::from(json)))) Ok(S3Response::new((StatusCode::OK, Body::from(json))))
@@ -361,7 +379,14 @@ impl Operation for GetBucketQuotaHandler {
impl Operation for ClearBucketQuotaHandler { impl Operation for ClearBucketQuotaHandler {
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle ClearBucketQuota"); warn!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "clear_bucket_quota",
state = "requested",
"admin quota state"
);
let Some(ref cred) = req.credentials else { let Some(ref cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required")); return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -385,10 +410,18 @@ impl Operation for ClearBucketQuotaHandler {
return Err(s3_error!(InvalidRequest, "bucket name is required")); return Err(s3_error!(InvalidRequest, "bucket name is required"));
} }
info!("Clearing quota for bucket: {}", bucket); info!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "clear_bucket_quota",
bucket = %bucket,
state = "clearing",
"admin quota state"
);
let metadata_sys_lock = let metadata_sys_lock = bucket_metadata_from_context()
bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?; .ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?;
let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
@@ -397,9 +430,17 @@ impl Operation for ClearBucketQuotaHandler {
quota_checker quota_checker
.set_quota_config(&bucket, quota.clone()) .set_quota_config(&bucket, quota.clone())
.await .await
.map_err(|e| s3_error!(InternalError, "Failed to clear quota: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to clear quota: {}", e))?;
info!("Successfully cleared quota for bucket: {}", bucket); info!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "clear_bucket_quota",
bucket = %bucket,
state = "cleared",
"admin quota state"
);
// Get real-time usage from data usage system // Get real-time usage from data usage system
let current_usage = current_usage_from_context(&bucket).await; let current_usage = current_usage_from_context(&bucket).await;
@@ -412,7 +453,7 @@ impl Operation for ClearBucketQuotaHandler {
}; };
let json = let json =
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?; serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
Ok(S3Response::new((StatusCode::OK, Body::from(json)))) Ok(S3Response::new((StatusCode::OK, Body::from(json))))
} }
@@ -422,7 +463,14 @@ impl Operation for ClearBucketQuotaHandler {
impl Operation for GetBucketQuotaStatsHandler { impl Operation for GetBucketQuotaStatsHandler {
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle GetBucketQuotaStats"); warn!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "get_bucket_quota_stats",
state = "requested",
"admin quota state"
);
let Some(ref cred) = req.credentials else { let Some(ref cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required")); return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -447,16 +495,16 @@ impl Operation for GetBucketQuotaStatsHandler {
) )
.await?; .await?;
let metadata_sys_lock = let metadata_sys_lock = bucket_metadata_from_context()
bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?; .ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?;
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
let (quota, current_usage_opt) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e { let (quota, current_usage_opt) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e {
QuotaError::ConfigNotFound { .. } => { QuotaError::ConfigNotFound { .. } => {
s3_error!(NoSuchBucket, "Bucket not found: {}", bucket) s3_error!(NoSuchBucket, "bucket not found: {}", bucket)
} }
_ => s3_error!(InternalError, "Failed to get quota stats: {}", e), _ => s3_error!(InternalError, "failed to get quota stats: {}", e),
})?; })?;
let current_usage = current_usage_opt.unwrap_or(0); let current_usage = current_usage_opt.unwrap_or(0);
@@ -479,7 +527,7 @@ impl Operation for GetBucketQuotaStatsHandler {
}; };
let json = let json =
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?; serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
Ok(S3Response::new((StatusCode::OK, Body::from(json)))) Ok(S3Response::new((StatusCode::OK, Body::from(json))))
} }
@@ -489,7 +537,14 @@ impl Operation for GetBucketQuotaStatsHandler {
impl Operation for CheckBucketQuotaHandler { impl Operation for CheckBucketQuotaHandler {
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle CheckBucketQuota"); warn!(
event = EVENT_ADMIN_QUOTA_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "check_bucket_quota",
state = "requested",
"admin quota state"
);
let Some(ref cred) = req.credentials else { let Some(ref cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required")); return Err(s3_error!(InvalidRequest, "authentication required"));
@@ -527,12 +582,19 @@ impl Operation for CheckBucketQuotaHandler {
}; };
debug!( debug!(
"Checking quota for bucket: {}, operation: {}, size: {}", event = EVENT_ADMIN_QUOTA_STATE,
bucket, request.operation_type, request.operation_size component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_QUOTA,
action = "check_bucket_quota",
bucket = %bucket,
operation = %request.operation_type,
operation_size = request.operation_size,
state = "checking",
"admin quota state"
); );
let metadata_sys_lock = let metadata_sys_lock = bucket_metadata_from_context()
bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?; .ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?;
let quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
@@ -547,7 +609,7 @@ impl Operation for CheckBucketQuotaHandler {
let result = quota_checker let result = quota_checker
.check_quota_with_usage_reporting(&bucket, operation, request.operation_size, true) .check_quota_with_usage_reporting(&bucket, operation, request.operation_size, true)
.await .await
.map_err(|e| s3_error!(InternalError, "Failed to check quota: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to check quota: {}", e))?;
let response = CheckQuotaResponse { let response = CheckQuotaResponse {
bucket, bucket,
@@ -560,7 +622,7 @@ impl Operation for CheckBucketQuotaHandler {
}; };
let json = let json =
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?; serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
Ok(S3Response::new((StatusCode::OK, Body::from(json)))) Ok(S3Response::new((StatusCode::OK, Body::from(json))))
} }
rustfs/src/admin/handlers/rebalance.rs
+117 -36
View File
@@ -41,7 +41,11 @@ use s3s::{
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::time::Duration; use std::time::Duration;
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::warn; use tracing::info;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_REBALANCE: &str = "rebalance";
const EVENT_ADMIN_REBALANCE_STATE: &str = "admin_rebalance_state";
pub fn register_rebalance_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> { pub fn register_rebalance_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert( r.insert(
@@ -219,10 +223,17 @@ pub struct RebalanceStart {}
impl Operation for RebalanceStart { impl Operation for RebalanceStart {
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle RebalanceStart"); info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "start",
state = "requested",
"admin rebalance state"
);
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "Failed to start rebalance: missing credentials")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -239,7 +250,7 @@ impl Operation for RebalanceStart {
.await?; .await?;
let Some(store) = new_object_layer_fn() else { let Some(store) = new_object_layer_fn() else {
return Err(s3_error!(InternalError, "Failed to start rebalance: object layer not initialized")); return Err(s3_error!(InternalError, "object layer is not initialized"));
}; };
if store.pools.len() == 1 { if store.pools.len() == 1 {
@@ -247,47 +258,74 @@ impl Operation for RebalanceStart {
} }
if store.is_decommission_running().await { if store.is_decommission_running().await {
return Err(s3_error!( return Err(s3_error!(InvalidRequest, "cannot start rebalance while decommission is in progress"));
InvalidRequest,
"Rebalance cannot be started, decommission is already in progress"
));
} }
if store.is_rebalance_conflicting_with_decommission().await { if store.is_rebalance_conflicting_with_decommission().await {
return Err(s3_error!(OperationAborted, "Rebalance already in progress")); return Err(s3_error!(OperationAborted, "rebalance is already in progress"));
} }
let bucket_infos = store let bucket_infos = store
.list_bucket(&BucketOptions::default()) .list_bucket(&BucketOptions::default())
.await .await
.map_err(|e| s3_error!(InternalError, "Failed to list buckets for rebalance: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to list buckets for rebalance: {}", e))?;
let buckets: Vec<String> = bucket_infos.into_iter().map(|bucket| bucket.name).collect(); let buckets: Vec<String> = bucket_infos.into_iter().map(|bucket| bucket.name).collect();
let id = match store.init_rebalance_meta(buckets).await { let id = match store.init_rebalance_meta(buckets).await {
Ok(id) => id, Ok(id) => id,
Err(e) => { Err(e) => {
return Err(s3_error!(InternalError, "Failed to initialize rebalance metadata: {}", e)); return Err(s3_error!(InternalError, "failed to initialize rebalance metadata: {}", e));
} }
}; };
store store
.start_rebalance() .start_rebalance()
.await .await
.map_err(|e| s3_error!(InternalError, "Failed to start rebalance: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to start rebalance: {}", e))?;
warn!("Rebalance started with id: {}", id); info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "start",
state = "started",
rebalance_id = %id,
"admin rebalance state"
);
if let Some(notification_sys) = get_global_notification_sys() { if let Some(notification_sys) = get_global_notification_sys() {
warn!("RebalanceStart Loading rebalance meta start"); info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "start",
state = "propagation_started",
"admin rebalance state"
);
if let Err(err) = notification_sys.load_rebalance_meta(true).await { if let Err(err) = notification_sys.load_rebalance_meta(true).await {
warn!("rebalance start propagation failed after local state update: {err}"); info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "start",
result = "propagation_failed",
error = %err,
"admin rebalance state"
);
} }
warn!("RebalanceStart Loading rebalance meta done"); info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "start",
state = "propagation_completed",
"admin rebalance state"
);
} }
let resp = RebalanceResp { id }; let resp = RebalanceResp { id };
let data = serde_json::to_string(&resp) let data = serde_json::to_string(&resp)
.map_err(|e| s3_error!(InternalError, "Failed to serialize rebalance start response: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to serialize rebalance start response: {}", e))?;
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
@@ -303,10 +341,17 @@ pub struct RebalanceStatus {}
impl Operation for RebalanceStatus { impl Operation for RebalanceStatus {
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle RebalanceStatus"); info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "status",
state = "requested",
"admin rebalance state"
);
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "Failed to load rebalance status: missing credentials")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -323,26 +368,26 @@ impl Operation for RebalanceStatus {
.await?; .await?;
let Some(store) = new_object_layer_fn() else { let Some(store) = new_object_layer_fn() else {
return Err(s3_error!(InternalError, "Failed to load rebalance status: object layer not initialized")); return Err(s3_error!(InternalError, "object layer is not initialized"));
}; };
if store.pools.is_empty() { if store.pools.is_empty() {
return Err(s3_error!(InternalError, "Failed to load rebalance status: no storage pools available")); return Err(s3_error!(InternalError, "no storage pools are available"));
} }
let first_pool = store let first_pool = store
.pools .pools
.first() .first()
.cloned() .cloned()
.ok_or_else(|| s3_error!(InternalError, "Failed to load rebalance status: no storage pools available"))?; .ok_or_else(|| s3_error!(InternalError, "no storage pools are available"))?;
let mut meta = RebalanceMeta::new(); let mut meta = RebalanceMeta::new();
if let Err(err) = meta.load(first_pool).await { if let Err(err) = meta.load(first_pool).await {
if err == StorageError::ConfigNotFound { if err == StorageError::ConfigNotFound {
return Err(s3_error!(NoSuchResource, "Pool rebalance is not started")); return Err(s3_error!(NoSuchResource, "pool rebalance is not started"));
} }
return Err(s3_error!(InternalError, "Failed to load rebalance metadata from pool 0: {}", err)); return Err(s3_error!(InternalError, "failed to load rebalance metadata from pool 0: {}", err));
} }
// Compute disk usage percentage // Compute disk usage percentage
@@ -366,7 +411,7 @@ impl Operation for RebalanceStatus {
}; };
let data = serde_json::to_string(&admin_status) let data = serde_json::to_string(&admin_status)
.map_err(|e| s3_error!(InternalError, "Failed to serialize rebalance status response: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to serialize rebalance status response: {}", e))?;
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
@@ -381,10 +426,17 @@ pub struct RebalanceStop {}
impl Operation for RebalanceStop { impl Operation for RebalanceStop {
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle RebalanceStop"); info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "stop",
state = "requested",
"admin rebalance state"
);
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "Failed to stop rebalance: missing credentials")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -401,37 +453,66 @@ impl Operation for RebalanceStop {
.await?; .await?;
let Some(store) = new_object_layer_fn() else { let Some(store) = new_object_layer_fn() else {
return Err(s3_error!(InternalError, "Failed to stop rebalance: object layer not initialized")); return Err(s3_error!(InternalError, "object layer is not initialized"));
}; };
if !store.is_rebalance_conflicting_with_decommission().await { if !store.is_rebalance_conflicting_with_decommission().await {
return Err(s3_error!(NoSuchResource, "Pool rebalance is not started")); return Err(s3_error!(NoSuchResource, "pool rebalance is not started"));
} }
if let Some(notification_sys) = get_global_notification_sys() { if let Some(notification_sys) = get_global_notification_sys() {
notification_sys notification_sys
.stop_rebalance() .stop_rebalance()
.await .await
.map_err(|e| s3_error!(InternalError, "Failed to stop rebalance via notification system: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to stop rebalance via notification system: {}", e))?;
} else { } else {
store store
.stop_rebalance() .stop_rebalance()
.await .await
.map_err(|e| s3_error!(InternalError, "Failed to stop rebalance: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to stop rebalance: {}", e))?;
store store
.save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt) .save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt)
.await .await
.map_err(|e| s3_error!(InternalError, "Failed to persist rebalance stop metadata: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to persist rebalance stop metadata: {}", e))?;
} }
warn!("handle RebalanceStop save_rebalance_stats done "); info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "stop",
state = "local_stop_persisted",
"admin rebalance state"
);
if let Some(notification_sys) = get_global_notification_sys() { if let Some(notification_sys) = get_global_notification_sys() {
warn!("handle RebalanceStop notification_sys load_rebalance_meta"); info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "stop",
state = "propagation_started",
"admin rebalance state"
);
if let Err(err) = notification_sys.load_rebalance_meta(false).await { if let Err(err) = notification_sys.load_rebalance_meta(false).await {
warn!("rebalance stop propagation failed after local state update: {err}"); info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "stop",
result = "propagation_failed",
error = %err,
"admin rebalance state"
);
} }
warn!("handle RebalanceStop notification_sys load_rebalance_meta done"); info!(
event = EVENT_ADMIN_REBALANCE_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_REBALANCE,
action = "stop",
state = "propagation_completed",
"admin rebalance state"
);
} }
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
rustfs/src/admin/handlers/service_account.rs
+68 -14
View File
@@ -50,6 +50,7 @@ use url::form_urlencoded;
const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_SERVICE_ACCOUNT: &str = "service_account"; const LOG_SUBSYSTEM_SERVICE_ACCOUNT: &str = "service_account";
const EVENT_ADMIN_SERVICE_ACCOUNT_STATE: &str = "admin_service_account_state";
fn sr_session_policy_from_value(value: Option<&serde_json::Value>) -> S3Result<SRSessionPolicy> { fn sr_session_policy_from_value(value: Option<&serde_json::Value>) -> S3Result<SRSessionPolicy> {
let Some(value) = value else { let Some(value) = value else {
@@ -110,10 +111,11 @@ fn map_service_account_lookup_error(err: rustfs_iam::error::Error, action: &str)
debug!( debug!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = "service_account_lookup_failed", event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
action, action,
result = "lookup_failed",
error = ?err, error = ?err,
"Service account lookup failed" "admin service account state"
); );
if is_err_no_such_service_account(&err) { if is_err_no_such_service_account(&err) {
iam_error_to_s3_error(err) iam_error_to_s3_error(err)
@@ -126,10 +128,11 @@ fn map_temp_account_lookup_error(err: rustfs_iam::error::Error, action: &str) ->
debug!( debug!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = "temporary_account_lookup_failed", event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
action, action,
result = "temporary_lookup_failed",
error = ?err, error = ?err,
"Temporary account lookup failed" "admin service account state"
); );
if is_err_no_such_temp_account(&err) { if is_err_no_such_temp_account(&err) {
iam_error_to_s3_error(err) iam_error_to_s3_error(err)
@@ -144,9 +147,10 @@ fn parse_service_account_policy(policy: &serde_json::Value) -> S3Result<Policy>
debug!( debug!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = "service_account_policy_parse_failed", event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
result = "policy_parse_failed",
error = ?e, error = ?e,
"Failed to parse service account policy" "admin service account state"
); );
match e { match e {
rustfs_policy::error::Error::PolicyError(rustfs_policy::policy::Error::NonResource) => { rustfs_policy::error::Error::PolicyError(rustfs_policy::policy::Error::NonResource) => {
@@ -372,10 +376,11 @@ impl Operation for AddServiceAccount {
debug!( debug!(
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = "service_account_create_failed", event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
target_user = %target_user, target_user = %target_user,
result = "create_failed",
error = ?e, error = ?e,
"Failed to create service account" "admin service account state"
); );
s3_error!(InternalError, "create service account failed, e: {:?}", e) s3_error!(InternalError, "create service account failed, e: {:?}", e)
})?; })?;
@@ -410,7 +415,16 @@ impl Operation for AddServiceAccount {
}) })
.await .await
{ {
warn!(access_key = %new_cred.access_key, error = ?err, "site replication add service account hook failed"); warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
access_key = %new_cred.access_key,
action = "create",
result = "site_replication_hook_failed",
error = ?err,
"admin service account state"
);
} }
let resp = AddServiceAccountResp { let resp = AddServiceAccountResp {
@@ -871,7 +885,14 @@ impl Operation for ListServiceAccount {
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key) check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key)
.await .await
.map_err(|e| { .map_err(|e| {
debug!("check key failed: {e:?}"); debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
result = "check_key_failed",
error = ?e,
"admin service account state"
);
s3_error!(InternalError, "check key failed") s3_error!(InternalError, "check key failed")
})?; })?;
@@ -927,7 +948,15 @@ impl Operation for ListServiceAccount {
}; };
let service_accounts = iam_store.list_service_accounts(&target_account).await.map_err(|e| { let service_accounts = iam_store.list_service_accounts(&target_account).await.map_err(|e| {
debug!("list service account failed: {e:?}"); debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
target_user = %target_account,
result = "list_service_accounts_failed",
error = ?e,
"admin service account state"
);
s3_error!(InternalError, "list service account failed") s3_error!(InternalError, "list service account failed")
})?; })?;
@@ -1114,7 +1143,15 @@ impl Operation for ListAccessKeysBulk {
if list_sts_keys { if list_sts_keys {
let sts_keys = iam_store.list_sts_accounts(&user).await.map_err(|e| { let sts_keys = iam_store.list_sts_accounts(&user).await.map_err(|e| {
debug!("list sts account failed: {e:?}"); debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
target_user = %user,
result = "list_sts_accounts_failed",
error = ?e,
"admin service account state"
);
s3_error!(InternalError, "list sts account failed") s3_error!(InternalError, "list sts account failed")
})?; })?;
@@ -1135,7 +1172,15 @@ impl Operation for ListAccessKeysBulk {
if list_service_accounts { if list_service_accounts {
let service_accounts = iam_store.list_service_accounts(&user).await.map_err(|e| { let service_accounts = iam_store.list_service_accounts(&user).await.map_err(|e| {
debug!("list service account failed: {e:?}"); debug!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
target_user = %user,
result = "list_service_accounts_failed",
error = ?e,
"admin service account state"
);
s3_error!(InternalError, "list service account failed") s3_error!(InternalError, "list service account failed")
})?; })?;
@@ -1279,7 +1324,16 @@ impl Operation for DeleteServiceAccount {
}) })
.await .await
{ {
warn!(access_key = %query.access_key, error = ?err, "site replication delete service account hook failed"); warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
access_key = %query.access_key,
action = "delete",
result = "site_replication_hook_failed",
error = ?err,
"admin service account state"
);
} }
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
rustfs/src/admin/handlers/site_replication.rs
+95 -11
View File
@@ -93,6 +93,10 @@ use tracing::warn;
use url::{Url, form_urlencoded}; use url::{Url, form_urlencoded};
use uuid::Uuid; use uuid::Uuid;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_SITE_REPLICATION: &str = "site_replication";
const EVENT_ADMIN_SITE_REPLICATION_STATE: &str = "admin_site_replication_state";
const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json"; const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json";
const SITE_REPL_ADD_SUCCESS: &str = "Requested sites were configured for replication successfully."; const SITE_REPL_ADD_SUCCESS: &str = "Requested sites were configured for replication successfully.";
const SITE_REPL_EDIT_SUCCESS: &str = "Requested site was updated successfully."; const SITE_REPL_EDIT_SUCCESS: &str = "Requested site was updated successfully.";
@@ -2000,7 +2004,15 @@ async fn build_status_info(state: &SiteReplicationState, local_peer: &PeerInfo,
reachable_peers.insert(deployment_id.clone()); reachable_peers.insert(deployment_id.clone());
} }
Err(err) => { Err(err) => {
warn!(peer = %peer.endpoint, error = ?err, "site replication peer metainfo fetch failed"); warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
peer = %peer.endpoint,
result = "peer_metainfo_fetch_failed",
error = ?err,
"admin site replication state"
);
site_infos.insert(deployment_id.clone(), SRInfo::default()); site_infos.insert(deployment_id.clone(), SRInfo::default());
} }
} }
@@ -2536,7 +2548,14 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local
let buckets = match store.list_bucket(&BucketOptions::default()).await { let buckets = match store.list_bucket(&BucketOptions::default()).await {
Ok(b) => b, Ok(b) => b,
Err(err) => { Err(err) => {
warn!(error = ?err, "site replication backfill: failed to list buckets"); warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
result = "backfill_list_buckets_failed",
error = ?err,
"admin site replication state"
);
return; return;
} }
}; };
@@ -2546,14 +2565,38 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local
let name = &bucket.name; let name = &bucket.name;
if let Err(err) = ensure_site_replication_bucket_versioning(name).await { if let Err(err) = ensure_site_replication_bucket_versioning(name).await {
warn!(bucket = %name, error = ?err, "site replication backfill: versioning setup failed"); warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_versioning_setup_failed",
error = ?err,
"admin site replication state"
);
continue; continue;
} }
if let Err(err) = ensure_site_replication_bucket_targets(name, state, local_peer, None).await { if let Err(err) = ensure_site_replication_bucket_targets(name, state, local_peer, None).await {
warn!(bucket = %name, error = ?err, "site replication backfill: targets setup failed"); warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_targets_setup_failed",
error = ?err,
"admin site replication state"
);
} }
if let Err(err) = ensure_site_replication_bucket_replication_config(name, state, local_peer).await { if let Err(err) = ensure_site_replication_bucket_replication_config(name, state, local_peer).await {
warn!(bucket = %name, error = ?err, "site replication backfill: replication config setup failed"); warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_replication_config_setup_failed",
error = ?err,
"admin site replication state"
);
} }
// Broadcast the bucket to peers so they create it too (idempotent on the peer side). // Broadcast the bucket to peers so they create it too (idempotent on the peer side).
// Read the real lock_enabled flag so peers recreate the bucket with the same object-lock // Read the real lock_enabled flag so peers recreate the bucket with the same object-lock
@@ -2561,12 +2604,29 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local
let lock_enabled = match metadata_sys::get(name).await { let lock_enabled = match metadata_sys::get(name).await {
Ok(bm) => bm.lock_enabled, Ok(bm) => bm.lock_enabled,
Err(err) => { Err(err) => {
warn!(bucket = %name, error = ?err, "site replication backfill: failed to read bucket metadata, assuming lock_enabled=false"); warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_metadata_read_failed",
fallback = "lock_enabled=false",
error = ?err,
"admin site replication state"
);
false false
} }
}; };
if let Err(err) = site_replication_make_bucket_hook(name, lock_enabled).await { if let Err(err) = site_replication_make_bucket_hook(name, lock_enabled).await {
warn!(bucket = %name, error = ?err, "site replication backfill: make-bucket broadcast failed"); warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_make_bucket_broadcast_failed",
error = ?err,
"admin site replication state"
);
} }
// Kick a resync toward every remote peer so existing objects travel across. // Kick a resync toward every remote peer so existing objects travel across.
for peer in state.peers.values() { for peer in state.peers.values() {
@@ -2575,8 +2635,16 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local
} }
let result = start_site_bucket_resync(name, peer, &resync_id).await; let result = start_site_bucket_resync(name, peer, &resync_id).await;
if result.status == "failed" { if result.status == "failed" {
warn!(bucket = %name, peer = %peer.endpoint, detail = %result.err_detail, warn!(
"site replication backfill: resync kick failed"); event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
peer = %peer.endpoint,
result = "backfill_resync_kick_failed",
detail = %result.err_detail,
"admin site replication state"
);
} }
} }
} }
@@ -3173,7 +3241,15 @@ impl Operation for SiteReplicationRemoveHandler {
.await .await
{ {
let err_detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint)); let err_detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint));
warn!(peer = %peer.endpoint, error = %err_detail, "site replication peer remove notification failed"); warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
peer = %peer.endpoint,
result = "peer_remove_notification_failed",
error = %err_detail,
"admin site replication state"
);
peer_errors.push(err_detail); peer_errors.push(err_detail);
} }
} }
@@ -3767,7 +3843,15 @@ impl Operation for SRRotateServiceAccountHandler {
send_peer_admin_request(&peer.endpoint, SITE_REPLICATION_PEER_JOIN_PATH, &svc_ak, &new_sk, &join_req).await send_peer_admin_request(&peer.endpoint, SITE_REPLICATION_PEER_JOIN_PATH, &svc_ak, &new_sk, &join_req).await
{ {
let detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint)); let detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint));
warn!(peer = %peer.endpoint, error = %detail, "site replication service account rotation failed for peer"); warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
peer = %peer.endpoint,
result = "service_account_rotation_failed",
error = %detail,
"admin site replication state"
);
peer_errors.push(detail); peer_errors.push(detail);
} }
} }
rustfs/src/admin/handlers/target_descriptor.rs
+35 -35
View File
@@ -472,7 +472,7 @@ where
if !allowed_keys.contains(key) { if !allowed_keys.contains(key) {
return Err(s3_error!( return Err(s3_error!(
InvalidArgument, InvalidArgument,
"key '{}' not allowed for {} type '{}'", "key '{}' is not allowed for {} type '{}'",
key, key,
target_label, target_label,
target_type target_type
@@ -492,7 +492,7 @@ where
pub(crate) async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> { pub(crate) async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> {
if !queue_dir.is_empty() { if !queue_dir.is_empty() {
if !Path::new(queue_dir).is_absolute() { if !Path::new(queue_dir).is_absolute() {
return Err(s3_error!(InvalidArgument, "queue_dir must be absolute path")); return Err(s3_error!(InvalidArgument, "queue_dir must be an absolute path"));
} }
retry_with_backoff( retry_with_backoff(
|| async { tokio::fs::metadata(queue_dir).await.map(|_| ()) }, || async { tokio::fs::metadata(queue_dir).await.map(|_| ()) },
@@ -502,7 +502,7 @@ pub(crate) async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> {
.await .await
.map_err(|e| match e.kind() { .map_err(|e| match e.kind() {
ErrorKind::NotFound => s3_error!(InvalidArgument, "queue_dir does not exist"), ErrorKind::NotFound => s3_error!(InvalidArgument, "queue_dir does not exist"),
ErrorKind::PermissionDenied => s3_error!(InvalidArgument, "queue_dir exists but permission denied"), ErrorKind::PermissionDenied => s3_error!(InvalidArgument, "queue_dir exists but permission is denied"),
_ => s3_error!(InvalidArgument, "failed to access queue_dir: {}", e), _ => s3_error!(InvalidArgument, "failed to access queue_dir: {}", e),
})?; })?;
} }
@@ -533,7 +533,7 @@ where
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?; .ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?;
timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, default_queue_dir)) timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, default_queue_dir))
.await .await
.map_err(|_| s3_error!(InvalidArgument, "target validation timed out"))??; .map_err(|_| s3_error!(InvalidArgument, "target validation timed out after 10s"))??;
let mut kvs = KVS::new(); let mut kvs = KVS::new();
for (key, value) in kv_map { for (key, value) in kv_map {
@@ -652,13 +652,13 @@ async fn validate_webhook_request(kv_map: &HashMap<String, String>) -> S3Result<
.get("endpoint") .get("endpoint")
.map(String::as_str) .map(String::as_str)
.ok_or_else(|| s3_error!(InvalidArgument, "endpoint is required"))?; .ok_or_else(|| s3_error!(InvalidArgument, "endpoint is required"))?;
let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?; let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint URL: {}", e))?;
match parsed_endpoint.scheme() { match parsed_endpoint.scheme() {
"http" | "https" => {} "http" | "https" => {}
other => { other => {
return Err(s3_error!( return Err(s3_error!(
InvalidArgument, InvalidArgument,
"unsupported endpoint scheme: {} (only http and https are allowed)", "unsupported endpoint scheme '{}'; only http and https are allowed",
other other
)); ));
} }
@@ -683,7 +683,7 @@ async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()>
let endpoint = kv_map let endpoint = kv_map
.get(MQTT_BROKER) .get(MQTT_BROKER)
.map(String::as_str) .map(String::as_str)
.ok_or_else(|| s3_error!(InvalidArgument, "broker endpoint is required"))?; .ok_or_else(|| s3_error!(InvalidArgument, "broker URL is required"))?;
let topic = kv_map let topic = kv_map
.get(MQTT_TOPIC) .get(MQTT_TOPIC)
.map(String::as_str) .map(String::as_str)
@@ -701,12 +701,12 @@ async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()>
.map_err(|e| s3_error!(InvalidArgument, "invalid MQTT TLS settings: {}", e))?; .map_err(|e| s3_error!(InvalidArgument, "invalid MQTT TLS settings: {}", e))?;
let parsed_broker = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid broker URL: {}", e))?; let parsed_broker = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid broker URL: {}", e))?;
rustfs_targets::target::mqtt::validate_mqtt_broker_url(&parsed_broker, &tls) rustfs_targets::target::mqtt::validate_mqtt_broker_url(&parsed_broker, &tls)
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?; .map_err(|e| s3_error!(InvalidArgument, "invalid broker URL: {}", e))?;
check_mqtt_broker_available_with_tls(parsed_broker.as_str(), topic, username, password, &tls) check_mqtt_broker_available_with_tls(parsed_broker.as_str(), topic, username, password, &tls)
.await .await
.map_err(|e| match e { .map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid MQTT configuration: {}", e),
_ => s3_error!(InvalidArgument, "MQTT broker check failed: {}", e), _ => s3_error!(InvalidArgument, "mqtt connectivity check failed: {}", e),
})?; })?;
if let Some(queue_dir) = kv_map.get("queue_dir") { if let Some(queue_dir) = kv_map.get("queue_dir") {
@@ -755,10 +755,10 @@ async fn validate_nats_request_impl(
validate_queue_dir(queue_dir.as_str()).await?; validate_queue_dir(queue_dir.as_str()).await?;
} }
let args = build_nats_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) let args = build_nats_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?; .map_err(|e| s3_error!(InvalidArgument, "invalid NATS configuration: {}", e))?;
check_nats_server_available(&args).await.map_err(|e| match e { check_nats_server_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid NATS configuration: {}", e),
_ => s3_error!(InvalidArgument, "NATS server check failed: {}", e), _ => s3_error!(InvalidArgument, "nats connectivity check failed: {}", e),
}) })
} }
@@ -780,17 +780,17 @@ async fn validate_kafka_request_impl(
} }
if !kv_map.contains_key(KAFKA_BROKERS) { if !kv_map.contains_key(KAFKA_BROKERS) {
return Err(s3_error!(InvalidArgument, "Kafka brokers are required")); return Err(s3_error!(InvalidArgument, "kafka brokers are required"));
} }
if !kv_map.contains_key(KAFKA_TOPIC) { if !kv_map.contains_key(KAFKA_TOPIC) {
return Err(s3_error!(InvalidArgument, "Kafka topic is required")); return Err(s3_error!(InvalidArgument, "kafka topic is required"));
} }
let args = build_kafka_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) let args = build_kafka_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?; .map_err(|e| s3_error!(InvalidArgument, "invalid Kafka configuration: {}", e))?;
check_kafka_broker_available(&args).await.map_err(|e| match e { check_kafka_broker_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid Kafka configuration: {}", e),
_ => s3_error!(InvalidArgument, "Kafka broker check failed: {}", e), _ => s3_error!(InvalidArgument, "kafka connectivity check failed: {}", e),
}) })
} }
@@ -811,10 +811,10 @@ async fn validate_amqp_request_impl(
validate_queue_dir(queue_dir.as_str()).await?; validate_queue_dir(queue_dir.as_str()).await?;
} }
let args = build_amqp_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) let args = build_amqp_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?; .map_err(|e| s3_error!(InvalidArgument, "invalid AMQP configuration: {}", e))?;
check_amqp_broker_available(&args).await.map_err(|e| match e { check_amqp_broker_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid AMQP configuration: {}", e),
_ => s3_error!(InvalidArgument, "AMQP broker check failed: {}", e), _ => s3_error!(InvalidArgument, "amqp connectivity check failed: {}", e),
}) })
} }
@@ -835,10 +835,10 @@ async fn validate_pulsar_request_impl(
validate_queue_dir(queue_dir.as_str()).await?; validate_queue_dir(queue_dir.as_str()).await?;
} }
let args = build_pulsar_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) let args = build_pulsar_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?; .map_err(|e| s3_error!(InvalidArgument, "invalid Pulsar configuration: {}", e))?;
check_pulsar_broker_available(&args).await.map_err(|e| match e { check_pulsar_broker_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid Pulsar configuration: {}", e),
_ => s3_error!(InvalidArgument, "Pulsar broker check failed: {}", e), _ => s3_error!(InvalidArgument, "pulsar connectivity check failed: {}", e),
}) })
} }
@@ -851,11 +851,11 @@ async fn validate_mysql_request(
validate_queue_dir(queue_dir.as_str()).await?; validate_queue_dir(queue_dir.as_str()).await?;
} }
let args = let args = build_mysql_args(&to_kvs(kv_map), default_queue_dir, target_type)
build_mysql_args(&to_kvs(kv_map), default_queue_dir, target_type).map_err(|e| s3_error!(InvalidArgument, "{}", e))?; .map_err(|e| s3_error!(InvalidArgument, "invalid MySQL configuration: {}", e))?;
check_mysql_server_available(&args).await.map_err(|e| match e { check_mysql_server_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid MySQL configuration: {}", e),
_ => s3_error!(InvalidArgument, "MySQL server check failed: {}", e), _ => s3_error!(InvalidArgument, "mysql connectivity check failed: {}", e),
}) })
} }
@@ -876,10 +876,10 @@ async fn validate_postgres_request_impl(
validate_queue_dir(queue_dir.as_str()).await?; validate_queue_dir(queue_dir.as_str()).await?;
} }
let args = build_postgres_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) let args = build_postgres_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?; .map_err(|e| s3_error!(InvalidArgument, "invalid PostgreSQL configuration: {}", e))?;
check_postgres_server_available(&args).await.map_err(|e| match e { check_postgres_server_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid PostgreSQL configuration: {}", e),
_ => s3_error!(InvalidArgument, "PostgreSQL server check failed: {}", e), _ => s3_error!(InvalidArgument, "postgres connectivity check failed: {}", e),
}) })
} }
@@ -894,13 +894,13 @@ async fn validate_redis_request(
} }
validate_redis_config(&to_kvs(kv_map), default_queue_dir, default_channel) validate_redis_config(&to_kvs(kv_map), default_queue_dir, default_channel)
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?; .map_err(|e| s3_error!(InvalidArgument, "invalid Redis configuration: {}", e))?;
let args = build_redis_args(&to_kvs(kv_map), default_queue_dir, default_channel, domain.runtime_target_type()) let args = build_redis_args(&to_kvs(kv_map), default_queue_dir, default_channel, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?; .map_err(|e| s3_error!(InvalidArgument, "invalid Redis configuration: {}", e))?;
check_redis_server_available(&args).await.map_err(|e| match e { check_redis_server_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid Redis configuration: {}", e),
_ => s3_error!(InvalidArgument, "Redis server check failed: {}", e), _ => s3_error!(InvalidArgument, "redis connectivity check failed: {}", e),
}) })
} }
rustfs/src/admin/handlers/tier.rs
+215 -47
View File
@@ -57,6 +57,10 @@ use time::OffsetDateTime;
use tokio::spawn; use tokio::spawn;
use tracing::{debug, warn}; use tracing::{debug, warn};
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_TIER: &str = "tier";
const EVENT_ADMIN_TIER_STATE: &str = "admin_tier_state";
#[derive(Debug, Clone, serde::Deserialize, Default)] #[derive(Debug, Clone, serde::Deserialize, Default)]
pub struct AddTierQuery { pub struct AddTierQuery {
#[serde(rename = "accessKey")] #[serde(rename = "accessKey")]
@@ -91,9 +95,14 @@ fn spawn_transition_tier_config_propagation(action: &'static str) {
for peer_result in notification_sys.load_transition_tier_config().await { for peer_result in notification_sys.load_transition_tier_config().await {
if let Some(err) = peer_result.err { if let Some(err) = peer_result.err {
warn!( warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = action,
host = if peer_result.host.is_empty() { "<unknown>" } else { peer_result.host.as_str() }, host = if peer_result.host.is_empty() { "<unknown>" } else { peer_result.host.as_str() },
result = "propagation_failed",
error = %err, error = %err,
"tier {action} propagation failed after local save" "admin tier state"
); );
} }
} }
@@ -174,7 +183,7 @@ impl Operation for AddTier {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: AddTierQuery = let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
AddTierQuery::default() AddTierQuery::default()
@@ -182,7 +191,7 @@ impl Operation for AddTier {
}; };
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -202,13 +211,21 @@ impl Operation for AddTier {
let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await { let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await {
Ok(b) => b, Ok(b) => b,
Err(e) => { Err(e) => {
warn!("get body failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
result = "body_read_failed",
error = ?e,
"admin tier state"
);
return Err(s3_error!(InvalidRequest, "tier configuration body too large or failed to read")); return Err(s3_error!(InvalidRequest, "tier configuration body too large or failed to read"));
} }
}; };
let mut args: TierConfig = serde_json::from_slice(&body) let mut args: TierConfig = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
match args.tier_type { match args.tier_type {
TierType::S3 => { TierType::S3 => {
@@ -240,33 +257,67 @@ impl Operation for AddTier {
} }
_ => (), _ => (),
} }
debug!("add tier args {:?}", args); debug!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
tier_name = %args.name,
tier_type = ?args.tier_type,
state = "decoded",
"admin tier state"
);
let tier_name_for_log = args.name.clone();
let mut force: bool = false; let mut force: bool = false;
let force_str = query.force.clone().unwrap_or_default(); let force_str = query.force.clone().unwrap_or_default();
if !force_str.is_empty() { if !force_str.is_empty() {
force = force_str.parse().map_err(|e| { force = force_str.parse().map_err(|e| {
warn!("parse force failed, e: {:?}", e); warn!(
s3_error!(InvalidRequest, "parse force failed") event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
result = "force_parse_failed",
error = ?e,
"admin tier state"
);
s3_error!(InvalidRequest, "invalid force flag")
})?; })?;
} }
match args.name.as_str() { match args.name.as_str() {
storageclass::STANDARD | storageclass::RRS => { storageclass::STANDARD | storageclass::RRS => {
warn!("tier reserved name, args.name: {}", args.name); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
tier_name = %args.name,
result = "reserved_name_rejected",
"admin tier state"
);
return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name")); return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name"));
} }
&_ => (), &_ => (),
} }
let Some(store) = resolve_object_store_handle() else { let Some(store) = resolve_object_store_handle() else {
return Err(s3_error!(InvalidRequest, "object store not init")); return Err(s3_error!(InternalError, "object store is not initialized"));
}; };
{ {
let tier_config_mgr_handle = resolve_tier_config_handle(); let tier_config_mgr_handle = resolve_tier_config_handle();
let mut tier_config_mgr = tier_config_mgr_handle.write().await; let mut tier_config_mgr = tier_config_mgr_handle.write().await;
if let Err(err) = tier_config_mgr.reload(store).await { if let Err(err) = tier_config_mgr.reload(store).await {
warn!("tier_config_mgr reload failed, e: {:?}", err); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
result = "reload_failed",
error = ?err,
"admin tier state"
);
return Err(S3Error::with_message( return Err(S3Error::with_message(
S3ErrorCode::Custom("TierAddFailed".into()), S3ErrorCode::Custom("TierAddFailed".into()),
format!("tier reload failed. {err}"), format!("tier reload failed. {err}"),
@@ -276,27 +327,36 @@ impl Operation for AddTier {
return if err.code == ERR_TIER_ALREADY_EXISTS.code { return if err.code == ERR_TIER_ALREADY_EXISTS.code {
Err(S3Error::with_message( Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameAlreadyExist".into()), S3ErrorCode::Custom("TierNameAlreadyExist".into()),
"tier name already exists!", "tier name already exists",
)) ))
} else if err.code == ERR_TIER_NAME_NOT_UPPERCASE.code { } else if err.code == ERR_TIER_NAME_NOT_UPPERCASE.code {
Err(S3Error::with_message( Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameNotUppercase".into()), S3ErrorCode::Custom("TierNameNotUppercase".into()),
"tier name not uppercase!", "tier name must be uppercase",
)) ))
} else if err.code == ERR_TIER_BACKEND_IN_USE.code { } else if err.code == ERR_TIER_BACKEND_IN_USE.code {
Err(S3Error::with_message( Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameBackendInUse!".into()), S3ErrorCode::Custom("TierNameBackendInUse!".into()),
"tier name backend in use!", "tier backend is already in use",
)) ))
} else if err.code == ERR_TIER_CONNECT_ERR.code { } else if err.code == ERR_TIER_CONNECT_ERR.code {
Err(S3Error::with_message( Err(S3Error::with_message(
S3ErrorCode::Custom("TierConnectError".into()), S3ErrorCode::Custom("TierConnectError".into()),
"tier connect error!", "tier connectivity check failed",
)) ))
} else if err.code == ERR_TIER_INVALID_CREDENTIALS.code { } else if err.code == ERR_TIER_INVALID_CREDENTIALS.code {
Err(S3Error::with_message(S3ErrorCode::Custom(err.code.clone().into()), err.message)) Err(S3Error::with_message(S3ErrorCode::Custom(err.code.clone().into()), err.message))
} else { } else {
warn!("tier_config_mgr add failed, e: {:?}", err); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
tier_name = %tier_name_for_log,
result = "add_failed",
error = ?err,
"admin tier state"
);
Err(S3Error::with_message( Err(S3Error::with_message(
S3ErrorCode::Custom("TierAddFailed".into()), S3ErrorCode::Custom("TierAddFailed".into()),
format!("tier add failed. {err}"), format!("tier add failed. {err}"),
@@ -304,7 +364,15 @@ impl Operation for AddTier {
}; };
} }
if let Err(e) = tier_config_mgr.save().await { if let Err(e) = tier_config_mgr.save().await {
warn!("tier_config_mgr save failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "add_tier",
result = "save_failed",
error = ?e,
"admin tier state"
);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier save failed")); return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier save failed"));
} }
} }
@@ -324,7 +392,7 @@ impl Operation for EditTier {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: AddTierQuery = let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
AddTierQuery::default() AddTierQuery::default()
@@ -332,7 +400,7 @@ impl Operation for EditTier {
}; };
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -352,27 +420,51 @@ impl Operation for EditTier {
let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await { let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await {
Ok(b) => b, Ok(b) => b,
Err(e) => { Err(e) => {
warn!("get body failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "edit_tier",
result = "body_read_failed",
error = ?e,
"admin tier state"
);
return Err(s3_error!(InvalidRequest, "tier configuration body too large or failed to read")); return Err(s3_error!(InvalidRequest, "tier configuration body too large or failed to read"));
} }
}; };
let creds: TierCreds = serde_json::from_slice(&body) let creds: TierCreds = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
debug!("edit tier args {:?}", creds);
let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default(); let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default();
debug!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "edit_tier",
tier_name = %tier_name,
state = "decoded",
"admin tier state"
);
let Some(store) = resolve_object_store_handle() else { let Some(store) = resolve_object_store_handle() else {
return Err(s3_error!(InvalidRequest, "object store not init")); return Err(s3_error!(InternalError, "object store is not initialized"));
}; };
{ {
let tier_config_mgr_handle = resolve_tier_config_handle(); let tier_config_mgr_handle = resolve_tier_config_handle();
let mut tier_config_mgr = tier_config_mgr_handle.write().await; let mut tier_config_mgr = tier_config_mgr_handle.write().await;
if let Err(err) = tier_config_mgr.reload(store).await { if let Err(err) = tier_config_mgr.reload(store).await {
warn!("tier_config_mgr reload failed, e: {:?}", err); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "edit_tier",
result = "reload_failed",
error = ?err,
"admin tier state"
);
return Err(S3Error::with_message( return Err(S3Error::with_message(
S3ErrorCode::Custom("TierEditFailed".into()), S3ErrorCode::Custom("TierEditFailed".into()),
format!("tier reload failed. {err}"), format!("tier reload failed. {err}"),
@@ -380,14 +472,23 @@ impl Operation for EditTier {
} }
if let Err(err) = tier_config_mgr.edit(&tier_name, creds).await { if let Err(err) = tier_config_mgr.edit(&tier_name, creds).await {
return if err.code == ERR_TIER_NOT_FOUND.code { return if err.code == ERR_TIER_NOT_FOUND.code {
Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!")) Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found"))
} else if err.code == ERR_TIER_MISSING_CREDENTIALS.code { } else if err.code == ERR_TIER_MISSING_CREDENTIALS.code {
Err(S3Error::with_message( Err(S3Error::with_message(
S3ErrorCode::Custom("TierMissingCredentials".into()), S3ErrorCode::Custom("TierMissingCredentials".into()),
"tier missing credentials!", "tier credentials are required",
)) ))
} else { } else {
warn!("tier_config_mgr edit failed, e: {:?}", err); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "edit_tier",
tier_name = %tier_name,
result = "edit_failed",
error = ?err,
"admin tier state"
);
Err(S3Error::with_message( Err(S3Error::with_message(
S3ErrorCode::Custom("TierEditFailed".into()), S3ErrorCode::Custom("TierEditFailed".into()),
format!("tier edit failed. {err}"), format!("tier edit failed. {err}"),
@@ -395,7 +496,15 @@ impl Operation for EditTier {
}; };
} }
if let Err(e) = tier_config_mgr.save().await { if let Err(e) = tier_config_mgr.save().await {
warn!("tier_config_mgr save failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "edit_tier",
result = "save_failed",
error = ?e,
"admin tier state"
);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed")); return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed"));
} }
} }
@@ -466,7 +575,7 @@ impl Operation for RemoveTier {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: AddTierQuery = let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
AddTierQuery::default() AddTierQuery::default()
@@ -474,7 +583,7 @@ impl Operation for RemoveTier {
}; };
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -494,22 +603,38 @@ impl Operation for RemoveTier {
let force_str = query.force.clone().unwrap_or_default(); let force_str = query.force.clone().unwrap_or_default();
if !force_str.is_empty() { if !force_str.is_empty() {
force = force_str.parse().map_err(|e| { force = force_str.parse().map_err(|e| {
warn!("parse force failed, e: {:?}", e); warn!(
s3_error!(InvalidRequest, "parse force failed") event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "remove_tier",
result = "force_parse_failed",
error = ?e,
"admin tier state"
);
s3_error!(InvalidRequest, "invalid force flag")
})?; })?;
} }
let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default(); let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default();
let Some(store) = resolve_object_store_handle() else { let Some(store) = resolve_object_store_handle() else {
return Err(s3_error!(InvalidRequest, "object store not init")); return Err(s3_error!(InternalError, "object store is not initialized"));
}; };
{ {
let tier_config_mgr_handle = resolve_tier_config_handle(); let tier_config_mgr_handle = resolve_tier_config_handle();
let mut tier_config_mgr = tier_config_mgr_handle.write().await; let mut tier_config_mgr = tier_config_mgr_handle.write().await;
if let Err(err) = tier_config_mgr.reload(store).await { if let Err(err) = tier_config_mgr.reload(store).await {
warn!("tier_config_mgr reload failed, e: {:?}", err); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "remove_tier",
result = "reload_failed",
error = ?err,
"admin tier state"
);
return Err(S3Error::with_message( return Err(S3Error::with_message(
S3ErrorCode::Custom("TierRemoveFailed".into()), S3ErrorCode::Custom("TierRemoveFailed".into()),
format!("tier reload failed. {err}"), format!("tier reload failed. {err}"),
@@ -517,11 +642,23 @@ impl Operation for RemoveTier {
} }
if let Err(err) = tier_config_mgr.remove(&tier_name, force).await { if let Err(err) = tier_config_mgr.remove(&tier_name, force).await {
return if err.code == ERR_TIER_NOT_FOUND.code { return if err.code == ERR_TIER_NOT_FOUND.code {
Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found.")) Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found"))
} else if err.code == ERR_TIER_BACKEND_NOT_EMPTY.code { } else if err.code == ERR_TIER_BACKEND_NOT_EMPTY.code {
Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse".into()), "tier is used.")) Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameBackendInUse".into()),
"tier backend is not empty",
))
} else { } else {
warn!("tier_config_mgr remove failed, e: {:?}", err); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "remove_tier",
tier_name = %tier_name,
result = "remove_failed",
error = ?err,
"admin tier state"
);
Err(S3Error::with_message( Err(S3Error::with_message(
S3ErrorCode::Custom("TierRemoveFailed".into()), S3ErrorCode::Custom("TierRemoveFailed".into()),
format!("tier remove failed. {err}"), format!("tier remove failed. {err}"),
@@ -530,7 +667,15 @@ impl Operation for RemoveTier {
} }
if let Err(e) = tier_config_mgr.save().await { if let Err(e) = tier_config_mgr.save().await {
warn!("tier_config_mgr save failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "remove_tier",
result = "save_failed",
error = ?e,
"admin tier state"
);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier save failed")); return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier save failed"));
} }
} }
@@ -549,7 +694,7 @@ pub struct VerifyTier {}
impl Operation for VerifyTier { impl Operation for VerifyTier {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -601,7 +746,7 @@ impl Operation for GetTierInfo {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: AddTierQuery = let input: AddTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
AddTierQuery::default() AddTierQuery::default()
@@ -683,7 +828,7 @@ impl Operation for ClearTier {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: ClearTierQuery = let input: ClearTierQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
ClearTierQuery::default() ClearTierQuery::default()
@@ -691,7 +836,7 @@ impl Operation for ClearTier {
}; };
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -718,23 +863,46 @@ impl Operation for ClearTier {
rand.insert_str(3, &t.day().to_string()); rand.insert_str(3, &t.day().to_string());
rand.insert_str(17, &t.month().to_string()); rand.insert_str(17, &t.month().to_string());
rand.insert_str(23, &t.year().to_string()); rand.insert_str(23, &t.year().to_string());
warn!("tier_config_mgr rand: {}", rand); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "clear_tier",
state = "challenge_generated",
"admin tier state"
);
if query.rand != Some(rand) { if query.rand != Some(rand) {
return Err(s3_error!(InvalidRequest, "get rand failed")); return Err(s3_error!(InvalidRequest, "invalid clear-tier confirmation token"));
}; };
let tier_config_mgr_handle = resolve_tier_config_handle(); let tier_config_mgr_handle = resolve_tier_config_handle();
let mut tier_config_mgr = tier_config_mgr_handle.write().await; let mut tier_config_mgr = tier_config_mgr_handle.write().await;
//tier_config_mgr.reload(api); //tier_config_mgr.reload(api);
if let Err(err) = tier_config_mgr.clear_tier(force).await { if let Err(err) = tier_config_mgr.clear_tier(force).await {
warn!("tier_config_mgr clear failed, e: {:?}", err); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "clear_tier",
result = "clear_failed",
error = ?err,
"admin tier state"
);
return Err(S3Error::with_message( return Err(S3Error::with_message(
S3ErrorCode::Custom("TierClearFailed".into()), S3ErrorCode::Custom("TierClearFailed".into()),
format!("tier clear failed. {err}"), format!("tier clear failed. {err}"),
)); ));
} }
if let Err(e) = tier_config_mgr.save().await { if let Err(e) = tier_config_mgr.save().await {
warn!("tier_config_mgr save failed, e: {:?}", e); warn!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_TIER,
action = "clear_tier",
result = "save_failed",
error = ?e,
"admin tier state"
);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed")); return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed"));
} }
rustfs/src/admin/handlers/user.rs
+82 -48
View File
@@ -51,6 +51,10 @@ use std::{collections::HashMap, io::Cursor, str::from_utf8};
use tracing::{debug, warn}; use tracing::{debug, warn};
use zip::{ZipArchive, ZipWriter, result::ZipError, write::SimpleFileOptions}; use zip::{ZipArchive, ZipWriter, result::ZipError, write::SimpleFileOptions};
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_USER: &str = "user";
const EVENT_ADMIN_USER_STATE: &str = "admin_user_state";
#[derive(Debug, Deserialize, Default)] #[derive(Debug, Deserialize, Default)]
pub struct AddUserQuery { pub struct AddUserQuery {
#[serde(rename = "accessKey", alias = "access-key")] #[serde(rename = "accessKey", alias = "access-key")]
@@ -184,7 +188,7 @@ impl Operation for AddUser {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: AddUserQuery = let input: AddUserQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
AddUserQuery::default() AddUserQuery::default()
@@ -192,7 +196,7 @@ impl Operation for AddUser {
}; };
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -210,39 +214,41 @@ impl Operation for AddUser {
// .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, format!("decrypt_data err {}", e)))?; // .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, format!("decrypt_data err {}", e)))?;
let args: AddOrUpdateUserReq = serde_json::from_slice(&body) let args: AddOrUpdateUserReq = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
if args.secret_key.is_empty() { if args.secret_key.is_empty() {
return Err(s3_error!(InvalidArgument, "access key is empty")); return Err(s3_error!(InvalidArgument, "secret key is required"));
} }
if let Some(sys_cred) = get_global_action_cred() if let Some(sys_cred) = get_global_action_cred()
&& constant_time_eq(&sys_cred.access_key, ak) && constant_time_eq(&sys_cred.access_key, ak)
{ {
return Err(s3_error!(InvalidArgument, "can't create user with system access key")); return Err(s3_error!(InvalidArgument, "cannot create a user with the system access key"));
} }
let Ok(iam_store) = rustfs_iam::get() else { let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InvalidRequest, "iam not init")); return Err(s3_error!(InternalError, "iam is not initialized"));
}; };
if let Some(user) = iam_store.get_user(ak).await { if let Some(user) = iam_store.get_user(ak).await {
if (user.credentials.is_temp() || user.credentials.is_service_account()) && cred.parent_user == ak { if (user.credentials.is_temp() || user.credentials.is_service_account()) && cred.parent_user == ak {
return Err(s3_error!(InvalidArgument, "can't create user with service account access key")); return Err(s3_error!(InvalidArgument, "cannot create a user with a service account access key"));
} }
} else if has_space_be(ak) { } else if has_space_be(ak) {
return Err(s3_error!(InvalidArgument, "access key has space")); return Err(s3_error!(InvalidArgument, "access key contains spaces"));
} }
if from_utf8(ak.as_bytes()).is_err() { if from_utf8(ak.as_bytes()).is_err() {
return Err(s3_error!(InvalidArgument, "access key is not utf8")); return Err(s3_error!(InvalidArgument, "access key is not valid UTF-8"));
} }
let check_deny_only = should_check_deny_only(ak, &cred); let check_deny_only = should_check_deny_only(ak, &cred);
debug!( debug!(
target = "rustfs::admin::handlers::user", component = LOG_COMPONENT_ADMIN,
operation = "AddUser", subsystem = LOG_SUBSYSTEM_USER,
event = EVENT_ADMIN_USER_STATE,
action = "add_user",
query_access_key = %ak, query_access_key = %ak,
signer_access_key = %cred.access_key, signer_access_key = %cred.access_key,
is_temp = cred.is_temp(), is_temp = cred.is_temp(),
@@ -252,7 +258,8 @@ impl Operation for AddUser {
jwt_parent_claim_present = cred.claims.as_ref().and_then(|c| c.get("parent")).is_some(), jwt_parent_claim_present = cred.claims.as_ref().and_then(|c| c.get("parent")).is_some(),
check_deny_only, check_deny_only,
is_owner = owner, is_owner = owner,
"authorization context before validate_admin_request (no secrets)" state = "authorization_context",
"admin user state"
); );
// For eligible self operations, only explicit Deny should block the request. // For eligible self operations, only explicit Deny should block the request.
@@ -269,7 +276,7 @@ impl Operation for AddUser {
let updated_at = iam_store let updated_at = iam_store
.create_user(ak, &args) .create_user(ak, &args)
.await .await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("create_user err {e}")))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to create user: {e}")))?;
if let Err(err) = site_replication_iam_change_hook(SRIAMItem { if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
r#type: "iam-user".to_string(), r#type: "iam-user".to_string(),
@@ -285,7 +292,16 @@ impl Operation for AddUser {
}) })
.await .await
{ {
warn!(access_key = %ak, error = ?err, "site replication create user hook failed"); warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_USER,
event = EVENT_ADMIN_USER_STATE,
access_key = %ak,
action = "create_user",
result = "site_replication_hook_failed",
error = ?err,
"admin user state"
);
} }
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
@@ -302,7 +318,7 @@ impl Operation for SetUserStatus {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: AddUserQuery = let input: AddUserQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
AddUserQuery::default() AddUserQuery::default()
@@ -316,11 +332,11 @@ impl Operation for SetUserStatus {
} }
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
if constant_time_eq(&input_cred.access_key, ak) { if constant_time_eq(&input_cred.access_key, ak) {
return Err(s3_error!(InvalidArgument, "can't change status of self")); return Err(s3_error!(InvalidArgument, "cannot change the status of the current user"));
} }
let (cred, owner) = let (cred, owner) =
@@ -340,13 +356,13 @@ impl Operation for SetUserStatus {
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, e))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, e))?;
let Ok(iam_store) = rustfs_iam::get() else { let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InvalidRequest, "iam not init")); return Err(s3_error!(InternalError, "iam is not initialized"));
}; };
iam_store iam_store
.set_user_status(ak, status) .set_user_status(ak, status)
.await .await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("set_user_status err {e}")))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to set user status: {e}")))?;
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -365,7 +381,7 @@ pub struct ListUsers {}
impl Operation for ListUsers { impl Operation for ListUsers {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -384,7 +400,7 @@ impl Operation for ListUsers {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: BucketQuery = let input: BucketQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
BucketQuery::default() BucketQuery::default()
@@ -392,7 +408,7 @@ impl Operation for ListUsers {
}; };
let Ok(iam_store) = rustfs_iam::get() else { let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InvalidRequest, "iam not init")); return Err(s3_error!(InternalError, "iam is not initialized"));
}; };
let users = { let users = {
@@ -410,7 +426,7 @@ impl Operation for ListUsers {
}; };
let data = serde_json::to_vec(&users) let data = serde_json::to_vec(&users)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal users err {e}")))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize response: {e}")))?;
let (data, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, data)?; let (data, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, data)?;
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
@@ -425,7 +441,7 @@ pub struct RemoveUser {}
impl Operation for RemoveUser { impl Operation for RemoveUser {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -444,7 +460,7 @@ impl Operation for RemoveUser {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: AddUserQuery = let input: AddUserQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
AddUserQuery::default() AddUserQuery::default()
@@ -458,37 +474,35 @@ impl Operation for RemoveUser {
} }
let sys_cred = get_global_action_cred() let sys_cred = get_global_action_cred()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "get_global_action_cred failed"))?; .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "failed to load global credentials"))?;
if ak == sys_cred.access_key || ak == cred.access_key || cred.parent_user == ak { if ak == sys_cred.access_key || ak == cred.access_key || cred.parent_user == ak {
return Err(s3_error!(InvalidArgument, "can't remove self")); return Err(s3_error!(InvalidArgument, "cannot remove the current user"));
} }
let Ok(iam_store) = rustfs_iam::get() else { let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InvalidRequest, "iam not init")); return Err(s3_error!(InternalError, "iam is not initialized"));
}; };
let (is_temp, _) = iam_store let (is_temp, _) = iam_store.is_temp_user(ak).await.map_err(|e| {
.is_temp_user(ak) S3Error::with_message(S3ErrorCode::InternalError, format!("failed to query temporary user state: {e}"))
.await })?;
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("is_temp_user err {e}")))?;
if is_temp { if is_temp {
return Err(s3_error!(InvalidArgument, "can't remove temp user")); return Err(s3_error!(InvalidArgument, "cannot remove a temporary user"));
} }
let (is_service_account, _) = iam_store let (is_service_account, _) = iam_store.is_service_account(ak).await.map_err(|e| {
.is_service_account(ak) S3Error::with_message(S3ErrorCode::InternalError, format!("failed to query service account state: {e}"))
.await })?;
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("is_service_account err {e}")))?;
if is_service_account { if is_service_account {
return Err(s3_error!(InvalidArgument, "can't remove service account")); return Err(s3_error!(InvalidArgument, "cannot remove a service account"));
} }
iam_store iam_store
.delete_user(ak, true) .delete_user(ak, true)
.await .await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("delete_user err {e}")))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to delete user: {e}")))?;
if let Err(err) = site_replication_iam_change_hook(SRIAMItem { if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
r#type: "iam-user".to_string(), r#type: "iam-user".to_string(),
@@ -504,7 +518,16 @@ impl Operation for RemoveUser {
}) })
.await .await
{ {
warn!(access_key = %ak, error = ?err, "site replication delete user hook failed"); warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_USER,
event = EVENT_ADMIN_USER_STATE,
access_key = %ak,
action = "delete_user",
result = "site_replication_hook_failed",
error = ?err,
"admin user state"
);
} }
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
@@ -521,7 +544,7 @@ impl Operation for GetUserInfo {
let query = { let query = {
if let Some(query) = req.uri.query() { if let Some(query) = req.uri.query() {
let input: AddUserQuery = let input: AddUserQuery =
from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?;
input input
} else { } else {
AddUserQuery::default() AddUserQuery::default()
@@ -535,11 +558,11 @@ impl Operation for GetUserInfo {
} }
let Ok(iam_store) = rustfs_iam::get() else { let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InvalidRequest, "iam not init")); return Err(s3_error!(InternalError, "iam is not initialized"));
}; };
let Some(input_cred) = req.credentials else { let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed")); return Err(s3_error!(InvalidRequest, "authentication required"));
}; };
let (cred, owner) = let (cred, owner) =
@@ -548,8 +571,10 @@ impl Operation for GetUserInfo {
let check_deny_only = should_check_deny_only(ak, &cred); let check_deny_only = should_check_deny_only(ak, &cred);
debug!( debug!(
target = "rustfs::admin::handlers::user", component = LOG_COMPONENT_ADMIN,
operation = "GetUserInfo", subsystem = LOG_SUBSYSTEM_USER,
event = EVENT_ADMIN_USER_STATE,
action = "get_user_info",
query_access_key = %ak, query_access_key = %ak,
signer_access_key = %cred.access_key, signer_access_key = %cred.access_key,
is_temp = cred.is_temp(), is_temp = cred.is_temp(),
@@ -559,7 +584,8 @@ impl Operation for GetUserInfo {
jwt_parent_claim_present = cred.claims.as_ref().and_then(|c| c.get("parent")).is_some(), jwt_parent_claim_present = cred.claims.as_ref().and_then(|c| c.get("parent")).is_some(),
check_deny_only, check_deny_only,
is_owner = owner, is_owner = owner,
"authorization context before validate_admin_request (no secrets)" state = "authorization_context",
"admin user state"
); );
// For eligible self operations, only explicit Deny should block the request. // For eligible self operations, only explicit Deny should block the request.
@@ -576,7 +602,7 @@ impl Operation for GetUserInfo {
let info = iam_store.get_user_info(ak).await.map_err(iam_error_to_s3_error)?; let info = iam_store.get_user_info(ak).await.map_err(iam_error_to_s3_error)?;
let data = serde_json::to_vec(&info) let data = serde_json::to_vec(&info)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal user err {e}")))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize response: {e}")))?;
let mut header = HeaderMap::new(); let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
@@ -845,7 +871,15 @@ impl Operation for ImportIam {
let body = match input.store_all_limited(MAX_IAM_IMPORT_SIZE).await { let body = match input.store_all_limited(MAX_IAM_IMPORT_SIZE).await {
Ok(b) => b, Ok(b) => b,
Err(e) => { Err(e) => {
warn!("get body failed, e: {:?}", e); warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_USER,
event = EVENT_ADMIN_USER_STATE,
action = "import_iam",
result = "body_read_failed",
error = ?e,
"admin user state"
);
return Err(s3_error!(InvalidRequest, "get body failed")); return Err(s3_error!(InvalidRequest, "get body failed"));
} }
}; };
rustfs/src/admin/router.rs
+41 -18
View File
@@ -104,6 +104,7 @@ use uuid::Uuid;
const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_OBJECT_LAMBDA: &str = "object_lambda"; const LOG_SUBSYSTEM_OBJECT_LAMBDA: &str = "object_lambda";
const LOG_SUBSYSTEM_LIVE_EVENTS: &str = "live_events"; const LOG_SUBSYSTEM_LIVE_EVENTS: &str = "live_events";
const EVENT_ADMIN_ROUTER_STATE: &str = "admin_router_state";
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ReplicationExtRoute { enum ReplicationExtRoute {
@@ -634,11 +635,12 @@ async fn load_current_server_config() -> S3Result<Config> {
Ok(config) => return Ok(config), Ok(config) => return Ok(config),
Err(err) => { Err(err) => {
warn!( warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA, subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
event = "object_lambda_config_reload_failed", result = "config_reload_failed",
error = %err, error = %err,
"Failed to reload current server config for object lambda request" "admin router state"
); );
} }
} }
@@ -663,11 +665,12 @@ fn build_object_lambda_http_client(config: &ObjectLambdaWebhookConfig) -> S3Resu
if config.skip_tls_verify { if config.skip_tls_verify {
warn!( warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA, subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
event = "object_lambda_tls_verification_disabled", result = "tls_verification_disabled",
endpoint = %config.endpoint, endpoint = %config.endpoint,
"Object Lambda target is configured to skip TLS certificate verification" "admin router state"
); );
builder = builder.danger_accept_invalid_certs(true); builder = builder.danger_accept_invalid_certs(true);
} else if !config.client_ca.is_empty() { } else if !config.client_ca.is_empty() {
@@ -1220,23 +1223,25 @@ async fn fan_in_remote_live_events(
Ok(Ok(batch)) => batch, Ok(Ok(batch)) => batch,
Ok(Err(err)) => { Ok(Err(err)) => {
warn!( warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "peer_live_events_fetch_failed",
peer = %peer.client.host, peer = %peer.client.host,
result = "peer_fetch_failed",
error = %err, error = %err,
"Failed to fetch live events from peer" "admin router state"
); );
break; break;
} }
Err(_) => { Err(_) => {
warn!( warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "peer_live_events_fetch_failed",
peer = %peer.client.host, peer = %peer.client.host,
result = "peer_fetch_timeout",
error = "timeout", error = "timeout",
"Failed to fetch live events from peer" "admin router state"
); );
break; break;
} }
@@ -1259,13 +1264,14 @@ async fn fan_in_remote_live_events(
} }
Err(err) => { Err(err) => {
warn!( warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "live_event_serialize_failed",
source = "remote_peer", source = "remote_peer",
peer = %peer.client.host, peer = %peer.client.host,
result = "event_serialize_failed",
error = %err, error = %err,
"Failed to serialize live event" "admin router state"
); );
} }
} }
@@ -1273,12 +1279,13 @@ async fn fan_in_remote_live_events(
} }
Err(err) => { Err(err) => {
warn!( warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "peer_live_events_decode_failed",
peer = %peer.client.host, peer = %peer.client.host,
result = "peer_decode_failed",
error = %err, error = %err,
"Failed to decode live events from peer" "admin router state"
); );
} }
} }
@@ -1333,23 +1340,25 @@ fn build_listen_notification_response(uri: &Uri, bucket: Option<&str>) -> S3Resu
} }
Err(err) => { Err(err) => {
warn!( warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "live_event_serialize_failed",
source = "local_stream", source = "local_stream",
result = "event_serialize_failed",
error = %err, error = %err,
"Failed to serialize live event" "admin router state"
); );
} }
} }
} }
Err(broadcast::error::RecvError::Lagged(skipped)) => { Err(broadcast::error::RecvError::Lagged(skipped)) => {
warn!( warn!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS,
event = "live_event_stream_lagged", result = "stream_lagged",
skipped, skipped,
"Live event stream lagged" "admin router state"
); );
} }
Err(broadcast::error::RecvError::Closed) => break, Err(broadcast::error::RecvError::Closed) => break,
@@ -1509,7 +1518,14 @@ async fn authorize_replication_extension_request(req: &mut S3Request<Body>, ext_
license_check().map_err(|er| match er.kind() { license_check().map_err(|er| match er.kind() {
std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"), std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"),
_ => { _ => {
error!("license check failed due to unexpected error: {er}"); error!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
result = "license_check_failed",
error = %er,
"admin router state"
);
s3_error!(InternalError, "License validation failed") s3_error!(InternalError, "License validation failed")
} }
})?; })?;
@@ -2231,7 +2247,14 @@ async fn authorize_misc_extension_request(req: &mut S3Request<Body>, route: &Mis
license_check().map_err(|er| match er.kind() { license_check().map_err(|er| match er.kind() {
std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"), std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"),
_ => { _ => {
error!("license check failed due to unexpected error: {er}"); error!(
event = EVENT_ADMIN_ROUTER_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
result = "license_check_failed",
error = %er,
"admin router state"
);
s3_error!(InternalError, "License validation failed") s3_error!(InternalError, "License validation failed")
} }
})?; })?;
rustfs/src/app/object_usecase.rs
+32 -32
View File
@@ -127,7 +127,7 @@ use tokio::io::{AsyncRead, ReadBuf};
use tokio::sync::RwLock; use tokio::sync::RwLock;
use tokio_tar::Archive; use tokio_tar::Archive;
use tokio_util::io::{ReaderStream, StreamReader}; use tokio_util::io::{ReaderStream, StreamReader};
use tracing::{debug, error, info, instrument, warn}; use tracing::{debug, error, instrument, warn};
use uuid::Uuid; use uuid::Uuid;
const ACCEPT_RANGES_BYTES: &str = "bytes"; const ACCEPT_RANGES_BYTES: &str = "bytes";
@@ -1120,15 +1120,18 @@ fn is_post_object_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap)
async fn resolve_put_object_expiration(bucket: &str, obj_info: &ObjectInfo) -> Option<String> { async fn resolve_put_object_expiration(bucket: &str, obj_info: &ObjectInfo) -> Option<String> {
let Ok((lifecycle_config, _)) = metadata_sys::get_lifecycle_config(bucket).await else { let Ok((lifecycle_config, _)) = metadata_sys::get_lifecycle_config(bucket).await else {
debug!("resolve_put_object_expiration: lifecycle config not found for bucket {bucket}"); debug!(bucket, state = "config_missing", "PUT object expiration config missing");
return None; return None;
}; };
let obj_opts = lifecycle::ObjectOpts::from_object_info(obj_info); let obj_opts = lifecycle::ObjectOpts::from_object_info(obj_info);
let event = lifecycle_config.predict_expiration(&obj_opts).await; let event = lifecycle_config.predict_expiration(&obj_opts).await;
debug!( debug!(
"resolve_put_object_expiration: bucket={bucket}, action={:?}, rule_id={}, due={:?}", bucket,
event.action, event.rule_id, event.due action = ?event.action,
rule_id = %event.rule_id,
due = ?event.due,
"PUT object expiration resolved"
); );
build_put_object_expiration_header(&event) build_put_object_expiration_header(&event)
} }
@@ -1184,7 +1187,7 @@ impl DefaultObjectUsecase {
), ),
)), )),
Err(e) => { Err(e) => {
warn!("Quota check failed for bucket {bucket}: {e}, allowing operation"); warn!(bucket, error = %e, state = "checker_failed", "Bucket quota check degraded to allow");
Ok(()) Ok(())
} }
_ => Ok(()), _ => Ok(()),
@@ -1427,7 +1430,7 @@ impl DefaultObjectUsecase {
match ContentType::from_str(content_type) { match ContentType::from_str(content_type) {
Ok(res) => Some(res), Ok(res) => Some(res),
Err(err) => { Err(err) => {
error!("parse content-type err {} {:?}", content_type, err); error!(content_type, error = ?err, "GET object content-type parse failed");
None None
} }
} }
@@ -1624,7 +1627,7 @@ impl DefaultObjectUsecase {
{ {
let (decrypted_checksums, _is_multipart) = let (decrypted_checksums, _is_multipart) =
info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| { info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| {
error!("decrypt_checksums error: {}", e); error!(error = %e, "GetObject checksum decryption failed");
ApiError::from(e) ApiError::from(e)
})?; })?;
@@ -1667,25 +1670,22 @@ impl DefaultObjectUsecase {
if should_buffer_encrypted_object { if should_buffer_encrypted_object {
let mut buf = Vec::with_capacity(response_content_length as usize); let mut buf = Vec::with_capacity(response_content_length as usize);
if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await {
error!("Failed to read decrypted object into memory: {}", e); error!(error = %e, "GetObject decrypted object buffering failed");
return Err(ApiError::from(StorageError::other(format!("Failed to read decrypted object: {e}"))).into()); return Err(ApiError::from(StorageError::other(format!("Failed to read decrypted object: {e}"))).into());
} }
if buf.len() != response_content_length as usize { if buf.len() != response_content_length as usize {
warn!( warn!(
"Encrypted object size mismatch during read: expected={} actual={}", expected = response_content_length,
response_content_length, actual = buf.len(),
buf.len() "Encrypted object size mismatch during read"
); );
} }
return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size)); return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size));
} }
info!( debug!(buffer_size = optimal_buffer_size, "Encrypted object uses streaming decrypt path");
"Encrypted object: Using unlimited stream for decryption with buffer size {}",
optimal_buffer_size
);
return Ok(Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size)); return Ok(Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size));
} }
@@ -1698,16 +1698,16 @@ impl DefaultObjectUsecase {
Ok(_) => { Ok(_) => {
if buf.len() != response_content_length as usize { if buf.len() != response_content_length as usize {
warn!( warn!(
"Object size mismatch during seek support read: expected={} actual={}", expected = response_content_length,
response_content_length, actual = buf.len(),
buf.len() "Object size mismatch during seek-support read"
); );
} }
return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size)); return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size));
} }
Err(e) => { Err(e) => {
error!("Failed to read object into memory for seek support: {}", e); error!(error = %e, "GetObject seek-support buffering failed");
} }
} }
} }
@@ -2780,7 +2780,7 @@ impl DefaultObjectUsecase {
&& src_bucket == bucket && src_bucket == bucket
&& src_key == key && src_key == key
{ {
error!("Rejected self-copy operation: bucket={}, key={}", bucket, key); error!(bucket, key, "Rejected self-copy operation");
return Err(s3_error!( return Err(s3_error!(
InvalidRequest, InvalidRequest,
"Cannot copy an object to itself. Source and destination must be different." "Cannot copy an object to itself. Source and destination must be different."
@@ -3679,7 +3679,7 @@ impl DefaultObjectUsecase {
let has_children = match probe_prefix_has_children(store, &bucket, &key, false).await { let has_children = match probe_prefix_has_children(store, &bucket, &key, false).await {
Ok(has_children) => has_children, Ok(has_children) => has_children,
Err(e) => { Err(e) => {
error!("Failed to probe children for prefix (bucket: {}, key: {}): {}", bucket, key, e); error!(bucket, key, error = %e, "Failed to probe children for prefix");
false false
} }
}; };
@@ -3751,7 +3751,7 @@ impl DefaultObjectUsecase {
match ContentType::from_str(content_type) { match ContentType::from_str(content_type) {
Ok(res) => Some(res), Ok(res) => Some(res),
Err(err) => { Err(err) => {
error!("parse content-type err {} {:?}", &content_type, err); error!(content_type = %content_type, error = ?err, "Archive content-type parse failed");
// //
None None
} }
@@ -3765,7 +3765,7 @@ impl DefaultObjectUsecase {
// TODO: range download // TODO: range download
let content_length = info.get_actual_size().map_err(|e| { let content_length = info.get_actual_size().map_err(|e| {
error!("get_actual_size error: {}", e); error!(error = %e, "Failed to resolve actual object size");
ApiError::from(e) ApiError::from(e)
})?; })?;
@@ -3881,7 +3881,7 @@ impl DefaultObjectUsecase {
if let Ok(header_value) = tag_count.to_string().parse::<HeaderValue>() { if let Ok(header_value) = tag_count.to_string().parse::<HeaderValue>() {
response.headers.insert(header_name, header_value); response.headers.insert(header_name, header_value);
} else { } else {
warn!("Failed to parse x-amz-tagging-count header value, skipping"); warn!("Failed to parse x-amz-tagging-count header; skipping");
} }
} }
if let Some(retain_date) = metadata_map if let Some(retain_date) = metadata_map
@@ -4127,7 +4127,7 @@ impl DefaultObjectUsecase {
err.to_string() err.to_string()
); );
} else { } else {
info!("successfully restored transitioned object: {}/{}", bucket_clone, object_clone); debug!(bucket = %bucket_clone, object = %object_clone, "Transitioned object restored");
} }
}); });
@@ -4302,13 +4302,13 @@ impl DefaultObjectUsecase {
let decoder = CompressionFormat::from_extension(&ext) let decoder = CompressionFormat::from_extension(&ext)
.get_decoder(ExtractArchiveEtagReader::new(archive_reader, archive_etag.clone())) .get_decoder(ExtractArchiveEtagReader::new(archive_reader, archive_etag.clone()))
.map_err(|e| { .map_err(|e| {
error!("get_decoder err {:?}", e); error!(error = ?e, "Archive decoder creation failed");
s3_error!(InvalidArgument, "get_decoder err") s3_error!(InvalidArgument, "get_decoder err")
})?; })?;
let mut ar = Archive::new(decoder); let mut ar = Archive::new(decoder);
let mut entries = ar.entries().map_err(|e| { let mut entries = ar.entries().map_err(|e| {
error!("get entries err {:?}", e); error!(error = ?e, "Archive entry listing failed");
s3_error!(InvalidArgument, "get entries err") s3_error!(InvalidArgument, "get entries err")
})?; })?;
@@ -4337,10 +4337,10 @@ impl DefaultObjectUsecase {
Ok(f) => f, Ok(f) => f,
Err(e) => { Err(e) => {
if extract_options.ignore_errors { if extract_options.ignore_errors {
warn!("Skipping archive entry because read failed and ignore-errors is enabled: {e}"); warn!(error = %e, "Archive entry read skipped due to ignore-errors");
continue; continue;
} }
error!("Failed to read archive entry: {}", e); error!(error = %e, "Archive entry read failed");
return Err(s3_error!(InvalidArgument, "Failed to read archive entry: {:?}", e)); return Err(s3_error!(InvalidArgument, "Failed to read archive entry: {:?}", e));
} }
}; };
@@ -4349,7 +4349,7 @@ impl DefaultObjectUsecase {
Ok(path) => path, Ok(path) => path,
Err(e) => { Err(e) => {
if extract_options.ignore_errors { if extract_options.ignore_errors {
warn!("Skipping archive entry because path decode failed and ignore-errors is enabled: {e}"); warn!(error = %e, "Archive path decode skipped due to ignore-errors");
continue; continue;
} }
return Err(s3_error!(InvalidArgument, "Failed to decode archive entry path")); return Err(s3_error!(InvalidArgument, "Failed to decode archive entry path"));
@@ -4361,7 +4361,7 @@ impl DefaultObjectUsecase {
Ok(fpath) => fpath, Ok(fpath) => fpath,
Err(err) => { Err(err) => {
if extract_options.ignore_errors { if extract_options.ignore_errors {
warn!("Skipping archive entry because path is unsafe and ignore-errors is enabled: {err}"); warn!(error = %err, "Unsafe archive path skipped due to ignore-errors");
continue; continue;
} }
return Err(err); return Err(err);
@@ -4491,7 +4491,7 @@ impl DefaultObjectUsecase {
Ok(info) => info, Ok(info) => info,
Err(e) => { Err(e) => {
if extract_options.ignore_errors { if extract_options.ignore_errors {
warn!("Skipping archive entry because object write failed and ignore-errors is enabled: {e}"); warn!(error = %e, "Archive object write skipped due to ignore-errors");
continue; continue;
} }
return Err(ApiError::from(e).into()); return Err(ApiError::from(e).into());
rustfs/src/auth.rs
+13 -13
View File
@@ -151,7 +151,7 @@ impl S3Auth for IAMAuth {
subsystem = LOG_SUBSYSTEM_KEYSTONE, subsystem = LOG_SUBSYSTEM_KEYSTONE,
principal = %MaskedAccessKey(&creds.parent_user), principal = %MaskedAccessKey(&creds.parent_user),
result = "token_auth", result = "token_auth",
"Keystone credentials found in task-local storage" "Keystone task-local credentials detected"
); );
// Return empty secret key - Keystone uses token validation, not AWS signatures // Return empty secret key - Keystone uses token validation, not AWS signatures
return Ok(SecretKey::from(String::new())); return Ok(SecretKey::from(String::new()));
@@ -170,7 +170,7 @@ impl S3Auth for IAMAuth {
subsystem = LOG_SUBSYSTEM_KEYSTONE, subsystem = LOG_SUBSYSTEM_KEYSTONE,
access_key = %MaskedAccessKey(access_key), access_key = %MaskedAccessKey(access_key),
result = "token_auth", result = "token_auth",
"Keystone access key detected for token-based auth" "Keystone token-auth access key detected"
); );
// Return empty secret key - Keystone uses token validation, not AWS signatures // Return empty secret key - Keystone uses token validation, not AWS signatures
// The actual credentials are stored in task-local storage by KeystoneAuthMiddleware // The actual credentials are stored in task-local storage by KeystoneAuthMiddleware
@@ -202,7 +202,7 @@ impl S3Auth for IAMAuth {
subsystem = LOG_SUBSYSTEM_CREDENTIALS, subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key), access_key = %MaskedAccessKey(access_key),
reason = "no_such_user", reason = "no_such_user",
"Secret key lookup failed" "Secret key lookup rejected"
); );
} }
Err(e) => { Err(e) => {
@@ -213,7 +213,7 @@ impl S3Auth for IAMAuth {
access_key = %MaskedAccessKey(access_key), access_key = %MaskedAccessKey(access_key),
error = ?e, error = ?e,
reason = "check_key_error", reason = "check_key_error",
"Secret key lookup failed" "Secret key lookup errored"
); );
return Err(iam_lookup_error_to_s3_error(&e)); return Err(iam_lookup_error_to_s3_error(&e));
} }
@@ -225,7 +225,7 @@ impl S3Auth for IAMAuth {
subsystem = LOG_SUBSYSTEM_CREDENTIALS, subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key), access_key = %MaskedAccessKey(access_key),
reason = "iam_not_initialized", reason = "iam_not_initialized",
"Secret key lookup failed" "Secret key lookup skipped"
); );
} }
@@ -257,7 +257,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
subsystem = LOG_SUBSYSTEM_CREDENTIALS, subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key), access_key = %MaskedAccessKey(access_key),
has_session_token = !session_token.is_empty(), has_session_token = !session_token.is_empty(),
"Starting access key validation" "Access key validation started"
); );
if let Ok(Some(credentials)) = KEYSTONE_CREDENTIALS.try_with(|creds| creds.clone()) { if let Ok(Some(credentials)) = KEYSTONE_CREDENTIALS.try_with(|creds| creds.clone()) {
debug!( debug!(
@@ -265,7 +265,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
component = LOG_COMPONENT_AUTH, component = LOG_COMPONENT_AUTH,
subsystem = LOG_SUBSYSTEM_KEYSTONE, subsystem = LOG_SUBSYSTEM_KEYSTONE,
result = "task_local", result = "task_local",
"Keystone credentials found in task-local storage" "Keystone task-local credentials detected"
); );
if !auth_keystone::is_keystone_enabled() { if !auth_keystone::is_keystone_enabled() {
@@ -283,7 +283,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
.and_then(|c| c.get("keystone_project_name")) .and_then(|c| c.get("keystone_project_name"))
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.is_some(), .is_some(),
"Validated Keystone credentials from task-local storage" "Keystone task-local credentials validated"
); );
// Determine if user is admin (owner-level access) // Determine if user is admin (owner-level access)
@@ -307,7 +307,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
subsystem = LOG_SUBSYSTEM_KEYSTONE, subsystem = LOG_SUBSYSTEM_KEYSTONE,
principal = %MaskedAccessKey(&credentials.parent_user), principal = %MaskedAccessKey(&credentials.parent_user),
is_owner, is_owner,
"Evaluated Keystone owner permissions" "Keystone owner permissions evaluated"
); );
return Ok((credentials, is_owner)); return Ok((credentials, is_owner));
@@ -320,7 +320,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
component = LOG_COMPONENT_AUTH, component = LOG_COMPONENT_AUTH,
subsystem = LOG_SUBSYSTEM_KEYSTONE, subsystem = LOG_SUBSYSTEM_KEYSTONE,
access_key = %MaskedAccessKey(access_key), access_key = %MaskedAccessKey(access_key),
"Keystone access key detected without task-local credentials" "Keystone context missing for access key"
); );
if !auth_keystone::is_keystone_enabled() { if !auth_keystone::is_keystone_enabled() {
@@ -360,7 +360,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
subsystem = LOG_SUBSYSTEM_CREDENTIALS, subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key), access_key = %MaskedAccessKey(access_key),
reason = "user_not_found", reason = "user_not_found",
"Access key validation failed" "Access key validation rejected"
); );
return Err(s3_error!(InvalidAccessKeyId, "check key failed")); return Err(s3_error!(InvalidAccessKeyId, "check key failed"));
}; };
@@ -372,7 +372,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
subsystem = LOG_SUBSYSTEM_CREDENTIALS, subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key), access_key = %MaskedAccessKey(access_key),
reason = "account_disabled", reason = "account_disabled",
"Access key validation failed" "Access key validation rejected"
); );
return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled")); return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled"));
} }
@@ -383,7 +383,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
subsystem = LOG_SUBSYSTEM_CREDENTIALS, subsystem = LOG_SUBSYSTEM_CREDENTIALS,
access_key = %MaskedAccessKey(access_key), access_key = %MaskedAccessKey(access_key),
reason = "validation_failed", reason = "validation_failed",
"Access key validation failed" "Access key validation rejected"
); );
return Err(s3_error!(InvalidRequest, "check key failed")); return Err(s3_error!(InvalidRequest, "check key failed"));
} }
rustfs/src/init.rs
+216 -28
View File
@@ -36,6 +36,8 @@ const LOG_SUBSYSTEM_KMS: &str = "kms";
const LOG_SUBSYSTEM_BUFFER: &str = "buffer_profile"; const LOG_SUBSYSTEM_BUFFER: &str = "buffer_profile";
const LOG_SUBSYSTEM_AUTOTUNER: &str = "autotuner"; const LOG_SUBSYSTEM_AUTOTUNER: &str = "autotuner";
const LOG_SUBSYSTEM_PROTOCOL: &str = "protocol"; const LOG_SUBSYSTEM_PROTOCOL: &str = "protocol";
const EVENT_PROTOCOL_RUNTIME_STATE: &str = "protocol_runtime_state";
const EVENT_PROTOCOL_SERVER_STATE: &str = "protocol_server_state";
#[instrument] #[instrument]
pub fn print_server_info() { pub fn print_server_info() {
@@ -666,7 +668,7 @@ where
protocol = protocol_name, protocol = protocol_name,
state = "runtime_failed", state = "runtime_failed",
error = %e, error = %e,
"Protocol server state changed" "Protocol server failed"
); );
} }
info!( info!(
@@ -676,7 +678,7 @@ where
subsystem = LOG_SUBSYSTEM_PROTOCOL, subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = protocol_name, protocol = protocol_name,
state = "stopped", state = "stopped",
"Protocol server state changed" "Protocol server stopped"
); );
}); });
@@ -786,7 +788,15 @@ pub async fn init_ftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::er
// Check if FTP is enabled // Check if FTP is enabled
let ftp_enable = rustfs_utils::get_env_bool(ENV_FTP_ENABLE, false); let ftp_enable = rustfs_utils::get_env_bool(ENV_FTP_ENABLE, false);
if !ftp_enable { if !ftp_enable {
debug!("FTP system is disabled"); debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftp",
state = "disabled",
"Protocol runtime disabled"
);
return Ok(None); return Ok(None);
} }
@@ -817,12 +827,21 @@ pub async fn init_ftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::er
let fs = crate::storage::ecfs::FS::new(); let fs = crate::storage::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs); let storage_client = ProtocolStorageClient::new(fs);
let server: FtpsServer<ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?; let server: FtpsServer<ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?;
let bind_addr = server.config().bind_addr;
let passive_ports = server.config().passive_ports.clone();
// Log server configuration // Log server configuration
info!( debug!(
"FTP server configured on {} with passive ports {:?}", target: "rustfs::init",
server.config().bind_addr, event = EVENT_PROTOCOL_RUNTIME_STATE,
server.config().passive_ports component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftp",
state = "configured",
bind_addr = %bind_addr,
passive_ports = ?passive_ports,
tls_enabled = false,
"Protocol runtime configured"
); );
// Start FTP server in background task with proper shutdown support // Start FTP server in background task with proper shutdown support
@@ -830,12 +849,39 @@ pub async fn init_ftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::er
let task_handle = tokio::spawn(async move { let task_handle = tokio::spawn(async move {
if let Err(e) = server.start(shutdown_rx).await { if let Err(e) = server.start(shutdown_rx).await {
error!("FTP server error: {}", e); error!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftp",
state = "runtime_failed",
error = %e,
"Protocol server failed"
);
} }
info!("FTP server shutdown completed"); info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftp",
state = "stopped",
"Protocol server stopped"
);
}); });
info!("FTP system initialized successfully"); info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftp",
state = "started",
bind_addr = %bind_addr,
tls_enabled = false,
"Protocol runtime started"
);
Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle))) Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle)))
} }
} }
@@ -859,7 +905,15 @@ pub async fn init_ftps_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
// Check if FTPS is enabled // Check if FTPS is enabled
let ftps_enable = rustfs_utils::get_env_bool(ENV_FTPS_ENABLE, false); let ftps_enable = rustfs_utils::get_env_bool(ENV_FTPS_ENABLE, false);
if !ftps_enable { if !ftps_enable {
debug!("FTPS system is disabled"); debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftps",
state = "disabled",
"Protocol runtime disabled"
);
return Ok(None); return Ok(None);
} }
@@ -893,12 +947,22 @@ pub async fn init_ftps_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
let fs = crate::storage::ecfs::FS::new(); let fs = crate::storage::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs); let storage_client = ProtocolStorageClient::new(fs);
let server: FtpsServer<ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?; let server: FtpsServer<ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?;
let bind_addr = server.config().bind_addr;
let passive_ports = server.config().passive_ports.clone();
let tls_enabled = server.config().tls_enabled;
// Log server configuration // Log server configuration
info!( debug!(
"FTPS server configured on {} with passive ports {:?}", target: "rustfs::init",
server.config().bind_addr, event = EVENT_PROTOCOL_RUNTIME_STATE,
server.config().passive_ports component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftps",
state = "configured",
bind_addr = %bind_addr,
passive_ports = ?passive_ports,
tls_enabled,
"Protocol runtime configured"
); );
// Start FTPS server in background task with proper shutdown support // Start FTPS server in background task with proper shutdown support
@@ -906,12 +970,39 @@ pub async fn init_ftps_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
let task_handle = tokio::spawn(async move { let task_handle = tokio::spawn(async move {
if let Err(e) = server.start(shutdown_rx).await { if let Err(e) = server.start(shutdown_rx).await {
error!("FTPS server error: {}", e); error!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftps",
state = "runtime_failed",
error = %e,
"Protocol server failed"
);
} }
info!("FTPS server shutdown completed"); info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftps",
state = "stopped",
"Protocol server stopped"
);
}); });
info!("FTPS system initialized successfully"); info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "ftps",
state = "started",
bind_addr = %bind_addr,
tls_enabled,
"Protocol runtime started"
);
Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle))) Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle)))
} }
} }
@@ -935,7 +1026,15 @@ pub async fn init_webdav_system() -> Result<Option<ShutdownHandle>, Box<dyn std:
// Check if WebDAV is enabled // Check if WebDAV is enabled
let webdav_enable = rustfs_utils::get_env_bool(ENV_WEBDAV_ENABLE, false); let webdav_enable = rustfs_utils::get_env_bool(ENV_WEBDAV_ENABLE, false);
if !webdav_enable { if !webdav_enable {
debug!("WebDAV system is disabled"); debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "webdav",
state = "disabled",
"Protocol runtime disabled"
);
return Ok(None); return Ok(None);
} }
@@ -966,21 +1065,64 @@ pub async fn init_webdav_system() -> Result<Option<ShutdownHandle>, Box<dyn std:
let fs = crate::storage::ecfs::FS::new(); let fs = crate::storage::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs); let storage_client = ProtocolStorageClient::new(fs);
let server: WebDavServer<crate::protocols::ProtocolStorageClient> = WebDavServer::new(config, storage_client).await?; let server: WebDavServer<crate::protocols::ProtocolStorageClient> = WebDavServer::new(config, storage_client).await?;
let bind_addr = server.config().bind_addr;
let tls_enabled = server.config().tls_enabled;
let max_body_size = server.config().max_body_size;
let request_timeout_secs = server.config().request_timeout_secs;
// Log server configuration // Log server configuration
info!("WebDAV server configured on {}", server.config().bind_addr); debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "webdav",
state = "configured",
bind_addr = %bind_addr,
tls_enabled,
max_body_size,
request_timeout_secs,
"Protocol runtime configured"
);
// Start WebDAV server in background task with proper shutdown support // Start WebDAV server in background task with proper shutdown support
let (shutdown_tx, shutdown_rx) = tokio::sync::broadcast::channel(1); let (shutdown_tx, shutdown_rx) = tokio::sync::broadcast::channel(1);
let task_handle = tokio::spawn(async move { let task_handle = tokio::spawn(async move {
if let Err(e) = server.start(shutdown_rx).await { if let Err(e) = server.start(shutdown_rx).await {
error!("WebDAV server error: {}", e); error!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "webdav",
state = "runtime_failed",
error = %e,
"Protocol server failed"
);
} }
info!("WebDAV server shutdown completed"); info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "webdav",
state = "stopped",
"Protocol server stopped"
);
}); });
info!("WebDAV system initialized successfully"); info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "webdav",
state = "started",
bind_addr = %bind_addr,
tls_enabled,
"Protocol runtime started"
);
Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle))) Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle)))
} }
} }
@@ -1003,7 +1145,15 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
let enabled = rustfs_utils::get_env_bool(ENV_SFTP_ENABLE, false); let enabled = rustfs_utils::get_env_bool(ENV_SFTP_ENABLE, false);
if !enabled { if !enabled {
debug!("SFTP system is disabled"); debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "sftp",
state = "disabled",
"Protocol runtime disabled"
);
return Ok(None); return Ok(None);
} }
@@ -1051,7 +1201,18 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
let server = SftpServer::new(config.clone(), storage_client, host_keys)?; let server = SftpServer::new(config.clone(), storage_client, host_keys)?;
info!("SFTP server configured on {}", config.bind_addr); debug!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "sftp",
state = "configured",
bind_addr = %config.bind_addr,
read_only = config.read_only,
host_key_dir = %config.host_key_dir.display(),
"Protocol runtime configured"
);
// Hook into shutdown support // Hook into shutdown support
let (shutdown_tx, shutdown_rx) = tokio::sync::broadcast::channel(1); let (shutdown_tx, shutdown_rx) = tokio::sync::broadcast::channel(1);
@@ -1059,12 +1220,39 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
// Start SFTP server in background task // Start SFTP server in background task
let task_handle = tokio::spawn(async move { let task_handle = tokio::spawn(async move {
if let Err(e) = server.start(shutdown_rx).await { if let Err(e) = server.start(shutdown_rx).await {
error!("SFTP server error: {}", e); error!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "sftp",
state = "runtime_failed",
error = %e,
"Protocol server failed"
);
} }
info!("SFTP server shutdown completed"); info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_SERVER_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "sftp",
state = "stopped",
"Protocol server stopped"
);
}); });
info!("SFTP system initialized successfully"); info!(
target: "rustfs::init",
event = EVENT_PROTOCOL_RUNTIME_STATE,
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_PROTOCOL,
protocol = "sftp",
state = "started",
bind_addr = %config.bind_addr,
read_only = config.read_only,
"Protocol runtime started"
);
Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle))) Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle)))
} }
} }
rustfs/src/main.rs
+63 -43
View File
@@ -89,6 +89,7 @@ const EVENT_DEFAULT_CREDENTIALS_DETECTED: &str = "default_credentials_detected";
const EVENT_SERVER_CONFIG_SANITIZED: &str = "server_config_sanitized"; const EVENT_SERVER_CONFIG_SANITIZED: &str = "server_config_sanitized";
const EVENT_SERVER_STARTING: &str = "server_starting"; const EVENT_SERVER_STARTING: &str = "server_starting";
const EVENT_SERVER_RUNTIME_FAILED: &str = "server_runtime_failed"; const EVENT_SERVER_RUNTIME_FAILED: &str = "server_runtime_failed";
const EVENT_ACTION_CREDENTIALS_INITIALIZED: &str = "action_credentials_initialized";
const EVENT_ACTION_CREDENTIALS_INITIALIZATION_FAILED: &str = "action_credentials_initialization_failed"; const EVENT_ACTION_CREDENTIALS_INITIALIZATION_FAILED: &str = "action_credentials_initialization_failed";
const EVENT_OBSERVABILITY_GUARD_SET: &str = "observability_guard_set"; const EVENT_OBSERVABILITY_GUARD_SET: &str = "observability_guard_set";
const EVENT_CRYPTO_PROVIDER_STATE: &str = "crypto_provider_state"; const EVENT_CRYPTO_PROVIDER_STATE: &str = "crypto_provider_state";
@@ -384,7 +385,14 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
// Set up AK and SK // Set up AK and SK
match init_global_action_credentials(Some(config.access_key.clone()), Some(config.secret_key.clone())) { match init_global_action_credentials(Some(config.access_key.clone()), Some(config.secret_key.clone())) {
Ok(_) => { Ok(_) => {
debug!(target: "rustfs::main::run", "Global action credentials initialized successfully."); debug!(
target: "rustfs::main::run",
event = EVENT_ACTION_CREDENTIALS_INITIALIZED,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_AUTH,
result = "ok",
"Initialized global action credentials"
);
} }
Err(e) => { Err(e) => {
let msg = format!("init global action credentials failed: {e:?}"); let msg = format!("init global action credentials failed: {e:?}");
@@ -419,9 +427,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
.inspect_err(|err| { .inspect_err(|err| {
error!( error!(
target: "rustfs::main::run", target: "rustfs::main::run",
event = EVENT_STARTUP_STORAGE_STAGE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STORAGE,
stage = "endpoint_parsing", stage = "endpoint_parsing",
state = "failed",
error = ?err, error = ?err,
"startup storage stage failed" "Endpoint parsing failed"
); );
}) })
.map_err(Error::other)?; .map_err(Error::other)?;
@@ -445,9 +457,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
.inspect_err(|err| { .inspect_err(|err| {
error!( error!(
target: "rustfs::main::run", target: "rustfs::main::run",
event = EVENT_STARTUP_STORAGE_STAGE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STORAGE,
stage = "local_disk_initialization", stage = "local_disk_initialization",
state = "failed",
error = ?err, error = ?err,
"startup storage stage failed" "Local disk initialization failed"
); );
}) })
.map_err(Error::other)?; .map_err(Error::other)?;
@@ -540,9 +556,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
.inspect_err(|err| { .inspect_err(|err| {
error!( error!(
target: "rustfs::main::run", target: "rustfs::main::run",
event = EVENT_STARTUP_STORAGE_STAGE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STORAGE,
stage = "ecstore_initialization", stage = "ecstore_initialization",
state = "failed",
error = ?err, error = ?err,
"startup storage stage failed" "ECStore initialization failed"
); );
})?; })?;
@@ -561,7 +581,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
state = "retrying", state = "retrying",
retry_count = retry_count + 1, retry_count = retry_count + 1,
error = ?e, error = ?e,
"Startup storage stage failed" "Global config initialization retry failed"
); );
// TODO: check error type // TODO: check error type
retry_count += 1; retry_count += 1;
@@ -580,13 +600,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
#[cfg(feature = "ftps")] #[cfg(feature = "ftps")]
let ftp_shutdown_tx = match init_ftp_system().await { let ftp_shutdown_tx = match init_ftp_system().await {
Ok(Some(tx)) => { Ok(Some(tx)) => {
info!( debug!(
event = EVENT_PROTOCOL_SYSTEM_STATE, event = EVENT_PROTOCOL_SYSTEM_STATE,
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftp", protocol = "ftp",
state = "initialized", state = "started",
"Protocol system state changed" "Protocol runtime started"
); );
Some(tx) Some(tx)
} }
@@ -597,7 +617,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftp", protocol = "ftp",
state = "disabled", state = "disabled",
"Protocol system state changed" "Protocol runtime disabled"
); );
None None
} }
@@ -609,7 +629,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
protocol = "ftp", protocol = "ftp",
state = "initialization_failed", state = "initialization_failed",
error = %e, error = %e,
"Protocol system state changed" "Protocol runtime initialization failed"
); );
return Err(Error::other(e)); return Err(Error::other(e));
} }
@@ -622,13 +642,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
#[cfg(feature = "ftps")] #[cfg(feature = "ftps")]
let ftps_shutdown_tx = match init_ftps_system().await { let ftps_shutdown_tx = match init_ftps_system().await {
Ok(Some(tx)) => { Ok(Some(tx)) => {
info!( debug!(
event = EVENT_PROTOCOL_SYSTEM_STATE, event = EVENT_PROTOCOL_SYSTEM_STATE,
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftps", protocol = "ftps",
state = "initialized", state = "started",
"Protocol system state changed" "Protocol runtime started"
); );
Some(tx) Some(tx)
} }
@@ -639,7 +659,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftps", protocol = "ftps",
state = "disabled", state = "disabled",
"Protocol system state changed" "Protocol runtime disabled"
); );
None None
} }
@@ -651,7 +671,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
protocol = "ftps", protocol = "ftps",
state = "initialization_failed", state = "initialization_failed",
error = %e, error = %e,
"Protocol system state changed" "Protocol runtime initialization failed"
); );
return Err(Error::other(e)); return Err(Error::other(e));
} }
@@ -664,13 +684,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
#[cfg(feature = "webdav")] #[cfg(feature = "webdav")]
let webdav_shutdown_tx = match init_webdav_system().await { let webdav_shutdown_tx = match init_webdav_system().await {
Ok(Some(tx)) => { Ok(Some(tx)) => {
info!( debug!(
event = EVENT_PROTOCOL_SYSTEM_STATE, event = EVENT_PROTOCOL_SYSTEM_STATE,
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "webdav", protocol = "webdav",
state = "initialized", state = "started",
"Protocol system state changed" "Protocol runtime started"
); );
Some(tx) Some(tx)
} }
@@ -681,7 +701,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "webdav", protocol = "webdav",
state = "disabled", state = "disabled",
"Protocol system state changed" "Protocol runtime disabled"
); );
None None
} }
@@ -693,7 +713,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
protocol = "webdav", protocol = "webdav",
state = "initialization_failed", state = "initialization_failed",
error = %e, error = %e,
"Protocol system state changed" "Protocol runtime initialization failed"
); );
return Err(Error::other(e)); return Err(Error::other(e));
} }
@@ -706,13 +726,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
#[cfg(feature = "sftp")] #[cfg(feature = "sftp")]
let sftp_shutdown_tx = match init_sftp_system().await { let sftp_shutdown_tx = match init_sftp_system().await {
Ok(Some(tx)) => { Ok(Some(tx)) => {
info!( debug!(
event = EVENT_PROTOCOL_SYSTEM_STATE, event = EVENT_PROTOCOL_SYSTEM_STATE,
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "sftp", protocol = "sftp",
state = "initialized", state = "started",
"Protocol system state changed" "Protocol runtime started"
); );
Some(tx) Some(tx)
} }
@@ -723,7 +743,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "sftp", protocol = "sftp",
state = "disabled", state = "disabled",
"Protocol system state changed" "Protocol runtime disabled"
); );
None None
} }
@@ -735,7 +755,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
protocol = "sftp", protocol = "sftp",
state = "initialization_failed", state = "initialization_failed",
error = %e, error = %e,
"Protocol system state changed" "Protocol runtime initialization failed"
); );
return Err(Error::other(e)); return Err(Error::other(e));
} }
@@ -758,7 +778,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
state = "started", state = "started",
"Audit system state changed" "Audit runtime started"
), ),
Err(e) => error!( Err(e) => error!(
target: "rustfs::main::run", target: "rustfs::main::run",
@@ -767,7 +787,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
state = "start_failed", state = "start_failed",
error = %e, error = %e,
"Audit system state changed" "Audit runtime failed to start"
), ),
} }
@@ -781,7 +801,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
state = "started", state = "started",
"Deadlock detector state changed" "Deadlock detector started"
); );
} else { } else {
info!( info!(
@@ -790,7 +810,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
state = "disabled", state = "disabled",
"Deadlock detector state changed" "Deadlock detector disabled"
); );
} }
@@ -888,7 +908,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
enable_scanner = enable_scanner, enable_scanner = enable_scanner,
enable_heal = enable_heal, enable_heal = enable_heal,
"Configured background services" "Background services configured"
); );
// Scanner depends on the heal channel/manager, so scanner implies heal. // Scanner depends on the heal channel/manager, so scanner implies heal.
@@ -907,7 +927,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
enable_heal = false, enable_heal = false,
ahm_state = "skipped", ahm_state = "skipped",
reason = "disabled", reason = "disabled",
"Background services configured" "Background services disabled"
); );
} }
@@ -935,7 +955,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
server_address = %server_address, server_address = %server_address,
started_at = %jiff::Zoned::now(), started_at = %jiff::Zoned::now(),
iam_bootstrap = ?iam_bootstrap, iam_bootstrap = ?iam_bootstrap,
"RustFS server started successfully" "RustFS server ready"
); );
if iam_bootstrap == IamBootstrapDisposition::ReadyInline { if iam_bootstrap == IamBootstrapDisposition::ReadyInline {
rustfs::server::publish_ready_when_runtime_ready(readiness.as_ref(), Some(state_manager.as_ref())).await?; rustfs::server::publish_ready_when_runtime_ready(readiness.as_ref(), Some(state_manager.as_ref())).await?;
@@ -971,7 +991,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
state = ?state_manager.current_state(), state = ?state_manager.current_state(),
result = "stopped", result = "stopped",
"Server shutdown state changed" "RustFS server stopped"
); );
Ok(()) Ok(())
} }
@@ -1051,7 +1071,7 @@ async fn handle_shutdown(
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
signal = shutdown_signal.log_label(), signal = shutdown_signal.log_label(),
"Shutdown signal received in main thread" "Shutdown signal received"
); );
// update the status to stopping first // update the status to stopping first
state_manager.update(ServiceState::Stopping); state_manager.update(ServiceState::Stopping);
@@ -1114,7 +1134,7 @@ async fn handle_shutdown(
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftp", protocol = "ftp",
state = "stopping", state = "stopping",
"Protocol system state changed" "Protocol runtime stopping"
); );
protocol_shutdowns.push(ftp_shutdown_tx.shutdown()); protocol_shutdowns.push(ftp_shutdown_tx.shutdown());
} }
@@ -1127,7 +1147,7 @@ async fn handle_shutdown(
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "ftps", protocol = "ftps",
state = "stopping", state = "stopping",
"Protocol system state changed" "Protocol runtime stopping"
); );
protocol_shutdowns.push(ftps_shutdown_tx.shutdown()); protocol_shutdowns.push(ftps_shutdown_tx.shutdown());
} }
@@ -1141,7 +1161,7 @@ async fn handle_shutdown(
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "webdav", protocol = "webdav",
state = "stopping", state = "stopping",
"Protocol system state changed" "Protocol runtime stopping"
); );
protocol_shutdowns.push(webdav_shutdown_tx.shutdown()); protocol_shutdowns.push(webdav_shutdown_tx.shutdown());
} }
@@ -1155,7 +1175,7 @@ async fn handle_shutdown(
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
protocol = "sftp", protocol = "sftp",
state = "stopping", state = "stopping",
"Protocol system state changed" "Protocol runtime stopping"
); );
protocol_shutdowns.push(sftp_shutdown_tx.shutdown()); protocol_shutdowns.push(sftp_shutdown_tx.shutdown());
} }
@@ -1178,7 +1198,7 @@ async fn handle_shutdown(
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stopping", state = "stopping",
"Audit system state changed" "Audit runtime stopping"
); );
match stop_audit_system().await { match stop_audit_system().await {
Ok(_) => info!( Ok(_) => info!(
@@ -1187,7 +1207,7 @@ async fn handle_shutdown(
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stopped", state = "stopped",
"Audit system state changed" "Audit runtime stopped"
), ),
Err(e) => error!( Err(e) => error!(
target: "rustfs::main::handle_shutdown", target: "rustfs::main::handle_shutdown",
@@ -1196,7 +1216,7 @@ async fn handle_shutdown(
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stop_failed", state = "stop_failed",
error = %e, error = %e,
"Audit system state changed" "Audit runtime failed to stop"
), ),
} }
@@ -1217,7 +1237,7 @@ async fn handle_shutdown(
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stopping", state = "stopping",
"Server shutdown state changed" "RustFS server stopping"
); );
if let Some(s3_shutdown_handle) = s3_shutdown_handle { if let Some(s3_shutdown_handle) = s3_shutdown_handle {
s3_shutdown_handle.shutdown().await; s3_shutdown_handle.shutdown().await;
@@ -1235,7 +1255,7 @@ async fn handle_shutdown(
component = LOG_COMPONENT_MAIN, component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP, subsystem = LOG_SUBSYSTEM_STARTUP,
state = "stopped", state = "stopped",
"Server shutdown state changed" "RustFS server stopped"
); );
} }
rustfs/src/profiling.rs
+201 -50
View File
@@ -16,26 +16,37 @@
mod unsupported_impl { mod unsupported_impl {
use std::path::PathBuf; use std::path::PathBuf;
use std::time::Duration; use std::time::Duration;
use tracing::info; use tracing::{debug, info};
const LOG_COMPONENT_PROFILING: &str = "profiling";
const LOG_SUBSYSTEM_PLATFORM: &str = "platform";
pub async fn init_from_env() { pub async fn init_from_env() {
let target_env = option_env!("CARGO_CFG_TARGET_ENV").unwrap_or("unknown"); let target_env = option_env!("CARGO_CFG_TARGET_ENV").unwrap_or("unknown");
info!( info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PLATFORM,
event = "profiling_runtime_skipped",
reason = "unsupported_platform",
target_os = std::env::consts::OS, target_os = std::env::consts::OS,
target_env, target_env,
target_arch = std::env::consts::ARCH, target_arch = std::env::consts::ARCH,
"Profiling initialization skipped on unsupported platform" "Profiling runtime skipped"
); );
} }
/// Stop all background profiling tasks /// Stop all background profiling tasks
pub fn shutdown_profiling() { pub fn shutdown_profiling() {
let target_env = option_env!("CARGO_CFG_TARGET_ENV").unwrap_or("unknown"); let target_env = option_env!("CARGO_CFG_TARGET_ENV").unwrap_or("unknown");
info!( debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PLATFORM,
event = "profiling_shutdown_skipped",
reason = "unsupported_platform",
target_os = std::env::consts::OS, target_os = std::env::consts::OS,
target_env, target_env,
target_arch = std::env::consts::ARCH, target_arch = std::env::consts::ARCH,
"profiling: shutdown called on unsupported platform (no-op)" "Profiling shutdown skipped"
); );
} }
@@ -80,7 +91,10 @@ mod linux_impl {
use tracing::{debug, info, warn}; use tracing::{debug, info, warn};
const LOG_COMPONENT_PROFILING: &str = "profiling"; const LOG_COMPONENT_PROFILING: &str = "profiling";
const LOG_SUBSYSTEM_PROFILING: &str = "profiling"; const LOG_SUBSYSTEM_CPU: &str = "cpu";
const LOG_SUBSYSTEM_MEMORY: &str = "memory";
const LOG_SUBSYSTEM_JEMALLOC: &str = "jemalloc";
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
static CPU_CONT_GUARD: OnceLock<Arc<Mutex<Option<pprof::ProfilerGuard<'static>>>>> = OnceLock::new(); static CPU_CONT_GUARD: OnceLock<Arc<Mutex<Option<pprof::ProfilerGuard<'static>>>>> = OnceLock::new();
static PROFILING_CANCEL_TOKEN: OnceLock<CancellationToken> = OnceLock::new(); static PROFILING_CANCEL_TOKEN: OnceLock<CancellationToken> = OnceLock::new();
@@ -100,7 +114,7 @@ mod linux_impl {
if let Err(e) = create_dir_all(&p) { if let Err(e) = create_dir_all(&p) {
warn!( warn!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_RUNTIME,
event = "profiling_output_dir_fallback", event = "profiling_output_dir_fallback",
path = %p.display(), path = %p.display(),
error = %e, error = %e,
@@ -143,7 +157,7 @@ mod linux_impl {
write_pprof_report_pb(&report, &out)?; write_pprof_report_pb(&report, &out)?;
info!( info!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_dump_exported", event = "profiling_dump_exported",
profile_type = "cpu", profile_type = "cpu",
path = %out.display(), path = %out.display(),
@@ -159,7 +173,7 @@ mod linux_impl {
if let Some(ref guard) = *guard_slot { if let Some(ref guard) = *guard_slot {
debug!( debug!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_dump_source", event = "profiling_dump_source",
profile_type = "cpu", profile_type = "cpu",
source = "continuous_guard", source = "continuous_guard",
@@ -195,7 +209,7 @@ mod linux_impl {
f.write_all(&bytes).map_err(|e| format!("write file failed: {e}"))?; f.write_all(&bytes).map_err(|e| format!("write file failed: {e}"))?;
info!( info!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_exported", event = "profiling_dump_exported",
profile_type = "memory", profile_type = "memory",
path = %out.display(), path = %out.display(),
@@ -215,24 +229,69 @@ mod linux_impl {
use tikv_jemalloc_ctl::{config, epoch, stats}; use tikv_jemalloc_ctl::{config, epoch, stats};
if let Err(e) = epoch::advance() { if let Err(e) = epoch::advance() {
warn!("jemalloc epoch advance failed: {e}"); warn!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_epoch_advance_failed",
error = %e,
"Jemalloc profiling state changed"
);
} }
match config::malloc_conf::read() { match config::malloc_conf::read() {
Ok(conf) => debug!("jemalloc malloc_conf: {}", conf), Ok(conf) => debug!(
Err(e) => debug!("jemalloc read malloc_conf failed: {e}"), component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_malloc_conf",
result = "ok",
malloc_conf = %conf,
"Jemalloc profiling state checked"
),
Err(e) => debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_malloc_conf",
result = "read_failed",
error = %e,
"Jemalloc profiling state checked"
),
} }
match std::env::var("MALLOC_CONF") { match std::env::var("MALLOC_CONF") {
Ok(v) => debug!("MALLOC_CONF={}", v), Ok(v) => debug!(
Err(_) => debug!("MALLOC_CONF is not set"), component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_malloc_conf_env",
state = "set",
malloc_conf = %v,
"Jemalloc profiling state checked"
),
Err(_) => debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_malloc_conf_env",
state = "unset",
"Jemalloc profiling state checked"
),
} }
if let Some(lock) = jemalloc_pprof::PROF_CTL.as_ref() { if let Some(lock) = jemalloc_pprof::PROF_CTL.as_ref() {
let ctl = lock.lock().await; let ctl = lock.lock().await;
info!(activated = ctl.activated(), "jemalloc profiling status"); info!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_profiling_status",
activated = ctl.activated(),
"Jemalloc profiling status checked"
);
} else { } else {
info!("jemalloc profiling controller is NOT available"); debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_profiling_status",
state = "unavailable",
"Jemalloc profiling status checked"
);
} }
let _ = epoch::advance(); let _ = epoch::advance();
@@ -253,7 +312,14 @@ mod linux_impl {
#[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] #[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
pub async fn check_jemalloc_profiling() { pub async fn check_jemalloc_profiling() {
debug!("jemalloc profiling status check skipped on unsupported target"); debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_JEMALLOC,
event = "jemalloc_profiling_status",
result = "skipped",
reason = "unsupported_target",
"Jemalloc profiling status checked"
);
} }
// Internal: start continuous CPU profiling // Internal: start continuous CPU profiling
@@ -263,11 +329,11 @@ mod linux_impl {
if slot.is_some() { if slot.is_some() {
warn!( warn!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state", event = "profiling_state",
profile_type = "cpu_continuous", profile_type = "cpu_continuous",
state = "already_running", state = "already_running",
"Profiling state changed" "CPU profiling already running"
); );
return; return;
} }
@@ -280,22 +346,22 @@ mod linux_impl {
*slot = Some(guard); *slot = Some(guard);
info!( info!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state", event = "profiling_state",
profile_type = "cpu_continuous", profile_type = "cpu_continuous",
state = "started", state = "started",
freq_hz, freq_hz,
"Profiling state changed" "CPU profiling started"
); );
} }
Err(e) => warn!( Err(e) => warn!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state", event = "profiling_state",
profile_type = "cpu_continuous", profile_type = "cpu_continuous",
state = "start_failed", state = "start_failed",
error = %e, error = %e,
"Profiling state changed" "CPU profiling failed to start"
), ),
} }
} }
@@ -304,14 +370,14 @@ mod linux_impl {
async fn start_cpu_periodic(freq_hz: i32, interval: Duration, duration: Duration, token: CancellationToken) { async fn start_cpu_periodic(freq_hz: i32, interval: Duration, duration: Duration, token: CancellationToken) {
info!( info!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state", event = "profiling_state",
profile_type = "cpu_periodic", profile_type = "cpu_periodic",
state = "started", state = "started",
freq_hz, freq_hz,
?interval, ?interval,
?duration, ?duration,
"Profiling state changed" "Periodic CPU profiling started"
); );
tokio::spawn(async move { tokio::spawn(async move {
loop { loop {
@@ -319,11 +385,11 @@ mod linux_impl {
_ = token.cancelled() => { _ = token.cancelled() => {
info!( info!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state", event = "profiling_state",
profile_type = "cpu_periodic", profile_type = "cpu_periodic",
state = "cancelled", state = "cancelled",
"Profiling state changed" "Periodic CPU profiling cancelled"
); );
break; break;
} }
@@ -337,7 +403,15 @@ mod linux_impl {
let guard = match pprof::ProfilerGuard::new(freq_hz) { let guard = match pprof::ProfilerGuard::new(freq_hz) {
Ok(g) => g, Ok(g) => g,
Err(e) => { Err(e) => {
warn!("periodic CPU profiler create failed: {e}"); warn!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_capture_failed",
profile_type = "cpu_periodic",
stage = "create_guard",
error = %e,
"Profiling capture failed"
);
continue; continue;
} }
}; };
@@ -346,11 +420,11 @@ mod linux_impl {
_ = token.cancelled() => { _ = token.cancelled() => {
info!( info!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_state", event = "profiling_state",
profile_type = "cpu_periodic", profile_type = "cpu_periodic",
state = "cancelled_during_capture", state = "cancelled_during_capture",
"Profiling state changed" "Periodic CPU profiling cancelled during capture"
); );
break; break;
} }
@@ -361,11 +435,20 @@ mod linux_impl {
Ok(report) => { Ok(report) => {
let out = output_dir().join(format!("cpu_profile_{}.pb", ts())); let out = output_dir().join(format!("cpu_profile_{}.pb", ts()));
if let Err(e) = write_pprof_report_pb(&report, &out) { if let Err(e) = write_pprof_report_pb(&report, &out) {
warn!("write periodic CPU pprof failed: {e}"); warn!(
} else {
info!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_dump_failed",
profile_type = "cpu_periodic",
stage = "write_dump",
path = %out.display(),
error = %e,
"Periodic CPU dump write failed"
);
} else {
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_dump_exported", event = "profiling_dump_exported",
profile_type = "cpu_periodic", profile_type = "cpu_periodic",
path = %out.display(), path = %out.display(),
@@ -373,7 +456,15 @@ mod linux_impl {
); );
} }
} }
Err(e) => warn!("periodic CPU report build failed: {e}"), Err(e) => warn!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_capture_failed",
profile_type = "cpu_periodic",
stage = "build_report",
error = %e,
"Profiling capture failed"
),
} }
} }
}); });
@@ -384,12 +475,12 @@ mod linux_impl {
async fn start_memory_periodic(interval: Duration, token: CancellationToken) { async fn start_memory_periodic(interval: Duration, token: CancellationToken) {
info!( info!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_state", event = "profiling_state",
profile_type = "memory_periodic", profile_type = "memory_periodic",
state = "started", state = "started",
?interval, ?interval,
"Profiling state changed" "Periodic memory profiling started"
); );
tokio::spawn(async move { tokio::spawn(async move {
loop { loop {
@@ -397,11 +488,11 @@ mod linux_impl {
_ = token.cancelled() => { _ = token.cancelled() => {
info!( info!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_state", event = "profiling_state",
profile_type = "memory_periodic", profile_type = "memory_periodic",
state = "cancelled", state = "cancelled",
"Profiling state changed" "Periodic memory profiling cancelled"
); );
break; break;
} }
@@ -409,29 +500,61 @@ mod linux_impl {
} }
let Some(lock) = jemalloc_pprof::PROF_CTL.as_ref() else { let Some(lock) = jemalloc_pprof::PROF_CTL.as_ref() else {
debug!("skip memory dump: PROF_CTL not available"); debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_skipped",
profile_type = "memory_periodic",
reason = "prof_ctl_unavailable",
"Profiling dump skipped"
);
continue; continue;
}; };
let mut ctl = lock.lock().await; let mut ctl = lock.lock().await;
if !ctl.activated() { if !ctl.activated() {
debug!("skip memory dump: jemalloc profiling not active"); debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_skipped",
profile_type = "memory_periodic",
reason = "jemalloc_inactive",
"Profiling dump skipped"
);
continue; continue;
} }
let out = output_dir().join(format!("mem_profile_periodic_{}.pb", ts())); let out = output_dir().join(format!("mem_profile_periodic_{}.pb", ts()));
match File::create(&out) { match File::create(&out) {
Err(e) => { Err(e) => {
tracing::error!("periodic mem dump create file failed: {}", e); tracing::error!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_failed",
profile_type = "memory_periodic",
stage = "create_file",
path = %out.display(),
error = %e,
"Periodic memory dump file creation failed"
);
continue; continue;
} }
Ok(mut f) => match ctl.dump_pprof() { Ok(mut f) => match ctl.dump_pprof() {
Ok(bytes) => { Ok(bytes) => {
if let Err(e) = f.write_all(&bytes) { if let Err(e) = f.write_all(&bytes) {
tracing::error!("periodic mem dump write failed: {}", e); tracing::error!(
} else {
info!(
component = LOG_COMPONENT_PROFILING, component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_PROFILING, subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_failed",
profile_type = "memory_periodic",
stage = "write_dump",
path = %out.display(),
error = %e,
"Periodic memory dump write failed"
);
} else {
debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_exported", event = "profiling_dump_exported",
profile_type = "memory_periodic", profile_type = "memory_periodic",
path = %out.display(), path = %out.display(),
@@ -439,7 +562,15 @@ mod linux_impl {
); );
} }
} }
Err(e) => tracing::error!("periodic mem dump failed: {}", e), Err(e) => tracing::error!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_dump_failed",
profile_type = "memory_periodic",
stage = "dump_pprof",
error = %e,
"Periodic memory dump export failed"
),
}, },
} }
} }
@@ -448,14 +579,27 @@ mod linux_impl {
#[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] #[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
async fn start_memory_periodic(_interval: Duration, _token: CancellationToken) { async fn start_memory_periodic(_interval: Duration, _token: CancellationToken) {
debug!("periodic memory profiling skipped on unsupported target"); debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_MEMORY,
event = "profiling_runtime_skipped",
profile_type = "memory_periodic",
reason = "unsupported_target",
"Profiling runtime skipped"
);
} }
// Public: unified init entry, avoid duplication/conflict // Public: unified init entry, avoid duplication/conflict
pub async fn init_from_env() { pub async fn init_from_env() {
let enabled = get_env_bool(ENV_ENABLE_PROFILING, DEFAULT_ENABLE_PROFILING); let enabled = get_env_bool(ENV_ENABLE_PROFILING, DEFAULT_ENABLE_PROFILING);
if !enabled { if !enabled {
debug!("profiling: disabled by env"); debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_RUNTIME,
event = "profiling_runtime_disabled",
reason = "env_flag",
"Profiling runtime disabled"
);
return; return;
} }
@@ -472,7 +616,14 @@ mod linux_impl {
let cpu_duration = Duration::from_secs(get_env_u64(ENV_CPU_DURATION_SECS, DEFAULT_CPU_DURATION_SECS)); let cpu_duration = Duration::from_secs(get_env_u64(ENV_CPU_DURATION_SECS, DEFAULT_CPU_DURATION_SECS));
match cpu_mode { match cpu_mode {
CpuMode::Off => debug!("profiling: CPU mode off"), CpuMode::Off => debug!(
component = LOG_COMPONENT_PROFILING,
subsystem = LOG_SUBSYSTEM_CPU,
event = "profiling_mode_selected",
profile_type = "cpu",
state = "off",
"Profiling mode selected"
),
CpuMode::Continuous => start_cpu_continuous(cpu_freq).await, CpuMode::Continuous => start_cpu_continuous(cpu_freq).await,
CpuMode::Periodic => start_cpu_periodic(cpu_freq, cpu_interval, cpu_duration, token.clone()).await, CpuMode::Periodic => start_cpu_periodic(cpu_freq, cpu_interval, cpu_duration, token.clone()).await,
} }
rustfs/src/protocols/client.rs
+2 -2
View File
@@ -207,7 +207,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
bucket, bucket,
object = %key, object = %key,
start_pos = ?start_pos, start_pos = ?start_pos,
"Protocol storage client request details" "Protocol storage client request"
); );
let mut builder = GetObjectInput::builder().bucket(bucket.to_string()).key(key.to_string()); let mut builder = GetObjectInput::builder().bucket(bucket.to_string()).key(key.to_string());
@@ -513,7 +513,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
object = %key, object = %key,
range_start = start_pos, range_start = start_pos,
range_length = length, range_length = length,
"Protocol storage client request details" "Protocol storage client request"
); );
let range = s3s::dto::Range::Int { let range = s3s::dto::Range::Int {
scripts/check_logging_guardrails.sh
+166
View File
@@ -6,15 +6,35 @@ cd "$repo_root"
checked_files=( checked_files=(
"rustfs/src/main.rs" "rustfs/src/main.rs"
"rustfs/src/init.rs"
"rustfs/src/profiling.rs"
"rustfs/src/startup_iam.rs" "rustfs/src/startup_iam.rs"
"rustfs/src/auth.rs" "rustfs/src/auth.rs"
"rustfs/src/protocols/client.rs" "rustfs/src/protocols/client.rs"
"rustfs/src/admin/router.rs"
"rustfs/src/admin/handlers/table_catalog.rs"
"rustfs/src/admin/handlers/service_account.rs"
"rustfs/src/admin/handlers/kms_dynamic.rs"
"rustfs/src/admin/handlers/site_replication.rs"
"rustfs/src/admin/handlers/group.rs"
"rustfs/src/admin/handlers/quota.rs"
"rustfs/src/admin/handlers/rebalance.rs"
"rustfs/src/admin/handlers/tier.rs"
"crates/audit/src/pipeline.rs" "crates/audit/src/pipeline.rs"
"crates/audit/src/system.rs" "crates/audit/src/system.rs"
"crates/audit/src/global.rs" "crates/audit/src/global.rs"
"crates/notify/src/config_manager.rs" "crates/notify/src/config_manager.rs"
"crates/notify/src/runtime_facade.rs" "crates/notify/src/runtime_facade.rs"
"crates/notify/src/integration.rs"
"crates/notify/src/notifier.rs" "crates/notify/src/notifier.rs"
"crates/notify/src/bucket_config_manager.rs"
"crates/notify/src/rule_engine.rs"
"crates/notify/src/global.rs"
"crates/audit/src/registry.rs"
"crates/audit/src/observability.rs"
"crates/targets/src/store.rs"
"crates/targets/src/target/mqtt.rs"
"crates/targets/src/target/webhook.rs"
"crates/ecstore/src/store/peer.rs" "crates/ecstore/src/store/peer.rs"
"crates/ecstore/src/store/init.rs" "crates/ecstore/src/store/init.rs"
"crates/ecstore/src/tier/tier.rs" "crates/ecstore/src/tier/tier.rs"
@@ -100,6 +120,25 @@ forbidden_patterns=(
'info!("Loaded notification config for bucket: {}"' 'info!("Loaded notification config for bucket: {}"'
'info!("Updated notification rules for bucket: {}"' 'info!("Updated notification rules for bucket: {}"'
'info!("Removed all notification rules for bucket: {}"' 'info!("Removed all notification rules for bucket: {}"'
'info!(event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,'
'info!("Notification system instance is being dropped"'
'info!("Notification shutdown metric snapshot"'
'info!("Notification system status snapshot"'
'debug!("Audit replay stream skipped"'
'warn!("Dropped queued audit payload"'
'error!("Queued audit payload failed permanently"'
'info!(target_id = %target_id.id, "Webhook target created")'
'info!("Webhook target {} initialized"'
'info!("Webhook sending queued payload to target: {}"'
'debug!("Event saved to store for target: {}"'
'debug!("Event sent from store and deleted for target: {}"'
'info!("Webhook target closed: {}"'
'info!(target_id = %target_id, "MQTT target created")'
'info!(target_id = %self.id, "MQTT target initialized and connected.")'
'info!(target_id = %target_id, "MQTT event loop task started.")'
'info!(target_id = %self.id, "MQTT target close method finished.")'
'debug!("Wrote event to store: {}"'
'debug!("Deleted event from store: {}"'
'info!("Audit configuration reloaded"' 'info!("Audit configuration reloaded"'
'info!("Audit system started"' 'info!("Audit system started"'
'info!("Audit metrics reset"' 'info!("Audit metrics reset"'
@@ -108,17 +147,117 @@ forbidden_patterns=(
'error!("Server encountered an error and is shutting down: {}"' 'error!("Server encountered an error and is shutting down: {}"'
'error!("Failed to initialize Keystone authentication: {}"' 'error!("Failed to initialize Keystone authentication: {}"'
'error!("new_global_notification_sys failed {:?}"' 'error!("new_global_notification_sys failed {:?}"'
'warn!(access_key = %new_cred.access_key, error = ?err, "site replication add service account hook failed")'
'warn!(access_key = %query.access_key, error = ?err, "site replication delete service account hook failed")'
'debug!("check key failed: {e:?}")'
'debug!("list service account failed: {e:?}")'
'debug!("list sts account failed: {e:?}")'
'error!("license check failed due to unexpected error: {er}")'
'info!("Created KMS key: {}"'
'info!("Successfully deleted KMS key: {}"'
'info!("Cancelled deletion for KMS key: {}"'
'info!("Listed {} KMS keys"'
'info!("Described KMS key: {}"'
'error!("Failed to create KMS key: {}"'
'error!("Failed to list KMS keys: {}"'
'error!("Failed to describe KMS key {}: {}"'
'warn!(access_key = %ak, error = ?err, "site replication create user hook failed")'
'info!("OIDC authorize redirect for provider'
'warn!("OIDC callback received error from IdP:'
'error!("OIDC code exchange failed: {}"'
'warn!("OIDC logout fallback triggered: {}"'
'warn!("get body failed, e: {:?}")'
'warn!("file path is invalid: {}")'
'warn!("bucket metadata not found: {e}")'
'warn!("get bucket metadata failed: {e}")'
'warn!("create bucket failed: {e}")'
'warn!("deserialize config failed: {e}")'
'warn!(policy = %query.name, error = ?err, "site replication policy add hook failed")'
'warn!(policy = %query.name, error = ?err, "site replication policy delete hook failed")'
'warn!(target = %query.user_or_group, error = ?err, "site replication policy mapping hook failed")'
'warn!(access_key = %ak, error = ?err, "site replication delete user hook failed")'
'warn!(target = %target_name, error = ?err, "site replication policy association hook failed")'
'error!("Failed to serialize response: {}"'
'warn!(peer = %peer.endpoint, error = ?err, "site replication peer metainfo fetch failed")'
'warn!(error = ?err, "site replication backfill: failed to list buckets")'
'warn!(bucket = %name, error = ?err, "site replication backfill: versioning setup failed")'
'warn!(bucket = %name, error = ?err, "site replication backfill: targets setup failed")'
'warn!(bucket = %name, error = ?err, "site replication backfill: replication config setup failed")'
'warn!(bucket = %name, error = ?err, "site replication backfill: failed to read bucket metadata, assuming lock_enabled=false")'
'warn!(bucket = %name, error = ?err, "site replication backfill: make-bucket broadcast failed")'
'warn!(bucket = %name, peer = %peer.endpoint, detail = %result.err_detail,'
'warn!(peer = %peer.endpoint, error = %err_detail, "site replication peer remove notification failed")'
'warn!(peer = %peer.endpoint, error = %detail, "site replication service account rotation failed for peer")'
'warn!("handle ListGroups")'
'warn!("handle GetGroup")'
'warn!("handle DeleteGroup")'
'warn!("handle SetGroupStatus")'
'warn!("handle UpdateGroupMembers")'
'warn!("list groups failed, e: {:?}")'
'warn!("get group failed, e: {:?}")'
'warn!("delete group failed, e: {:?}")'
'warn!("site replication group delete hook failed, err: {err}")'
'warn!("enable group failed, e: {:?}")'
'warn!("UpdateGroupMembers args {:?}")'
'warn!("remove group members")'
'warn!("remove group members failed, e: {:?}")'
'warn!("add group members")'
'warn!("add group members failed, e: {:?}")'
'warn!("site replication group membership hook failed, err: {err}")'
'warn!("handle SetBucketQuota")'
'warn!("handle GetBucketQuota")'
'warn!("handle ClearBucketQuota")'
'warn!("handle GetBucketQuotaStats")'
'warn!("handle CheckBucketQuota")'
'info!("Clearing quota for bucket: {}")'
'info!("Successfully cleared quota for bucket: {}")'
'debug!("Checking quota for bucket: {}, operation: {}, size: {}")'
'warn!("handle RebalanceStart")'
'warn!("Rebalance started with id: {}")'
'warn!("RebalanceStart Loading rebalance meta start")'
'warn!("rebalance start propagation failed after local state update: {err}")'
'warn!("RebalanceStart Loading rebalance meta done")'
'warn!("handle RebalanceStatus")'
'warn!("handle RebalanceStop")'
'warn!("handle RebalanceStop save_rebalance_stats done ")'
'warn!("handle RebalanceStop notification_sys load_rebalance_meta")'
'warn!("rebalance stop propagation failed after local state update: {err}")'
'warn!("handle RebalanceStop notification_sys load_rebalance_meta done")'
'warn!("get body failed, e: {:?}")'
'debug!("add tier args {:?}")'
'warn!("parse force failed, e: {:?}")'
'warn!("tier reserved name, args.name: {}")'
'warn!("tier_config_mgr reload failed, e: {:?}")'
'warn!("tier_config_mgr add failed, e: {:?}")'
'debug!("edit tier args {:?}")'
'warn!("tier_config_mgr edit failed, e: {:?}")'
'warn!("tier_config_mgr remove failed, e: {:?}")'
'warn!("tier_config_mgr rand: {}")'
'warn!("tier_config_mgr clear failed, e: {:?}")'
'warn!("tier_config_mgr save failed, e: {:?}")'
'warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
event = "object_lambda_tls_verification_disabled"'
'info!("FTP system initialized successfully"' 'info!("FTP system initialized successfully"'
'debug!("FTP system is disabled")'
'info!("FTP system disabled"' 'info!("FTP system disabled"'
'info!("FTP server shutdown completed")'
'error!("Failed to initialize FTP system: {}"' 'error!("Failed to initialize FTP system: {}"'
'info!("FTPS system initialized successfully"' 'info!("FTPS system initialized successfully"'
'debug!("FTPS system is disabled")'
'info!("FTPS system disabled"' 'info!("FTPS system disabled"'
'info!("FTPS server shutdown completed")'
'error!("Failed to initialize FTPS system: {}"' 'error!("Failed to initialize FTPS system: {}"'
'info!("WebDAV system initialized successfully"' 'info!("WebDAV system initialized successfully"'
'debug!("WebDAV system is disabled")'
'info!("WebDAV system disabled"' 'info!("WebDAV system disabled"'
'info!("WebDAV server shutdown completed")'
'error!("Failed to initialize WebDAV system: {}"' 'error!("Failed to initialize WebDAV system: {}"'
'info!("SFTP system initialized successfully"' 'info!("SFTP system initialized successfully"'
'debug!("SFTP system is disabled")'
'info!("SFTP system disabled"' 'info!("SFTP system disabled"'
'info!("SFTP server shutdown completed")'
'error!("Failed to initialize SFTP system: {}"' 'error!("Failed to initialize SFTP system: {}"'
'warn!("prewarm_local_disk_id_map: failed to load disk id for {}: {}"' 'warn!("prewarm_local_disk_id_map: failed to load disk id for {}: {}"'
'info!("retrying get formats after {:?}"' 'info!("retrying get formats after {:?}"'
@@ -174,6 +313,12 @@ forbidden_patterns=(
'warn!("KMS initialization skipped: {e}")' 'warn!("KMS initialization skipped: {e}")'
'warn!("Audit system: {e}")' 'warn!("Audit system: {e}")'
'warn!("notification system: {e}")' 'warn!("notification system: {e}")'
'info!("jemalloc profiling controller is NOT available")'
'warn!("periodic CPU profiler create failed: {e}")'
'warn!("write periodic CPU pprof failed: {e}")'
'warn!("periodic CPU report build failed: {e}")'
'debug!("profiling: disabled by env")'
'debug!("profiling: CPU mode off")'
'info!("Starting HealManager")' 'info!("Starting HealManager")'
'info!("Stopping HealManager")' 'info!("Stopping HealManager")'
'info!("HealManager started successfully")' 'info!("HealManager started successfully")'
@@ -196,6 +341,27 @@ forbidden_patterns=(
'info!("Heal queue has {} pending requests, {} tasks active")' 'info!("Heal queue has {} pending requests, {} tasks active")'
'error!("Heal channel processor failed: {}")' 'error!("Heal channel processor failed: {}")'
'info!("Heal manager with channel processor initialized successfully")' 'info!("Heal manager with channel processor initialized successfully")'
'"Heal object workflow started"'
'"Heal object recovery started"'
'"Heal object completed"'
'"Heal object cleanup completed"'
'"Heal metadata workflow started"'
'"Heal metadata completed"'
'"Heal MRF workflow started"'
'"Heal MRF completed"'
'"Heal EC decode workflow started"'
'"Heal EC decode completed"'
'"Heal erasure set workflow started"'
'"Heal erasure set format repair completed"'
'"Heal erasure set completed"'
'"Heal storage repair finished"'
'"Heal storage admin operation finished"'
'"Heal storage request finished"'
'"Erasure set object heal completed"'
'"Erasure set heal resumed from checkpoint"'
'"Scanner disk bucket state updated"'
'"Scanner lifecycle action evaluation started"'
'"Scanner state persisted"'
'"scanner deep heal downgraded to normal during new-object cooldown"' '"scanner deep heal downgraded to normal during new-object cooldown"'
'"scanner detected folder with excessive direct subfolders"' '"scanner detected folder with excessive direct subfolders"'
'"scan_folder: failed to get size for item {}: {}"' '"scan_folder: failed to get size for item {}: {}"'