From e8012bd1ba071184745a29f9a93c5f05b3700fb1 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 14 Jun 2026 13:27:10 +0800 Subject: [PATCH] refactor(logging): normalize admin telemetry and error messages (#3430) --- Cargo.lock | 239 ++++---- Cargo.toml | 12 +- crates/audit/src/observability.rs | 6 +- crates/audit/src/pipeline.rs | 26 +- crates/audit/src/registry.rs | 6 +- crates/audit/src/system.rs | 16 +- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 18 +- .../ecstore/src/cache_value/metacache_set.rs | 98 +++- crates/ecstore/src/disk/local.rs | 32 +- crates/ecstore/src/pools.rs | 366 ++++++++++-- crates/ecstore/src/rebalance.rs | 206 ++++++- crates/ecstore/src/rpc/remote_disk.rs | 400 ++++++++++++-- crates/ecstore/src/set_disk.rs | 120 +++- crates/heal/src/heal/channel.rs | 34 +- crates/heal/src/heal/erasure_healer.rs | 54 +- crates/heal/src/heal/manager.rs | 92 ++-- crates/heal/src/heal/resume.rs | 10 +- crates/heal/src/heal/storage.rs | 66 +-- crates/heal/src/heal/task.rs | 102 ++-- crates/iam/src/lib.rs | 90 ++- crates/iam/src/manager.rs | 24 +- crates/iam/src/oidc.rs | 10 +- crates/iam/src/store/object.rs | 64 ++- crates/kms/src/backends/local.rs | 24 +- crates/kms/src/backends/vault.rs | 26 +- crates/kms/src/service.rs | 26 +- crates/kms/src/service_manager.rs | 72 ++- crates/notify/src/bucket_config_manager.rs | 8 +- crates/notify/src/config_manager.rs | 30 +- crates/notify/src/global.rs | 11 +- crates/notify/src/integration.rs | 6 +- crates/notify/src/notifier.rs | 12 +- crates/notify/src/rule_engine.rs | 4 +- crates/notify/src/runtime_facade.rs | 10 +- crates/obs/src/telemetry/dial9.rs | 4 +- crates/obs/src/telemetry/local.rs | 6 +- crates/protocols/src/ftps/driver.rs | 10 +- crates/protocols/src/ftps/server.rs | 48 +- crates/protocols/src/sftp/driver.rs | 4 +- crates/protocols/src/sftp/server.rs | 16 +- crates/protocols/src/sftp/write.rs | 30 +- crates/protocols/src/webdav/driver.rs | 28 +- crates/protocols/src/webdav/server.rs | 22 +- crates/scanner/src/scanner.rs | 48 +- crates/scanner/src/scanner_folder.rs | 12 +- crates/scanner/src/scanner_io.rs | 28 +- crates/targets/src/store.rs | 124 ++++- crates/targets/src/target/mqtt.rs | 520 ++++++++++++++++-- crates/targets/src/target/webhook.rs | 142 ++++- rustfs/src/admin/handlers/bucket_meta.rs | 236 ++++++-- rustfs/src/admin/handlers/group.rs | 252 +++++++-- rustfs/src/admin/handlers/kms_dynamic.rs | 132 +++-- rustfs/src/admin/handlers/kms_keys.rs | 194 +++++-- rustfs/src/admin/handlers/oidc.rs | 59 +- rustfs/src/admin/handlers/policies.rs | 172 ++++-- rustfs/src/admin/handlers/quota.rs | 122 +++- rustfs/src/admin/handlers/rebalance.rs | 153 ++++-- rustfs/src/admin/handlers/service_account.rs | 82 ++- rustfs/src/admin/handlers/site_replication.rs | 106 +++- .../src/admin/handlers/target_descriptor.rs | 70 +-- rustfs/src/admin/handlers/tier.rs | 262 +++++++-- rustfs/src/admin/handlers/user.rs | 130 +++-- rustfs/src/admin/router.rs | 59 +- rustfs/src/app/object_usecase.rs | 64 +-- rustfs/src/auth.rs | 26 +- rustfs/src/init.rs | 244 +++++++- rustfs/src/main.rs | 106 ++-- rustfs/src/profiling.rs | 251 +++++++-- rustfs/src/protocols/client.rs | 4 +- scripts/check_logging_guardrails.sh | 166 ++++++ 70 files changed, 4807 insertions(+), 1445 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b83cf02ca..a68d54e70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,9 +35,9 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "aead" -version = "0.6.0-rc.10" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b657e772794c6b04730ea897b66a058ccd866c16d1967da05eeeecec39043fe" +checksum = "ef60ac202874e574ce7a7158cc8bca7313dd344322482e4fadee288bf4a306b8" dependencies = [ "crypto-common 0.2.2", "inout 0.2.2", @@ -277,9 +277,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "ar_archive_writer" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" dependencies = [ "object 0.37.3", ] @@ -887,9 +887,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.4" +version = "1.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ed8e8c52d2dc2390ad9f15647fe663f71e9780b4262c190fbb823a32721566" +checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -915,9 +915,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.135.0" +version = "1.136.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f97e3e7e7d86fd26fcdc18bc382da5ca9e8b2ff8d54030d187fd0dac8a236d96" +checksum = "fbd03e531a7d981fba45114c5813a7565dd470a1bd3ef1188ca98c98ebdfc668" dependencies = [ "arc-swap", "aws-credential-types", @@ -951,9 +951,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.101.0" +version = "1.102.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b647baea49ff551960b904f905681e9b4765a6c4ea08631e89dc52d8bd3f5896" +checksum = "8c82b3ac19f1431854f7ace3a7531674633e286bfdde21976893bfee36fd493b" dependencies = [ "arc-swap", "aws-credential-types", @@ -976,9 +976,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.103.0" +version = "1.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ae401c65ff288aa7873117fe535cd32b7b1bb0bc43751d28901a1d5f20636b9" +checksum = "321000d2b4c5519ee573f73167f612efd7329322d9b26969ad1979f0427f1913" dependencies = [ "arc-swap", "aws-credential-types", @@ -1001,9 +1001,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.106.0" +version = "1.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c80de7bb7d03e9ca8c9fd7b489f20f3948d3f3be91a7953591347d238115408" +checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560" dependencies = [ "arc-swap", "aws-credential-types", @@ -1086,9 +1086,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.20" +version = "0.60.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf09d74e5e32f76b8762da505a3cd59303e367a664ca67295387baa8c1d7548" +checksum = "78d8391e65fcea47c586a22e1a41f173b38615b112b2c6b7a44e80cec3e6b706" dependencies = [ "aws-smithy-types", "bytes", @@ -1509,9 +1509,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ "hybrid-array", "zeroize", @@ -1560,9 +1560,9 @@ dependencies = [ [[package]] name = "bon" -version = "3.9.1" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" +checksum = "b2f04f6fef12d70d42a77b1433c9e0f065238479a6cefc4f5bab105e9873a3c3" dependencies = [ "bon-macros", "rustversion", @@ -1570,9 +1570,9 @@ dependencies = [ [[package]] name = "bon-macros" -version = "3.9.1" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" +checksum = "7d0bd4c2f75335ad98052a37efb54f428b492f64340257143b3429c8a508fa7b" dependencies = [ "darling 0.23.0", "ident_case", @@ -1661,9 +1661,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" +checksum = "49e78e506b9d7633710dab98996f22f95f3d0f488e8f1aa162830556ed9fc14d" [[package]] name = "bytestring" @@ -1742,9 +1742,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.63" +version = "1.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" dependencies = [ "find-msvc-tools", "jobserver", @@ -1856,7 +1856,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ - "block-buffer 0.12.0", + "block-buffer 0.12.1", "crypto-common 0.2.2", "inout 0.2.2", "zeroize", @@ -2348,12 +2348,11 @@ dependencies = [ [[package]] name = "crypto-primes" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21f41f23de7d24cdbda7f0c4d9c0351f99a4ceb258ef30e5c1927af8987ffe5a" +checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f" dependencies = [ "crypto-bigint 0.7.3", - "libm", "rand_core 0.10.1", ] @@ -3380,7 +3379,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -3551,7 +3549,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer 0.12.0", + "block-buffer 0.12.1", "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", @@ -3576,7 +3574,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3912,7 +3910,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4294,11 +4292,10 @@ dependencies = [ [[package]] name = "getset" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912" +checksum = "6cf442baaabe4213ce7d1239afc26c039180b6456da2cededa316ae2c8a77a77" dependencies = [ - "proc-macro-error2", "proc-macro2", "quote", "syn 2.0.117", @@ -5345,7 +5342,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5512,13 +5509,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -5978,6 +5974,29 @@ dependencies = [ "zerocopy-derive", ] +[[package]] +name = "manyhow" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" +dependencies = [ + "manyhow-macros", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "manyhow-macros" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + [[package]] name = "mappings" version = "0.7.2" @@ -6040,9 +6059,9 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" @@ -6104,9 +6123,9 @@ dependencies = [ [[package]] name = "metrique" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2638fb6c2325d3e35c7b0d1ce49cf8ce74b89b7439d4dc5ce1c9fff0f7fa18a1" +checksum = "ffdfd0c9954a916c5e1db230cc7593569858afec7744cebfd0fbef45de7b07c7" dependencies = [ "itoa", "jiff", @@ -6124,9 +6143,9 @@ dependencies = [ [[package]] name = "metrique-core" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b99ea3484ab9afe6337413bacc842d516e65623a476699f9873803028e6a9e" +checksum = "f56ef332b47a4d2dc14209bf9ce4334682dd2fec43a4b84e5ed063bf2f4ea29a" dependencies = [ "itertools 0.14.0", "metrique-writer-core", @@ -6134,9 +6153,9 @@ dependencies = [ [[package]] name = "metrique-macro" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a02c5cb6e55c1ec5eb3ef2d10d137805547c486ad54b587d42d0a49713e6ef9" +checksum = "403c61cd847b11680ae6cf1feb384cbe06e412e4dc221f1e16ccf48038d190bb" dependencies = [ "Inflector", "darling 0.23.0", @@ -6147,9 +6166,9 @@ dependencies = [ [[package]] name = "metrique-service-metrics" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb71714973a9ba53d609a577311aae4d8beb7df37869b5a92cb777a3f4c905b8" +checksum = "93045b7f5c8b998946b715de07523b1c6d81932512c5c27590bf73328aaf3832" dependencies = [ "metrique-writer", ] @@ -6162,9 +6181,9 @@ checksum = "d607939211e4eaaa8cd35394fa5e57faffb7390d0ac513b39992edcaf3cc526c" [[package]] name = "metrique-writer" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f770fe8a45a7c2f285c7ce81b9364e0e5604cbb7daf693fef1f624bf8f0bf8" +checksum = "ae9e23184d81616fcdfcdc5768e14c2bca765096094d08e7895259fe658378e4" dependencies = [ "ahash 0.8.12", "crossbeam-queue", @@ -6183,9 +6202,9 @@ dependencies = [ [[package]] name = "metrique-writer-core" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a0b67a72a8849987122ca8ceacaf81d47eaba36cd673767fc3742a086dd04ac" +checksum = "23421d85733c0182613b32f32094080e07cd5e98be5893f1cd3baf7eed16c4d0" dependencies = [ "derive-where", "itertools 0.14.0", @@ -6337,15 +6356,15 @@ checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" [[package]] name = "mysql-common-derive" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66f62cad7623a9cb6f8f64037f0c4f69c8db8e82914334a83c9788201c2c1bfa" +checksum = "a4db8a44120571277accfaa3f3d91e7d3989d601d817c2fc01a9391b86135666" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "heck", + "manyhow", "num-bigint", "proc-macro-crate", - "proc-macro-error2", "proc-macro2", "quote", "syn 2.0.117", @@ -6571,7 +6590,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6743,7 +6762,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "chrono", "getrandom 0.2.17", "http 1.4.2", @@ -7798,9 +7817,9 @@ dependencies = [ [[package]] name = "primefield" -version = "0.14.0-rc.10" +version = "0.14.0-rc.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f845ec3240cd5ed5e1e31cf3ff633a5bf47c698dc4092ba9e767415b3d393406" +checksum = "b1d7e42f46a29abc16fb621a3466ee453358ebaae48a9e515f287e0af052ed8f" dependencies = [ "crypto-bigint 0.7.3", "crypto-common 0.2.2", @@ -7838,25 +7857,14 @@ dependencies = [ ] [[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" +name = "proc-macro-utils" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" dependencies = [ "proc-macro2", "quote", -] - -[[package]] -name = "proc-macro-error2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" -dependencies = [ - "proc-macro-error-attr2", - "proc-macro2", - "quote", - "syn 2.0.117", + "smallvec", ] [[package]] @@ -8480,9 +8488,9 @@ dependencies = [ [[package]] name = "redis" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6b5f4d8ef33944e833e2b1859ad478deab6e431d7337b30ee2efe21f7543" +checksum = "f9fd510128eda94d1d49b9f81487744d5c451422431cce41238fe2853d29f4cc" dependencies = [ "arc-swap", "arcstr", @@ -10216,7 +10224,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10290,7 +10298,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11177,9 +11185,9 @@ dependencies = [ [[package]] name = "starshard" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "472e5a677707be0fbe7bd851ca25ed7e7757e83dc07a05447cdb6fbdd27db5e2" +checksum = "4dafd0cbefb050fa7a7f95ac229f7978ed4c99866a1a5b62e34a154b52d73d77" dependencies = [ "async-trait", "hashbrown 0.17.1", @@ -11441,10 +11449,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11582,12 +11590,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" dependencies = [ "deranged", - "itoa", "js-sys", "libc", "num-conv", @@ -11600,15 +11607,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" dependencies = [ "num-conv", "time-core", @@ -12372,9 +12379,9 @@ dependencies = [ [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen 0.57.1", ] @@ -12399,9 +12406,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ "cfg-if", "once_cell", @@ -12412,9 +12419,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.72" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" dependencies = [ "js-sys", "wasm-bindgen", @@ -12422,9 +12429,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -12432,9 +12439,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ "bumpalo", "proc-macro2", @@ -12445,9 +12452,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ "unicode-ident", ] @@ -12501,9 +12508,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.99" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ "js-sys", "wasm-bindgen", @@ -12608,7 +12615,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -13067,18 +13074,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", @@ -13108,18 +13115,18 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index eb1b4d980..aff67c87e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,7 +159,7 @@ tower-http = { version = "0.6.11", features = ["cors"] } # Serialization and Data Formats bytes = { version = "1.11.1", features = ["serde"] } -bytesize = "2.3.1" +bytesize = "2.4.0" byteorder = "1.5.0" flatbuffers = "25.12.19" form_urlencoded = "1.2.2" @@ -188,13 +188,13 @@ rustls-pki-types = "1.14.1" sha1 = "0.11.0" sha2 = "0.11.0" subtle = "2.6" -zeroize = { version = "1.8.2", features = ["derive"] } +zeroize = { version = "1.9.0", features = ["derive"] } # Time and Date chrono = { version = "0.4.45", features = ["serde"] } humantime = "2.3.0" jiff = { version = "0.2.28", features = ["serde"] } -time = { version = "=0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] } +time = { version = "0.3.49", features = ["std", "parsing", "formatting", "macros", "serde"] } # Database deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] } @@ -209,7 +209,7 @@ atoi = "3.0.0" atomic_enum = "0.3.0" aws-config = { version = "1.8.18" } aws-credential-types = { version = "1.2.14" } -aws-sdk-s3 = { version = "1.135.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } +aws-sdk-s3 = { version = "1.136.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } aws-smithy-http-client = { version = "1.1.13", default-features = false, features = ["default-client", "rustls-aws-lc"] } aws-smithy-types = { version = "1.5.0" } base64 = "0.22.1" @@ -264,7 +264,7 @@ reed-solomon-erasure = { version = "6.0", default-features = false, features = [ reed-solomon-simd = "3.1.0" regex = { version = "1.12.4" } rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket"] } -redis = { version = "1.2.2", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] } +redis = { version = "1.2.3", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] } rustix = { version = "1.1.4", features = ["fs"] } rust-embed = { version = "8.11.0" } rustc-hash = { version = "2.1.2" } @@ -276,7 +276,7 @@ smallvec = { version = "1.15.2", features = ["serde"] } smartstring = "1.0.1" snafu = "0.9.1" snap = "1.1.1" -starshard = { version = "2.2.0", features = ["rayon", "async", "serde"] } +starshard = { version = "2.2.1", features = ["rayon", "async", "serde"] } strum = { version = "0.28.0", features = ["derive"] } sysinfo = "0.39.3" temp-env = "0.3.6" diff --git a/crates/audit/src/observability.rs b/crates/audit/src/observability.rs index 0fecc05ae..16e15d8bb 100644 --- a/crates/audit/src/observability.rs +++ b/crates/audit/src/observability.rs @@ -160,7 +160,7 @@ impl AuditMetrics { component = LOG_COMPONENT_AUDIT, subsystem = LOG_SUBSYSTEM_OBSERVABILITY, state = "config_reloaded", - "Audit observability state updated" + "audit observability state" ); } @@ -173,7 +173,7 @@ impl AuditMetrics { component = LOG_COMPONENT_AUDIT, subsystem = LOG_SUBSYSTEM_OBSERVABILITY, state = "system_started", - "Audit observability state updated" + "audit observability state" ); } @@ -249,7 +249,7 @@ impl AuditMetrics { component = LOG_COMPONENT_AUDIT, subsystem = LOG_SUBSYSTEM_OBSERVABILITY, state = "metrics_reset", - "Audit observability state updated" + "audit observability state" ); } diff --git a/crates/audit/src/pipeline.rs b/crates/audit/src/pipeline.rs index 6b7f195b0..84bc01f31 100644 --- a/crates/audit/src/pipeline.rs +++ b/crates/audit/src/pipeline.rs @@ -266,7 +266,7 @@ impl AuditRuntimeView { subsystem = LOG_SUBSYSTEM_PIPELINE, target_id = %target_id, state = "enabled", - "Changed audit target state" + "audit target state" ); Ok(()) } else { @@ -283,7 +283,7 @@ impl AuditRuntimeView { subsystem = LOG_SUBSYSTEM_PIPELINE, target_id = %target_id, state = "disabled", - "Changed audit target state" + "audit target state" ); Ok(()) } else { @@ -300,7 +300,7 @@ impl AuditRuntimeView { subsystem = LOG_SUBSYSTEM_PIPELINE, target_id = %target_id, state = "removed", - "Changed audit target state" + "audit target state" ); Ok(()) } else { @@ -323,7 +323,7 @@ impl AuditRuntimeView { subsystem = LOG_SUBSYSTEM_PIPELINE, target_id = %target_id, state = "upserted", - "Changed audit target state" + "audit target state" ); Ok(()) } @@ -349,7 +349,7 @@ impl AuditRuntimeFacade { subsystem = LOG_SUBSYSTEM_PIPELINE, target_id = %target.id(), replay_key = %key, - "Delivered queued audit event" + "audit replay delivery" ); observability::record_target_success(); } @@ -361,7 +361,7 @@ impl AuditRuntimeFacade { subsystem = LOG_SUBSYSTEM_PIPELINE, target_id = %target.id(), reason = "not_connected", - "Retrying queued audit event delivery" + "audit replay delivery" ); } rustfs_targets::TargetError::Timeout(_) => { @@ -371,7 +371,7 @@ impl AuditRuntimeFacade { subsystem = LOG_SUBSYSTEM_PIPELINE, target_id = %target.id(), reason = "timeout", - "Retrying queued audit event delivery" + "audit replay delivery" ); } _ => {} @@ -383,7 +383,7 @@ impl AuditRuntimeFacade { subsystem = LOG_SUBSYSTEM_PIPELINE, target_id = %target.id(), reason = %reason, - "Dropped queued audit payload" + "audit replay delivery" ); observability::record_target_failure(); } @@ -395,7 +395,7 @@ impl AuditRuntimeFacade { target_id = %target.id(), error = %error, reason = "permanent_failure", - "Queued audit payload failed permanently" + "audit replay delivery" ); target.record_final_failure(); observability::record_target_failure(); @@ -408,7 +408,7 @@ impl AuditRuntimeFacade { target_id = %target.id(), replay_key = %key, reason = "retry_exhausted", - "Dropped queued audit payload after retry exhaustion" + "audit replay delivery" ); target.record_final_failure(); observability::record_target_failure(); @@ -422,7 +422,7 @@ impl AuditRuntimeFacade { replay_key = %key, error = %error, reason = "unreadable_entry", - "Skipped unreadable audit store entry" + "audit replay delivery" ); } } @@ -436,7 +436,7 @@ impl AuditRuntimeFacade { subsystem = LOG_SUBSYSTEM_PIPELINE, target_id = %target_id, replay_enabled = true, - "Audit replay stream started" + "audit replay stream" ); } else { debug!( @@ -446,7 +446,7 @@ impl AuditRuntimeFacade { target_id = %target_id, replay_enabled = false, reason = "no_store_configured", - "Audit replay stream skipped" + "audit replay stream" ); } }), diff --git a/crates/audit/src/registry.rs b/crates/audit/src/registry.rs index 7da7e0a1d..30581b603 100644 --- a/crates/audit/src/registry.rs +++ b/crates/audit/src/registry.rs @@ -198,7 +198,7 @@ impl AuditRegistry { target_type = %target_type, target_id = %target_id, registry_key = %key, - "Created audit target registry key" + "audit target registry state" ); key.to_string() } @@ -221,7 +221,7 @@ impl AuditRegistry { target_type = %target_type, target_id = %target_id, state = "enabled", - "Audit target registry state changed" + "audit target registry state" ); Ok(()) } else { @@ -250,7 +250,7 @@ impl AuditRegistry { target_type = %target_type, target_id = %target_id, state = "disabled", - "Audit target registry state changed" + "audit target registry state" ); Ok(()) } else { diff --git a/crates/audit/src/system.rs b/crates/audit/src/system.rs index 6deca7586..065bbc151 100644 --- a/crates/audit/src/system.rs +++ b/crates/audit/src/system.rs @@ -120,7 +120,7 @@ impl AuditSystem { subsystem = LOG_SUBSYSTEM_SYSTEM, state = "targets_created", target_count = targets.len(), - "Created audit targets" + "audit system state" ); let activation = self.runtime_facade().activate_targets_with_replay(targets).await; @@ -159,7 +159,7 @@ impl AuditSystem { component = LOG_COMPONENT_AUDIT, subsystem = LOG_SUBSYSTEM_SYSTEM, state = "starting", - "Starting audit system" + "audit system state" ); // Record system start @@ -268,7 +268,7 @@ impl AuditSystem { component = LOG_COMPONENT_AUDIT, subsystem = LOG_SUBSYSTEM_SYSTEM, state = "stopping", - "Stopping audit system" + "audit system state" ); // Stop all stream tasks first @@ -441,7 +441,7 @@ impl AuditSystem { component = LOG_COMPONENT_AUDIT, subsystem = LOG_SUBSYSTEM_SYSTEM, state = "reloading", - "Reloading audit configuration" + "audit config reload" ); observability::record_config_reload(); @@ -465,7 +465,7 @@ impl AuditSystem { component = LOG_COMPONENT_AUDIT, subsystem = LOG_SUBSYSTEM_SYSTEM, state = "reloaded", - "Reloaded audit configuration" + "audit config reload" ); Ok(()) } @@ -513,7 +513,7 @@ fn info_audit_state(state: &str, reason: Option<&str>, target_count: Option, error: Option<&str>, tar reason = reason.unwrap_or_default(), error = error.unwrap_or_default(), target_count, - "Observed audit system state" + "audit system state" ); } @@ -537,7 +537,7 @@ fn warn_audit_state(state: &str, reason: Option<&str>) { subsystem = LOG_SUBSYSTEM_SYSTEM, state, reason = reason.unwrap_or_default(), - "Audit system state transition skipped" + "audit system state" ); } diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index b22b862f6..8405e0ba4 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -77,7 +77,7 @@ use tokio::sync::mpsc::{Receiver, Sender}; use tokio::sync::{RwLock, mpsc}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, warn}; use uuid::Uuid; use xxhash_rust::xxh64; @@ -762,13 +762,13 @@ impl TransitionState { "Transition compensation backfill failed" ); } else { - info!( + debug!( event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, bucket = %bucket, state = "completed", - "Completed transition compensation backfill" + "Transition compensation completed" ); } @@ -958,7 +958,7 @@ impl TransitionState { pub async fn init(api: Arc) { let (configured, absolute_max, n) = resolve_transition_worker_count(); - info!( + debug!( event = EVENT_LIFECYCLE_WORKER_STATE, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, @@ -968,7 +968,7 @@ impl TransitionState { transition_queue_capacity = GLOBAL_TransitionState.transition_queue_capacity, transition_queue_send_timeout_ms = GLOBAL_TransitionState.transition_queue_send_timeout.as_millis() as u64, state = "configured", - "Lifecycle worker state resolved" + "Lifecycle worker configuration resolved" ); //let mut transition_state = GLOBAL_TransitionState.write().await; @@ -1158,7 +1158,7 @@ impl TransitionState { GLOBAL_TransitionState.num_workers.store(current_workers, Ordering::SeqCst); GLOBAL_TransitionState.record_scanner_transition_state(); - info!( + debug!( event = EVENT_LIFECYCLE_WORKER_STATE, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, @@ -1169,7 +1169,7 @@ impl TransitionState { current_transition_workers = current_workers, pruned_finished_transition_workers = pruned_finished_workers, state = "resized", - "Lifecycle worker state updated" + "Lifecycle worker pool resized" ); } } @@ -2008,14 +2008,14 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn { Arc::new(move || { let tier = tier.clone(); Box::pin(async move { - info!( + debug!( event = EVENT_LIFECYCLE_TIER_AUDIT, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, tier = %tier, bytes = bytes, state = "transition_completed", - "Lifecycle tier transition audit completed" + "Lifecycle tier transition recorded" ); }) }) diff --git a/crates/ecstore/src/cache_value/metacache_set.rs b/crates/ecstore/src/cache_value/metacache_set.rs index 20e3cfef0..ca2c4ec5a 100644 --- a/crates/ecstore/src/cache_value/metacache_set.rs +++ b/crates/ecstore/src/cache_value/metacache_set.rs @@ -29,7 +29,11 @@ use tokio::io::AsyncRead; use tokio::spawn; use tokio::time::timeout; use tokio_util::sync::CancellationToken; -use tracing::{error, info, warn}; +use tracing::{error, warn}; + +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_METACACHE: &str = "metacache"; +const EVENT_METACACHE_LISTING: &str = "metacache_listing"; pub type AgreedFn = Box Pin + Send>> + Send + 'static>; pub type PartialFn = @@ -114,6 +118,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d return Err(DiskError::ErasureReadQuorum); } + let log_bucket = opts.bucket.clone(); + let log_path = opts.path.clone(); + let mut jobs: Vec>> = Vec::new(); let mut readers = Vec::with_capacity(opts.disks.len()); let fds = opts.fallback_disks.iter().flatten().cloned().collect::>(); @@ -180,7 +187,17 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d match disk.walk_dir(wakl_opts, &mut wr).await { Ok(_res) => {} Err(err) => { - info!("walk dir err {:?}", &err); + warn!( + event = EVENT_METACACHE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_METACACHE, + bucket = %opts_clone.bucket, + path = %opts_clone.path, + disk_index = disk_idx, + state = "walk_dir_failed", + error = ?err, + "Metacache walk_dir failed" + ); last_err = Some(err); need_fallback = true; } @@ -205,7 +222,16 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d } let Some(disk) = disk_op else { - warn!("list_path_raw: fallback disk is none"); + warn!( + event = EVENT_METACACHE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_METACACHE, + bucket = %opts_clone.bucket, + path = %opts_clone.path, + disk_index = disk_idx, + state = "fallback_disk_missing", + "Metacache fallback disk missing" + ); let err = last_err.unwrap_or(DiskError::DiskNotFound); record_producer_error(&producer_errs_clone, disk_idx, &err); return Err(err); @@ -234,7 +260,17 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d last_err = None; } Err(err) => { - error!("walk dir2 err {:?}", &err); + error!( + event = EVENT_METACACHE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_METACACHE, + bucket = %opts_clone.bucket, + path = %opts_clone.path, + disk_index = disk_idx, + state = "fallback_walk_dir_failed", + error = ?err, + "Metacache fallback walk_dir failed" + ); last_err = Some(err); } } @@ -355,11 +391,15 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d ) .increment(1); warn!( + event = EVENT_METACACHE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_METACACHE, drive = %endpoint, bucket = %opts.bucket, path = %opts.path, timeout_ms = peek_timeout.as_millis(), - "list_path_raw reader peek timed out; excluding drive from current merge" + state = "peek_timed_out", + "Metacache reader peek timed out" ); let (detached_rd, write_half) = tokio::io::duplex(1); drop(write_half); @@ -439,8 +479,14 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d }); error!( - "list_path_raw: has_err > 0 && has_err > opts.disks.len() - opts.min_disks break, err: {:?}", - &combined_err.join(", ") + event = EVENT_METACACHE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_METACACHE, + bucket = %opts.bucket, + path = %opts.path, + state = "quorum_failed", + error = %combined_err.join(", "), + "Metacache listing quorum failed" ); return Err(DiskError::other(combined_err.join(", "))); } @@ -493,7 +539,16 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d }); if let Err(err) = revjob.await.map_err(std::io::Error::other)? { - error!("list_path_raw: revjob err {:?}", err); + error!( + event = EVENT_METACACHE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_METACACHE, + bucket = %log_bucket, + path = %log_path, + state = "merge_job_failed", + error = ?err, + "Metacache merge job failed" + ); cancel_rx.cancel(); for job in jobs { job.abort(); @@ -521,9 +576,23 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d Ok(Ok(())) => {} Ok(Err(err)) => { if matches!(err, DiskError::FileNotFound | DiskError::VolumeNotFound) { - warn!("list_path_raw producer missing path {:?}", err); + warn!( + event = EVENT_METACACHE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_METACACHE, + state = "producer_missing_path", + error = ?err, + "Metacache producer missing path" + ); } else { - error!("list_path_raw producer err {:?}", err); + error!( + event = EVENT_METACACHE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_METACACHE, + state = "producer_failed", + error = ?err, + "Metacache producer failed" + ); } job_errs.push(err); } @@ -531,7 +600,14 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d if err.is_cancelled() { continue; } - error!("list_path_raw join err {:?}", err); + error!( + event = EVENT_METACACHE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_METACACHE, + state = "producer_join_failed", + error = ?err, + "Metacache producer join failed" + ); job_errs.push(err.into()); } } diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index f12337fef..ee25f5f59 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -472,7 +472,15 @@ fn resolve_local_disk_root(ep_path: &str) -> Result { impl LocalDisk { pub async fn new(ep: &Endpoint, cleanup: bool) -> Result { - debug!("Creating local disk"); + debug!( + event = EVENT_DISK_LOCAL_STARTUP_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + endpoint = %ep, + state = "create_started", + cleanup, + "Local disk creation started" + ); let endpoint_path = ep.get_file_path(); let root = resolve_local_disk_root(&endpoint_path).inspect_err(|err| { log_startup_disk_error("resolve_local_disk_root", Path::new(&endpoint_path), err); @@ -501,13 +509,21 @@ impl LocalDisk { root = ?root, state = "failed", error = ?err, - "Disk local startup cleanup failed" + "Local disk startup cleanup failed" ); } // Use optimized path resolution instead of absolutize_virtually let format_path = root.join(RUSTFS_META_BUCKET).join(super::FORMAT_CONFIG_FILE); - debug!("format_path: {:?}", format_path); + debug!( + event = EVENT_DISK_LOCAL_STARTUP_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + root = ?root, + format_path = ?format_path, + state = "format_path_resolved", + "Local disk format path resolved" + ); let (format_data, format_meta) = read_file_exists(&format_path).await.inspect_err(|err| { log_startup_disk_error("read_format_json", &format_path, err); })?; @@ -639,7 +655,15 @@ impl LocalDisk { let root = disk.root.clone(); tokio::spawn(Self::cleanup_deleted_objects_loop(root, exit_rx)); - debug!("LocalDisk created: {:?}", disk); + debug!( + event = EVENT_DISK_LOCAL_STARTUP_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + endpoint = %disk.endpoint, + root = ?disk.root, + state = "created", + "Local disk created" + ); Ok(disk) } diff --git a/crates/ecstore/src/pools.rs b/crates/ecstore/src/pools.rs index e17cdf5f8..33745b7a5 100644 --- a/crates/ecstore/src/pools.rs +++ b/crates/ecstore/src/pools.rs @@ -71,6 +71,12 @@ use time::{Duration, OffsetDateTime}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_POOLS: &str = "pools"; +const EVENT_DECOMMISSION_STATE: &str = "decommission_state"; +const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket"; +const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry"; + pub const POOL_META_NAME: &str = "pool.bin"; pub const POOL_META_FORMAT: u16 = 1; pub const POOL_META_VERSION: u16 = 1; @@ -1280,7 +1286,15 @@ impl ECStore { take_decommission_canceler(cancelers.as_mut_slice(), idx) }; if !cancel_decommission_canceler(canceler) { - warn!("decommission_cancel: no active canceler found for pool {}", idx); + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "cancel_skipped", + reason = "no_active_canceler", + "Decommission cancel skipped" + ); } if should_reload_pool_meta && let Some(notification_sys) = get_global_notification_sys() { @@ -1338,7 +1352,15 @@ impl ECStore { let store = store.clone(); tokio::spawn(async move { if let Err(err) = store.do_decommission_in_routine(canceler, idx).await { - error!("decommission: routine failed for idx {}: {err}", idx); + error!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "routine_failed", + error = %err, + "Decommission routine failed" + ); } }); } @@ -1350,7 +1372,14 @@ impl ECStore { pub async fn decommission(&self, rx: CancellationToken, indices: Vec) -> Result<()> { let indices = dedup_indices(&indices); - warn!("decommission: {:?}", indices); + info!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_indices = ?indices, + state = "requested", + "Decommission requested" + ); validate_start_decommission_request(&indices, self.single_pool())?; ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?; @@ -1363,8 +1392,13 @@ impl ECStore { for idx in indices { if let Err(cancel_err) = self.decommission_cancel(idx).await { error!( - "decommission: failed to rollback decommission state for idx {} after spawn error: {:?}", - idx, cancel_err + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "rollback_failed", + error = ?cancel_err, + "Decommission rollback failed after spawn error" ); if rollback_err.is_none() { rollback_err = Some(Error::other(format!("decommission rollback failed for idx {idx}: {cancel_err}"))); @@ -1390,10 +1424,28 @@ impl ECStore { lock_retention: Option, replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, ) -> Result<()> { - warn!("decommission_entry: {} {}", &bucket, &entry.name); + debug!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %entry.name, + state = "started", + "Decommission entry started" + ); wk.give().await; if entry.is_dir() { - warn!("decommission_entry: skip dir {}", &entry.name); + debug!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %entry.name, + state = "skipped_directory", + "Decommission entry skipped directory" + ); return Ok(()); } @@ -1426,7 +1478,16 @@ impl ECStore { if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) { // decommissioned += 1; - info!("decommission_pool: DELETE marked object with no other non-current versions will be skipped"); + debug!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %version.name, + state = "skipped_delete_marker", + "Decommission delete marker skipped" + ); continue; } @@ -1447,8 +1508,16 @@ impl ECStore { { if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { warn!( - "decommission_pool: ignore delete-marker copy for {}/{} version {:?}: {:?}", - &bucket, &version.name, &version_id, &err + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %version.name, + version_id = ?version_id, + state = "ignored_delete_marker_copy", + error = ?err, + "Decommission delete marker copy ignored" ); ignore = true; cleanup_ignored = true; @@ -1463,7 +1532,16 @@ impl ECStore { if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) { decommissioned += 1; } - info!("decommission_pool: ignore {}", &version.name); + debug!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %version.name, + state = "ignored", + "Decommission entry ignored" + ); continue; } @@ -1483,9 +1561,17 @@ impl ECStore { decommissioned += 1; } - info!( - "decommission_pool: DecomCopyDeleteMarker {} {} {:?} {:?}", - &bucket, &version.name, &version_id, error + debug!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %version.name, + version_id = ?version_id, + result = ?error, + state = "delete_marker_copied", + "Decommission delete marker copied" ); continue; } @@ -1571,8 +1657,15 @@ impl ECStore { } warn!( - "decommission_pool: decommission_object done {}/{} {}", - &bucket_name, &object_name, &version.name + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket_name, + object = %object_name, + version = %version.name, + state = "object_migrated", + "Decommission object migrated" ); failure = false; @@ -1583,7 +1676,16 @@ impl ECStore { if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) { decommissioned += 1; } - info!("decommission_pool: ignore {}", &version.name); + debug!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %version.name, + state = "ignored", + "Decommission entry ignored" + ); continue; } @@ -1624,12 +1726,17 @@ impl ECStore { resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())? } else if decommissioned != fivs.versions.len() || expired > 0 { warn!( - "decommission_pool: source object retained for {}/{} because only {}/{} versions were decommissioned and {} expired by lifecycle", - &bucket, - &entry.name, + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %entry.name, decommissioned, - fivs.versions.len(), - expired + total_versions = fivs.versions.len(), + expired, + state = "source_retained", + "Decommission source object retained" ); } @@ -1667,7 +1774,16 @@ impl ECStore { } } - warn!("decommission_pool: decommission_entry done {} {}", &bucket, &entry.name); + debug!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %entry.name, + state = "completed", + "Decommission entry completed" + ); Ok(()) } @@ -1705,7 +1821,16 @@ impl ECStore { for (set_idx, set) in pool.disk_set.iter().enumerate() { wk.clone().take().await; - warn!("decommission_pool: decommission_pool {} {}", set_idx, &bi.name); + debug!( + event = EVENT_DECOMMISSION_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_idx, + bucket = %bi.name, + state = "listing_worker_started", + "Decommission listing worker started" + ); let decommission_entry: ListCallback = Arc::new({ let this = Arc::clone(self); @@ -1753,23 +1878,69 @@ impl ECStore { let worker = tokio::spawn(async move { loop { if rx_clone.is_cancelled() { - warn!("decommission_pool: cancel {}", set_id); + debug!( + event = EVENT_DECOMMISSION_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_id, + bucket = %bi.name, + state = "listing_worker_cancelled", + "Decommission listing worker cancelled" + ); break; } - warn!("decommission_pool: list_objects_to_decommission {} {}", set_id, &bi.name); + debug!( + event = EVENT_DECOMMISSION_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_id, + bucket = %bi.name, + state = "listing_started", + "Decommission listing started" + ); match set .list_objects_to_decommission(rx_clone.clone(), bi.clone(), decommission_entry.clone()) .await { Ok(_) => { - warn!("decommission_pool: list_objects_to_decommission {} done", set_id); + debug!( + event = EVENT_DECOMMISSION_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_id, + bucket = %bi.name, + state = "listing_completed", + "Decommission listing completed" + ); break; } Err(err) => { - error!("decommission_pool: list_objects_to_decommission {} err {:?}", set_id, &err); + error!( + event = EVENT_DECOMMISSION_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_id, + bucket = %bi.name, + state = "listing_failed", + error = ?err, + "Decommission listing failed" + ); if is_err_bucket_not_found(&err) { - warn!("decommission_pool: list_objects_to_decommission {} volume not found", set_id); + warn!( + event = EVENT_DECOMMISSION_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_id, + bucket = %bi.name, + state = "listing_bucket_missing", + "Decommission listing bucket missing" + ); break; } @@ -1783,7 +1954,15 @@ impl ECStore { listing_workers.push((set_id, worker)); } - warn!("decommission_pool: decommission_pool wait {} {}", idx, &bi.name); + debug!( + event = EVENT_DECOMMISSION_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bi.name, + state = "waiting_for_workers", + "Decommission waiting for workers" + ); let mut listing_worker_error = None; for (set_id, worker) in listing_workers { @@ -1807,11 +1986,28 @@ impl ECStore { } if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) { - warn!("decommission_pool: canceled after wait {} {}", idx, &bi.name); + warn!( + event = EVENT_DECOMMISSION_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bi.name, + state = "cancelled_after_wait", + error = %err, + "Decommission bucket cancelled after wait" + ); return Err(err); } - warn!("decommission_pool: decommission_pool done {} {}", idx, &bi.name); + debug!( + event = EVENT_DECOMMISSION_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bi.name, + state = "completed", + "Decommission bucket completed" + ); Ok(()) } @@ -1821,7 +2017,14 @@ impl ECStore { defer!(|| async { let mut cancelers = self.decommission_cancelers.write().await; if take_decommission_canceler(cancelers.as_mut_slice(), idx).is_none() { - warn!("decommission: canceler already cleared for pool {}", idx); + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "canceler_already_cleared", + "Decommission canceler already cleared" + ); } }); @@ -1830,7 +2033,14 @@ impl ECStore { let (final_state, canceled, cmd_line) = { let pool_meta = self.pool_meta.read().await; let Some(pool) = pool_meta.pools.get(idx) else { - error!("decommission: pool metadata missing for idx {}", idx); + error!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "pool_metadata_missing", + "Decommission pool metadata missing" + ); return Err(Error::other(format!( "failed to resolve decommission final state: pool metadata missing for idx {idx}" ))); @@ -1849,29 +2059,75 @@ impl ECStore { }; if let Err(err) = result { - error!("decom err {:?}", &err); + error!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "background_failed", + error = ?err, + "Decommission background routine failed" + ); if is_err_operation_canceled(&err) || should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) { - warn!("decommission: canceled for pool {}, preserving canceled state", cmd_line); + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + cmd_line = %cmd_line, + state = "cancelled_preserved", + "Decommission cancelled; preserving canceled state" + ); return Ok(()); } resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?; - warn!("decommission: decommission_failed {}", idx); + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "marked_failed", + "Decommission marked failed" + ); return Ok(()); } - warn!("decommission: decommission_in_background complete {}", idx); + debug!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "background_complete", + "Decommission background routine completed" + ); if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) { - warn!("decommission: canceled for pool {}, skipping terminal state overwrite", cmd_line); + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + cmd_line = %cmd_line, + state = "terminal_state_preserved", + "Decommission terminal state preserved after cancellation" + ); return Ok(()); } match final_state { DecommissionFinalState::Complete => { - warn!("Decommissioning complete for pool {}, verifying for any pending objects", cmd_line); + debug!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + cmd_line = %cmd_line, + state = "verifying_completion", + "Decommission completion verification started" + ); if let Err(err) = self.check_after_decommission(idx).await { resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?; return Err(Error::other(format!( @@ -1879,16 +2135,40 @@ impl ECStore { ))); } - warn!("Decommissioning complete for pool {}, marking completed state", cmd_line); + info!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + cmd_line = %cmd_line, + state = "marking_completed", + "Decommission marking completed state" + ); resolve_decommission_terminal_mark_result(self.complete_decommission(idx).await, "completed", &cmd_line)?; } DecommissionFinalState::Failed => { - warn!("Decommissioning finished with failed items for pool {}, marking failed state", cmd_line); + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + cmd_line = %cmd_line, + state = "marking_failed", + "Decommission marking failed state" + ); resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?; } } - warn!("Decommissioning complete for pool {}", cmd_line); + info!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + cmd_line = %cmd_line, + state = "completed", + "Decommission completed" + ); Ok(()) } diff --git a/crates/ecstore/src/rebalance.rs b/crates/ecstore/src/rebalance.rs index 0404bcc2d..dc8c9dedd 100644 --- a/crates/ecstore/src/rebalance.rs +++ b/crates/ecstore/src/rebalance.rs @@ -858,7 +858,14 @@ impl ECStore { "init_rebalance_meta", )?; - info!("init_rebalance_meta: rebalance meta saved"); + info!( + event = EVENT_REBALANCE_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + state = "metadata_initialized", + bucket_count = bucktes.len(), + "Rebalance metadata initialized" + ); let id = meta.id.clone(); @@ -898,9 +905,16 @@ impl ECStore { #[tracing::instrument(skip(self))] pub async fn next_rebal_bucket(&self, pool_index: usize) -> Result> { - info!("next_rebal_bucket: pool_index: {}", pool_index); let rebalance_meta = self.rebalance_meta.read().await; - info!("next_rebal_bucket: rebalance_meta: {:?}", rebalance_meta); + debug!( + event = EVENT_REBALANCE_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index, + has_meta = rebalance_meta.is_some(), + state = "next_bucket_lookup", + "Rebalance next bucket lookup" + ); resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index) } @@ -931,20 +945,38 @@ impl ECStore { let rebalance_meta = self.rebalance_meta.read().await; if let Some(meta) = rebalance_meta.as_ref() { meta.pool_stats.iter().enumerate().for_each(|(i, v)| { - info!( - "is_rebalance_started: pool_index: {}, participating: {:?}, status: {:?}", - i, v.participating, v.info.status + debug!( + event = EVENT_REBALANCE_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index = i, + participating = v.participating, + status = ?v.info.status, + state = "status_inspected", + "Rebalance status inspected" ); }); let started = is_rebalance_conflicting_with_decommission(meta); if started { - info!("is_rebalance_started: rebalance started"); + debug!( + event = EVENT_REBALANCE_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + state = "running", + "Rebalance is running" + ); return true; } } - info!("is_rebalance_started: rebalance not started"); + debug!( + event = EVENT_REBALANCE_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + state = "not_running", + "Rebalance is not running" + ); false } @@ -1108,15 +1140,23 @@ impl ECStore { workers_started += 1; tokio::spawn(async move { if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx).await { - error!("Rebalance failed for pool {}: {}", pool_idx, err); + error!( + event = EVENT_REBALANCE_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index = pool_idx, + state = "pool_failed", + error = %err, + "Rebalance pool failed" + ); } else { - info!( + debug!( event = EVENT_REBALANCE_STATE, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_REBALANCE, pool_index = pool_idx, state = "completed", - "Completed rebalance pool" + "Rebalance pool completed" ); } }); @@ -1139,7 +1179,8 @@ impl ECStore { component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_REBALANCE, state = "started", - "Started rebalance" + worker_count = workers_started, + "Rebalance started" ); Ok(()) } @@ -1240,7 +1281,7 @@ impl ECStore { subsystem = LOG_SUBSYSTEM_REBALANCE, pool_index, state = "pool_started", - "Started rebalance worker" + "Rebalance worker started" ); let mut final_result: Result<()> = Ok(()); let mut deferred_buckets = HashSet::new(); @@ -1267,7 +1308,15 @@ impl ECStore { let next_bucket = match self.next_rebal_bucket(pool_index).await { Ok(bucket) => bucket, Err(err) => { - error!("next_rebal_bucket failed for pool {}: {:?}", pool_index, err); + error!( + event = EVENT_REBALANCE_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index, + state = "next_bucket_failed", + error = ?err, + "Rebalance next bucket lookup failed" + ); final_result = Err(resolve_rebalance_terminal_error( err.clone(), send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, @@ -1294,7 +1343,16 @@ impl ECStore { ) { Ok(outcome) => outcome, Err(err) => { - error!("Error rebalancing bucket {}: {:?}", bucket, err); + error!( + event = EVENT_REBALANCE_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index, + bucket = %bucket, + state = "bucket_failed", + error = ?err, + "Rebalance bucket failed" + ); final_result = Err(resolve_rebalance_terminal_error( err.clone(), send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, @@ -1308,7 +1366,16 @@ impl ECStore { let err = Error::other(format!( "rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}" )); - error!("Error rebalancing bucket {}: {:?}", bucket, err); + error!( + event = EVENT_REBALANCE_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index, + bucket = %bucket, + state = "bucket_deferred_repeatedly", + error = ?err, + "Rebalance bucket failed after repeated deferral" + ); final_result = Err(resolve_rebalance_terminal_error( err.clone(), send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, @@ -1327,7 +1394,16 @@ impl ECStore { "Deferred rebalance bucket after transient object failures" ); if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await { - error!("defer_rebalance_bucket failed for pool {}: {:?}", pool_index, err); + error!( + event = EVENT_REBALANCE_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index, + bucket = %bucket, + state = "defer_failed", + error = ?err, + "Rebalance bucket defer failed" + ); final_result = Err(resolve_rebalance_terminal_error( err.clone(), send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, @@ -1347,7 +1423,15 @@ impl ECStore { "Completed rebalance bucket" ); if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await { - error!("bucket_rebalance_done failed for pool {}: {:?}", pool_index, err); + error!( + event = EVENT_REBALANCE_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index, + state = "bucket_done_mark_failed", + error = ?err, + "Rebalance bucket completion mark failed" + ); final_result = Err(resolve_rebalance_terminal_error( err.clone(), send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await, @@ -1374,7 +1458,7 @@ impl ECStore { subsystem = LOG_SUBSYSTEM_REBALANCE, pool_index, state = "pool_done", - "Finished rebalance worker" + "Rebalance worker finished" ); if final_result.is_ok() @@ -1388,7 +1472,14 @@ impl ECStore { { final_result = Err(err); } - info!("Pool {} rebalancing is done2", pool_index); + debug!( + event = EVENT_REBALANCE_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index, + state = "pool_result_returned", + "Rebalance worker result returned" + ); final_result } @@ -2591,7 +2682,15 @@ impl ECStore { ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?; // Placeholder for actual bucket rebalance logic - info!("Rebalancing bucket {} in pool {}", bucket, pool_index); + debug!( + event = EVENT_REBALANCE_BUCKET, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index, + bucket = %bucket, + state = "entry_scan_started", + "Rebalance bucket entry scan started" + ); // TODO: other config // if bucket != RUSTFS_META_BUCKET{ @@ -2765,9 +2864,14 @@ impl ECStore { let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?; - info!( - "save_rebalance_stats: save rebalance meta, pool_idx: {}, opt: {:?}, meta: {:?}", - pool_idx, opt, meta_to_save + debug!( + event = EVENT_REBALANCE_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index = pool_idx, + save_opt = ?opt, + state = "metadata_save_requested", + "Rebalance metadata save requested" ); let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}"); resolve_rebalance_meta_save_result( @@ -2838,12 +2942,27 @@ impl SetDisks { bucket: String, cb: ListCallback, ) -> Result<()> { - info!("list_objects_to_rebalance: start list_objects_to_rebalance"); + debug!( + event = EVENT_REBALANCE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + bucket = %bucket, + state = "started", + "Rebalance listing started" + ); // Placeholder for actual object listing logic let (disks, _) = self.get_online_disks_with_healing(false).await; ensure_rebalance_listing_disks_available(!disks.is_empty(), &bucket)?; - info!("list_objects_to_rebalance: get online disks with healing"); + debug!( + event = EVENT_REBALANCE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + bucket = %bucket, + disk_count = disks.len(), + state = "disks_resolved", + "Rebalance listing disks resolved" + ); let listing_quorum = self.set_drive_count.div_ceil(2); let resolver = MetadataResolutionParams { @@ -2863,7 +2982,14 @@ impl SetDisks { min_disks: listing_quorum, skip_walkdir_total_timeout: true, agreed: Some(Box::new(move |entry: MetaCacheEntry| { - info!("list_objects_to_rebalance: agreed: {:?}", &entry.name); + debug!( + event = EVENT_REBALANCE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + entry = %entry.name, + state = "agreed_entry", + "Rebalance listing agreed entry" + ); Box::pin(cb1(entry)) })), partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { @@ -2873,11 +2999,24 @@ impl SetDisks { match entries.resolve(resolver) { Some(entry) => { - info!("list_objects_to_rebalance: list_objects_to_decommission get {}", &entry.name); + debug!( + event = EVENT_REBALANCE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + entry = %entry.name, + state = "resolved_partial_entry", + "Rebalance listing resolved partial entry" + ); Box::pin(async move { cb(entry).await }) } None => { - info!("list_objects_to_rebalance: list_objects_to_decommission get none"); + debug!( + event = EVENT_REBALANCE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + state = "partial_entry_missing", + "Rebalance listing partial entry missing" + ); Box::pin(async {}) } } @@ -2887,7 +3026,14 @@ impl SetDisks { ) .await?; - info!("list_objects_to_rebalance: list_objects_to_rebalance done"); + debug!( + event = EVENT_REBALANCE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + bucket = %bucket, + state = "completed", + "Rebalance listing completed" + ); Ok(()) } } diff --git a/crates/ecstore/src/rpc/remote_disk.rs b/crates/ecstore/src/rpc/remote_disk.rs index e676f600b..45f8ecb99 100644 --- a/crates/ecstore/src/rpc/remote_disk.rs +++ b/crates/ecstore/src/rpc/remote_disk.rs @@ -65,7 +65,7 @@ use tokio::{ }; use tokio_util::sync::CancellationToken; use tonic::{Request, service::interceptor::InterceptedService, transport::Channel}; -use tracing::{debug, info, warn}; +use tracing::{debug, warn}; use uuid::Uuid; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -76,6 +76,10 @@ enum FailureHealthAction { const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2; const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20); +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk"; +const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health"; +const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc"; async fn copy_stream_with_buffer(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result where @@ -275,7 +279,16 @@ impl RemoteDisk { if initial_probe_ok { health.record_operation_success(&endpoint, "connectivity_probe_success"); } else if health.mark_failure(&endpoint, "connectivity_probe_failed") { - warn!("Remote disk health check failed for {}: marking as faulty", addr); + warn!( + event = EVENT_REMOTE_DISK_HEALTH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %endpoint, + addr, + state = "initial_probe_failed", + result = "mark_faulty", + "Remote disk initial health probe failed" + ); // Start recovery monitoring let health_clone = Arc::clone(&health); @@ -291,7 +304,15 @@ impl RemoteDisk { loop { tokio::select! { _ = cancel_token.cancelled() => { - debug!("Health monitoring cancelled for remote disk: {}", addr); + debug!( + event = EVENT_REMOTE_DISK_HEALTH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %endpoint, + addr, + state = "monitor_cancelled", + "Remote disk health monitor cancelled" + ); return; } _ = interval.tick() => { @@ -320,7 +341,16 @@ impl RemoteDisk { if Self::perform_connectivity_check(&addr).await.is_ok() { health.record_operation_success(&endpoint, "connectivity_probe_success"); } else if health.mark_failure(&endpoint, "connectivity_probe_failed") { - warn!("Remote disk health check failed for {}: marking as faulty", addr); + warn!( + event = EVENT_REMOTE_DISK_HEALTH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %endpoint, + addr, + state = "probe_failed", + result = "mark_faulty", + "Remote disk health probe failed" + ); // Start recovery monitoring let health_clone = Arc::clone(&health); @@ -354,9 +384,25 @@ impl RemoteDisk { _ = interval.tick() => { if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() { let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success"); - info!("Remote disk recovery probe succeeded: {}", addr); + debug!( + event = EVENT_REMOTE_DISK_HEALTH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %endpoint, + addr, + state = "recovery_probe_succeeded", + "Remote disk recovery probe succeeded" + ); if became_online { - info!("Remote disk recovered: {}", addr); + debug!( + event = EVENT_REMOTE_DISK_HEALTH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %endpoint, + addr, + state = "recovered", + "Remote disk recovered" + ); return; } } else { @@ -469,7 +515,16 @@ impl RemoteDisk { { // Check if disk is faulty if self.health.is_faulty() { - warn!("remote disk {} health is faulty, returning error", self.to_string()); + warn!( + event = EVENT_REMOTE_DISK_HEALTH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + addr = %self.addr, + op, + state = "faulty_short_circuit", + "Remote disk operation short-circuited by faulty state" + ); return Err(DiskError::FaultyDisk); } @@ -519,10 +574,14 @@ impl RemoteDisk { self.mark_faulty_and_evict("operation_timeout").await; } warn!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, endpoint = %self.endpoint, addr = %self.addr, op, timeout_ms = timeout_duration.as_millis(), + state = "timeout", "Remote disk operation timed out" ); Err(DiskError::Timeout) @@ -547,11 +606,15 @@ impl RemoteDisk { ) .increment(1); warn!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, endpoint = %self.endpoint, addr = %self.addr, op, timeout_ms = timeout_duration.as_millis(), - "Remote disk operation returned a network-like error" + state = "network_like_error", + "Remote disk operation returned network-like error" ); if failure_health_action == FailureHealthAction::MarkFailure { self.mark_faulty_and_evict("operation_network_error").await; @@ -574,13 +637,26 @@ impl RemoteDisk { .increment(1); if transitioned_to_offline { warn!( - "Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}", - self.endpoint, self.addr, reason + event = EVENT_REMOTE_DISK_HEALTH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + addr = %self.addr, + reason, + state = "marked_faulty", + "Remote disk marked faulty" ); } else { warn!( - "Remote disk marked suspect after timeout: endpoint={}, addr={}, reason={}, state={:?}", - self.endpoint, self.addr, reason, state + event = EVENT_REMOTE_DISK_HEALTH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + addr = %self.addr, + reason, + runtime_state = ?state, + state = "marked_suspect", + "Remote disk marked suspect" ); } counter!( @@ -589,11 +665,15 @@ impl RemoteDisk { "reason" => reason.to_string() ) .increment(1); - info!( + debug!( + event = EVENT_REMOTE_DISK_HEALTH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, endpoint = %self.endpoint, addr = %self.addr, reason, - "Evicting cached remote disk connection after fault transition" + state = "evict_cached_connection", + "Remote disk cached connection evicted" ); evict_failed_connection(&self.addr).await; } @@ -705,7 +785,16 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn make_volume(&self, volume: &str) -> Result<()> { - info!("make_volume"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + op = "make_volume", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -733,7 +822,16 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { - info!("make_volumes"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume_count = volumes.len(), + op = "make_volumes", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -761,7 +859,15 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn list_volumes(&self) -> Result> { - info!("list_volumes"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + op = "list_volumes", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -794,7 +900,16 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn stat_volume(&self, volume: &str) -> Result { - info!("stat_volume"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + op = "stat_volume", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -824,7 +939,16 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn delete_volume(&self, volume: &str) -> Result<()> { - info!("delete_volume {}/{}", self.endpoint.to_string(), volume); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + op = "delete_volume", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -909,7 +1033,17 @@ impl DiskAPI for RemoteDisk { force_del_marker: bool, opts: DeleteOptions, ) -> Result<()> { - info!("delete_version"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path, + op = "delete_version", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -946,7 +1080,17 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) -> Vec> { - info!("delete_versions"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + version_count = versions.len(), + op = "delete_versions", + state = "started", + "Remote disk RPC started" + ); if self.health.is_faulty() { return vec![Some(DiskError::FaultyDisk); versions.len()]; @@ -1041,7 +1185,17 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { - info!("delete_paths"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path_count = paths.len(), + op = "delete_paths", + state = "started", + "Remote disk RPC started" + ); let paths = paths.to_owned(); self.execute_with_timeout( @@ -1071,7 +1225,17 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { - info!("write_metadata {}/{}", volume, path); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path, + op = "write_metadata", + state = "started", + "Remote disk RPC started" + ); let file_info = serde_json::to_string(&fi)?; let file_info_bin = encode_msgpack(&fi)?; @@ -1134,7 +1298,17 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> { - info!("update_metadata"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path, + op = "update_metadata", + state = "started", + "Remote disk RPC started" + ); let file_info = serde_json::to_string(&fi)?; let opts_str = serde_json::to_string(&opts)?; let file_info_bin = encode_msgpack(&fi)?; @@ -1180,7 +1354,18 @@ impl DiskAPI for RemoteDisk { version_id: &str, opts: &ReadOptions, ) -> Result { - info!("read_version"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path, + version_id, + op = "read_version", + state = "started", + "Remote disk RPC started" + ); let opts_str = serde_json::to_string(opts)?; let opts_bin = encode_msgpack(opts)?; @@ -1217,7 +1402,18 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(level = "debug", skip(self))] async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { - info!("read_xl {}/{}/{}", self.endpoint.to_string(), volume, path); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path, + read_data, + op = "read_xl", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -1257,7 +1453,19 @@ impl DiskAPI for RemoteDisk { dst_volume: &str, dst_path: &str, ) -> Result { - info!("rename_data {}/{}/{}/{}", self.addr, self.endpoint.to_string(), dst_volume, dst_path); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + src_volume, + src_path, + dst_volume, + dst_path, + op = "rename_data", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout_for_op( "rename_data", @@ -1325,7 +1533,17 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self, wr))] async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> { - info!("walk_dir {}", self.endpoint.to_string()); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + bucket = %opts.bucket, + base_dir = %opts.base_dir, + op = "walk_dir", + state = "started", + "Remote disk RPC started" + ); let disk = self.disk_ref().await; let body = serde_json::to_vec(&opts)?; @@ -1443,7 +1661,17 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(level = "debug", skip(self))] async fn append_file(&self, volume: &str, path: &str) -> Result { - info!("append_file {}/{}", volume, path); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path, + op = "append_file", + state = "started", + "Remote disk RPC started" + ); if self.health.is_faulty() { return Err(DiskError::FaultyDisk); @@ -1487,7 +1715,19 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(level = "debug", skip(self))] async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { - info!("rename_file"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + src_volume, + src_path, + dst_volume, + dst_path, + op = "rename_file", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -1518,7 +1758,19 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> { - info!("rename_part {}/{}", src_volume, src_path); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + src_volume, + src_path, + dst_volume, + dst_path, + op = "rename_part", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -1550,7 +1802,19 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { - info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path, + recursive = opt.recursive, + immediate = opt.immediate, + op = "delete", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -1581,7 +1845,17 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - info!("verify_file"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path, + op = "verify_file", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -1614,6 +1888,17 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result> { + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + bucket, + path_count = paths.len(), + op = "read_parts", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { let mut client = self @@ -1642,7 +1927,17 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - info!("check_parts"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path, + op = "check_parts", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -1675,7 +1970,17 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn read_multiple(&self, req: ReadMultipleReq) -> Result> { - info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + bucket = %req.bucket, + prefix = %req.prefix, + op = "read_multiple", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -1721,7 +2026,18 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> { - info!("write_all"); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path, + bytes = data.len(), + op = "write_all", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { @@ -1779,7 +2095,17 @@ impl DiskAPI for RemoteDisk { #[tracing::instrument(skip(self))] async fn read_all(&self, volume: &str, path: &str) -> Result { - info!("read_all {}/{}", volume, path); + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + volume, + path, + op = "read_all", + state = "started", + "Remote disk RPC started" + ); self.execute_with_timeout( || async { diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index b1143d094..ae07ca9d7 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -130,6 +130,12 @@ use tracing::error; use tracing::{debug, info, warn}; use uuid::Uuid; +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_SET_DISK: &str = "set_disk"; +const EVENT_SET_DISK_MULTIPART: &str = "set_disk_multipart"; +const EVENT_SET_DISK_WRITE: &str = "set_disk_write"; +const EVENT_SET_DISK_HEAL: &str = "set_disk_heal"; + use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _}; pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024; @@ -948,7 +954,16 @@ impl ObjectIO for SetDisks { ) .await { - error!("get_object_with_fileinfo {bucket}/{object} err {:?}", e); + error!( + event = EVENT_SET_DISK_WRITE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + state = "read_pipeline_failed", + error = ?e, + "Set disk object read pipeline failed" + ); }; }); @@ -1065,7 +1080,15 @@ impl ObjectIO for SetDisks { { Ok(writer) => (Some(writer), None), Err(err) => { - warn!("create_bitrot_writer disk {}, err {:?}, skipping operation", disk.to_string(), err); + warn!( + event = EVENT_SET_DISK_WRITE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + disk = ?disk, + state = "bitrot_writer_skipped", + error = ?err, + "Set disk bitrot writer skipped" + ); (None, Some(err)) } } @@ -1085,7 +1108,18 @@ impl ObjectIO for SetDisks { let nil_count = errors.iter().filter(|&e| e.is_none()).count(); if nil_count < write_quorum { - error!("not enough disks to write: {:?}", errors); + error!( + event = EVENT_SET_DISK_WRITE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + write_quorum, + available_writers = nil_count, + state = "write_quorum_unavailable", + error = ?errors, + "Set disk write quorum unavailable" + ); if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) { return Err(to_object_err(write_err.into(), vec![bucket, object])); } @@ -1136,7 +1170,17 @@ impl ObjectIO for SetDisks { // } if (w_size as i64) < data.size() { - warn!("put_object write size < data.size(), w_size={}, data.size={}", w_size, data.size()); + warn!( + event = EVENT_SET_DISK_WRITE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + written_size = w_size, + expected_size = data.size(), + state = "short_write", + "Set disk write produced fewer bytes than expected" + ); return Err(Error::other(format!( "put_object write size < data.size(), w_size={}, data.size={}", w_size, @@ -2976,7 +3020,15 @@ impl MultipartOperations for SetDisks { { Ok(writer) => writer, Err(err) => { - warn!("create_bitrot_writer disk {}, err {:?}, skipping operation", disk.to_string(), err); + warn!( + event = EVENT_SET_DISK_MULTIPART, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + disk = ?disk, + state = "bitrot_writer_skipped", + error = ?err, + "Set disk multipart bitrot writer skipped" + ); errors.push(Some(err)); writers.push(None); continue; @@ -3021,7 +3073,18 @@ impl MultipartOperations for SetDisks { let _ = mem::replace(&mut data.stream, reader); if (w_size as i64) < data.size() { - warn!("put_object_part write size < data.size(), w_size={}, data.size={}", w_size, data.size()); + warn!( + event = EVENT_SET_DISK_MULTIPART, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + part_number = part_id, + written_size = w_size, + expected_size = data.size(), + state = "short_write", + "Set disk multipart write produced fewer bytes than expected" + ); return Err(Error::other(format!( "put_object_part write size < data.size(), w_size={}, data.size={}", w_size, @@ -3657,7 +3720,17 @@ impl MultipartOperations for SetDisks { ); return Err(Error::InvalidPart(p.part_num, "".to_owned(), p.etag.clone().unwrap_or_default())); }; - info!(target:"rustfs_ecstore::set_disk", part_number = p.part_num, part_size = ext_part.size, part_actual_size = ext_part.actual_size, "Completing multipart part"); + debug!( + target:"rustfs_ecstore::set_disk", + event = EVENT_SET_DISK_MULTIPART, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + part_number = p.part_num, + part_size = ext_part.size, + part_actual_size = ext_part.actual_size, + state = "part_validated", + "Set disk multipart part validated" + ); // Normalize ETags by removing quotes before comparison (PR #592 compatibility) let client_etag = p.etag.as_ref().map(|e| rustfs_utils::path::trim_etag(e)); @@ -3988,7 +4061,16 @@ impl HealOperations for SetDisks { let disks = disks.clone(); let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false).await?; if DiskError::is_all_not_found(&errs) { - debug!(bucket, object, version_id, "heal_object skipped missing object"); + debug!( + event = EVENT_SET_DISK_HEAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + version_id, + state = "missing_object_skipped", + "Set disk heal skipped missing object" + ); let err = if !version_id.is_empty() { Error::FileVersionNotFound } else { @@ -4356,7 +4438,16 @@ async fn disks_with_all_parts( verify_resp = v; } Err(err) => { - info!("verify_file failed: {err:?}, object_name={}, index: {index}", object_name); + debug!( + event = EVENT_SET_DISK_HEAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + object = %object_name, + disk_index = index, + state = "verify_failed", + error = ?err, + "Set disk verify_file failed" + ); verify_err = Some(err); } } @@ -4366,7 +4457,16 @@ async fn disks_with_all_parts( verify_resp = v; } Err(err) => { - info!("check_parts failed: {err:?}, object_name={}, index: {index}", object_name); + debug!( + event = EVENT_SET_DISK_HEAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + object = %object_name, + disk_index = index, + state = "check_parts_failed", + error = ?err, + "Set disk check_parts failed" + ); verify_err = Some(err); } } diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 3c3e3fe35..54a13a6a3 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -69,7 +69,7 @@ impl HealChannelProcessor { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_CHANNEL, state = "started", - "Heal channel processor state updated" + "Heal channel started" ); loop { @@ -85,7 +85,7 @@ impl HealChannelProcessor { subsystem = LOG_SUBSYSTEM_CHANNEL, state = "process_failed", error = %e, - "Heal channel request processing failed" + "Heal channel processing failed" ); } } @@ -96,7 +96,7 @@ impl HealChannelProcessor { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_CHANNEL, state = "receiver_closed", - "Heal channel processor state updated" + "Heal channel receiver closed" ); break; } @@ -105,7 +105,7 @@ impl HealChannelProcessor { response = self.response_receiver.recv() => { if let Some(response) = response { // Handle response if needed - info!( + debug!( target: "rustfs::heal::channel", event = EVENT_HEAL_CHANNEL_RESPONSE, component = LOG_COMPONENT_HEAL, @@ -113,7 +113,7 @@ impl HealChannelProcessor { request_id = %response.request_id, success = response.success, state = "received_local", - "Heal channel response observed" + "Heal response observed" ); } } @@ -126,7 +126,7 @@ impl HealChannelProcessor { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_CHANNEL, state = "stopped", - "Heal channel processor state updated" + "Heal channel stopped" ); Ok(()) } @@ -154,7 +154,7 @@ impl HealChannelProcessor { request: HealChannelRequest, response_tx: oneshot::Sender>, ) -> Result<()> { - info!( + debug!( target: "rustfs::heal::channel", event = EVENT_HEAL_CHANNEL_REQUEST, component = LOG_COMPONENT_HEAL, @@ -163,7 +163,7 @@ impl HealChannelProcessor { bucket = %request.bucket, object_prefix = %request.object_prefix.as_deref().unwrap_or(""), state = "start_received", - "Heal channel start request received" + "Heal start received" ); // Convert channel request to heal request @@ -185,7 +185,7 @@ impl HealChannelProcessor { // Submit to heal manager match self.heal_manager.submit_heal_request(heal_request).await { Ok(admission) => { - info!( + debug!( target: "rustfs::heal::channel", event = EVENT_HEAL_CHANNEL_REQUEST, component = LOG_COMPONENT_HEAL, @@ -193,7 +193,7 @@ impl HealChannelProcessor { request_id = %request.id, admission = admission.result_label(), state = "admission_decided", - "Heal channel admission decision completed" + "Heal admission decided" ); let _ = response_tx.send(Ok(admission)); @@ -225,7 +225,7 @@ impl HealChannelProcessor { request_id = %request.id, state = "submit_failed", error = %error_text, - "Heal channel start request failed" + "Heal start submission failed" ); let _ = response_tx.send(Err(error_text.clone())); @@ -251,7 +251,7 @@ impl HealChannelProcessor { client_token: String, response_tx: oneshot::Sender>, ) -> Result<()> { - info!( + debug!( target: "rustfs::heal::channel", event = EVENT_HEAL_CHANNEL_REQUEST, component = LOG_COMPONENT_HEAL, @@ -259,7 +259,7 @@ impl HealChannelProcessor { request_id = %client_token, heal_path = %heal_path, state = "query_received", - "Heal channel query request received" + "Heal query received" ); let (summary, detail, items) = match self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await { @@ -332,7 +332,7 @@ impl HealChannelProcessor { client_token: String, response_tx: oneshot::Sender>, ) -> Result<()> { - info!( + debug!( target: "rustfs::heal::channel", event = EVENT_HEAL_CHANNEL_REQUEST, component = LOG_COMPONENT_HEAL, @@ -340,7 +340,7 @@ impl HealChannelProcessor { request_id = %client_token, heal_path = %heal_path, state = "cancel_received", - "Heal channel cancel request received" + "Heal cancel received" ); let request_id = if client_token.is_empty() { @@ -451,7 +451,7 @@ impl HealChannelProcessor { request_id = %response.request_id, state = "enqueue_local_failed", error = %e, - "Heal channel response enqueue failed" + "Heal response local enqueue failed" ); } // Always attempt to broadcast, even if local send failed @@ -464,7 +464,7 @@ impl HealChannelProcessor { subsystem = LOG_SUBSYSTEM_CHANNEL, state = "broadcast_failed", error = %e, - "Heal channel response broadcast failed" + "Heal response broadcast failed" ); } } diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 975784570..ac3ccd58a 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -27,7 +27,7 @@ use std::sync::{ atomic::{AtomicUsize, Ordering}, }; use tokio::sync::{RwLock, Semaphore}; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_ERASURE_HEALER: &str = "erasure_healer"; @@ -99,7 +99,7 @@ impl ErasureSetHealer { /// execute erasure set heal with resume #[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))] pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> { - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_RESUME_STATE, component = LOG_COMPONENT_HEAL, @@ -162,7 +162,7 @@ impl ErasureSetHealer { Ok(manager) => { let state = manager.get_state().await; if state.set_disk_id == set_disk_id && ResumeUtils::can_resume_task(&self.disk, &task_id).await { - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_RESUME_STATE, component = LOG_COMPONENT_HEAL, @@ -170,7 +170,7 @@ impl ErasureSetHealer { task_id, set_disk_id, state = "resume_found", - "Erasure set resume state selected" + "Erasure set resume selected" ); return Ok(task_id); } @@ -193,7 +193,7 @@ impl ErasureSetHealer { // create new task id let task_id = format!("{}_{}", set_disk_id, ResumeUtils::generate_task_id()); - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_RESUME_STATE, component = LOG_COMPONENT_HEAL, @@ -215,7 +215,7 @@ impl ErasureSetHealer { ) -> Result<(ResumeManager, CheckpointManager)> { // check if resume state exists if ResumeManager::has_resume_state(&self.disk, task_id).await { - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_RESUME_STATE, component = LOG_COMPONENT_HEAL, @@ -235,7 +235,7 @@ impl ErasureSetHealer { Ok((resume_manager, checkpoint_manager)) } else { - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_RESUME_STATE, component = LOG_COMPONENT_HEAL, @@ -243,7 +243,7 @@ impl ErasureSetHealer { task_id, set_disk_id, state = "creating_new", - "Erasure set resume state creating" + "Erasure set resume created" ); let resume_manager = ResumeManager::new( @@ -273,7 +273,7 @@ impl ErasureSetHealer { let state = resume_manager.get_state().await; let checkpoint = checkpoint_manager.get_checkpoint().await; - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_RESUME_STATE, component = LOG_COMPONENT_HEAL, @@ -282,7 +282,7 @@ impl ErasureSetHealer { current_bucket_index = checkpoint.current_bucket_index, current_object_index = checkpoint.current_object_index, state = "resuming", - "Erasure set heal resumed from checkpoint" + "Erasure set resumed" ); // 2. initialize progress @@ -349,7 +349,7 @@ impl ErasureSetHealer { match bucket_result { Ok(_) => { resume_manager.complete_bucket(bucket).await?; - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_BUCKET_STATE, component = LOG_COMPONENT_HEAL, @@ -357,7 +357,7 @@ impl ErasureSetHealer { set_disk_id, bucket, state = "completed", - "Erasure set bucket heal completed" + "Erasure set bucket completed" ); } Err(e) => { @@ -383,14 +383,14 @@ impl ErasureSetHealer { // 5. mark task completed resume_manager.mark_completed().await?; - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_RESUME_STATE, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, set_disk_id, state = "completed", - "Erasure set heal completed" + "Erasure set completed" ); Ok(()) } @@ -411,7 +411,7 @@ impl ErasureSetHealer { resume_manager: &ResumeManager, checkpoint_manager: &CheckpointManager, ) -> Result<()> { - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_BUCKET_STATE, component = LOG_COMPONENT_HEAL, @@ -421,7 +421,7 @@ impl ErasureSetHealer { bucket_index, current_object_index = *current_object_index, state = "started", - "Erasure set bucket heal started" + "Erasure set bucket started" ); // 1. get bucket info @@ -584,7 +584,7 @@ impl ErasureSetHealer { Ok(true) => { *successful_objects += 1; checkpoint_manager.add_processed_object(object.clone()).await?; - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_OBJECT_STATE, component = LOG_COMPONENT_HEAL, @@ -593,13 +593,13 @@ impl ErasureSetHealer { bucket, object = %object, state = "healed", - "Erasure set object heal completed" + "Erasure set object healed" ); } Ok(false) => { checkpoint_manager.add_processed_object(object.clone()).await?; *successful_objects += 1; - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_OBJECT_STATE, component = LOG_COMPONENT_HEAL, @@ -608,7 +608,7 @@ impl ErasureSetHealer { bucket, object = %object, state = "missing_treated_as_ok", - "Erasure set object heal skipped because object disappeared" + "Erasure set missing object treated as ok" ); } Err(Error::TaskCancelled) => { @@ -743,14 +743,14 @@ impl ErasureSetHealer { bucket: &str, progress: &Arc>, ) -> Result<()> { - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_BUCKET_STATE, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, bucket, state = "started", - "Erasure set bucket heal started" + "Erasure set bucket started" ); // 1. get bucket info @@ -847,7 +847,7 @@ impl ErasureSetHealer { p.set_current_object(Some(format!("completed bucket: {bucket}"))); } - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_BUCKET_STATE, component = LOG_COMPONENT_HEAL, @@ -857,7 +857,7 @@ impl ErasureSetHealer { total_failed, total_scanned, state = "completed", - "Erasure set bucket heal completed" + "Erasure set bucket completed" ); Ok(()) @@ -890,7 +890,7 @@ impl ErasureSetHealer { match storage.heal_object(&bucket, &object, None, &heal_opts).await { Ok((_result, None)) => { - info!( + debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_OBJECT_STATE, component = LOG_COMPONENT_HEAL, @@ -898,7 +898,7 @@ impl ErasureSetHealer { bucket, object = %object, state = "healed", - "Erasure set object heal completed" + "Erasure set object healed" ); Ok(()) } @@ -956,7 +956,7 @@ impl ErasureSetHealer { success_count, total, state = "summary", - "Erasure set heal summary recorded" + "Erasure set summary recorded" ); if failure_count > 0 { diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index cc707729c..f5091926a 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -558,7 +558,7 @@ impl HealManager { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_MANAGER, state = "already_running", - "Heal manager state unchanged" + "Heal manager already running" ); return Ok(()); } @@ -571,7 +571,7 @@ impl HealManager { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_MANAGER, state = "starting", - "Heal manager state updated" + "Heal manager starting" ); // start scheduler @@ -586,7 +586,7 @@ impl HealManager { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_MANAGER, state = "running", - "Heal manager state updated" + "Heal manager started" ); Ok(()) } @@ -599,7 +599,7 @@ impl HealManager { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_MANAGER, state = "stopping", - "Heal manager state updated" + "Heal manager stopping" ); // cancel all tasks @@ -617,7 +617,7 @@ impl HealManager { state = "task_cancel_failed", task_id = %task.id, error = %e, - "Heal manager failed to cancel active task" + "Heal active task cancellation failed" ); } } @@ -636,7 +636,7 @@ impl HealManager { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_MANAGER, state = "stopped", - "Heal manager state updated" + "Heal manager stopped" ); Ok(()) } @@ -659,7 +659,7 @@ impl HealManager { match admission { HealAdmissionResult::Merged => { - info!( + debug!( target: "rustfs::heal::manager", event = EVENT_HEAL_QUEUE_ADMISSION, component = LOG_COMPONENT_HEAL, @@ -667,7 +667,7 @@ impl HealManager { request_id = %request.id, priority = ?request.priority, result = "merged_duplicate", - "Heal queue admission decided" + "Heal queue request merged" ); } HealAdmissionResult::Dropped(reason) => { @@ -680,7 +680,7 @@ impl HealManager { priority = ?request.priority, reason = reason.as_str(), result = "dropped_duplicate", - "Heal queue admission decided" + "Heal queue request dropped" ); } HealAdmissionResult::Accepted | HealAdmissionResult::Full => {} @@ -707,7 +707,7 @@ impl HealManager { queue_len, queue_capacity, result = "accepted_by_displacement", - "Heal queue admission decided" + "Heal queue request accepted by displacement" ); drop(queue); if config.event_driven_scheduler_enable { @@ -726,7 +726,7 @@ impl HealManager { queue_len, queue_capacity, result = "full_no_displacement_candidate", - "Heal queue admission decided" + "Heal queue request rejected without displacement" ); return Ok(HealAdmissionResult::Full); } @@ -745,7 +745,7 @@ impl HealManager { queue_capacity, reason = reason.as_str(), result = "dropped_full", - "Heal queue admission decided" + "Heal queue request dropped" ); } HealAdmissionResult::Full => { @@ -759,7 +759,7 @@ impl HealManager { queue_len, queue_capacity, result = "rejected_full", - "Heal queue admission decided" + "Heal queue request rejected" ); } HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {} @@ -779,7 +779,7 @@ impl HealManager { queue_capacity, queue_usage_pct = (queue_len * 100) / queue_capacity, result = "queue_pressure_high", - "Heal queue pressure increased" + "Heal queue pressure high" ); } @@ -793,7 +793,7 @@ impl HealManager { // Log queue statistics periodically (when adding high/urgent priority items) if matches!(priority, HealPriority::High | HealPriority::Urgent) { let stats = queue.get_priority_stats(); - info!( + debug!( target: "rustfs::heal::manager", event = EVENT_HEAL_QUEUE_ADMISSION, component = LOG_COMPONENT_HEAL, @@ -812,7 +812,7 @@ impl HealManager { drop(queue); - info!( + debug!( target: "rustfs::heal::manager", event = EVENT_HEAL_QUEUE_ADMISSION, component = LOG_COMPONENT_HEAL, @@ -821,7 +821,7 @@ impl HealManager { priority = ?priority, queue_len = queue_len + 1, result = "accepted", - "Heal queue admission decided" + "Heal queue request accepted" ); if config.event_driven_scheduler_enable { self.notify.notify_one(); @@ -1119,7 +1119,7 @@ impl HealManager { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_MANAGER, state = "shutdown", - "Heal scheduler state updated" + "Heal scheduler stopped" ); break; } @@ -1158,13 +1158,17 @@ impl HealManager { subsystem = LOG_SUBSYSTEM_DISK_SCANNER, state = "started", interval = ?duration, - "Heal auto disk scanner state updated" + "Heal auto disk scanner started" ); tokio::spawn(async move { let mut interval = interval(duration); loop { + let mut candidate_count = 0usize; + let mut skipped_duplicate_count = 0usize; + let mut skipped_invalid_count = 0usize; + let mut enqueued_count = 0usize; tokio::select! { _ = cancel_token.cancelled() => { info!( @@ -1173,7 +1177,7 @@ impl HealManager { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_DISK_SCANNER, state = "shutdown", - "Heal auto disk scanner state updated" + "Heal auto disk scanner stopped" ); break; } @@ -1185,14 +1189,15 @@ impl HealManager { // detect unformatted disk via get_disk_id() match disk.get_disk_id().await { Err(DiskError::UnformattedDisk) => { - info!( + candidate_count += 1; + debug!( target: "rustfs::heal::manager", event = EVENT_HEAL_AUTO_SCAN_DISK, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_DISK_SCANNER, endpoint = %disk.endpoint(), disk_state = "unformatted", - "Heal auto disk scanner detected candidate disk" + "Heal auto-scan candidate detected" ); endpoints.push(disk.endpoint()); } @@ -1205,7 +1210,7 @@ impl HealManager { endpoint = %disk.endpoint(), disk_state = "check_failed", error = ?e, - "Heal auto disk scanner failed to inspect disk" + "Heal auto-scan disk inspection failed" ); } Ok(_) => { @@ -1222,7 +1227,7 @@ impl HealManager { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_DISK_SCANNER, state = "idle", - "Heal auto disk scanner found no candidate disks" + "Heal auto disk scanner idle" ); continue; } @@ -1238,7 +1243,7 @@ impl HealManager { subsystem = LOG_SUBSYSTEM_DISK_SCANNER, state = "bucket_list_failed", error = %e, - "Heal auto disk scanner failed to list buckets" + "Heal auto-scan bucket listing failed" ); continue; } @@ -1256,8 +1261,9 @@ impl HealManager { subsystem = LOG_SUBSYSTEM_DISK_SCANNER, endpoint = %ep, result = "skipped_invalid_set_disk_id", - "Heal auto disk scanner skipped enqueue" + "Heal auto-scan enqueue skipped" ); + skipped_invalid_count += 1; continue; }; // skip if already queued or healing @@ -1283,7 +1289,8 @@ impl HealManager { } if skip { - info!( + skipped_duplicate_count += 1; + debug!( target: "rustfs::heal::manager", event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, component = LOG_COMPONENT_HEAL, @@ -1291,7 +1298,7 @@ impl HealManager { endpoint = %ep, set_disk_id, result = "skipped_duplicate", - "Heal auto disk scanner skipped enqueue" + "Heal auto-scan duplicate skipped" ); continue; } @@ -1312,7 +1319,8 @@ impl HealManager { if config.event_driven_scheduler_enable { notify.notify_one(); } - info!( + enqueued_count += 1; + debug!( target: "rustfs::heal::manager", event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, component = LOG_COMPONENT_HEAL, @@ -1321,10 +1329,22 @@ impl HealManager { set_disk_id, bucket_count = buckets.len(), result = "enqueued", - "Heal auto disk scanner enqueued task" + "Heal auto-scan task enqueued" ); } } + info!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + state = "cycle_completed", + candidate_count, + enqueued_count, + skipped_duplicate_count, + skipped_invalid_count, + "Heal auto-scan cycle completed" + ); } } } @@ -1405,7 +1425,7 @@ impl HealManager { // start heal task tokio::spawn(async move { - info!( + debug!( target: "rustfs::heal::manager", event = EVENT_HEAL_SCHEDULER_STATE, component = LOG_COMPONENT_HEAL, @@ -1415,12 +1435,12 @@ impl HealManager { heal_type = %task_type_label_for_spawn, set = %task_set_label_for_spawn, state = "task_started", - "Heal scheduler started task" + "Heal scheduler task started" ); let result = task.execute().await; match result { Ok(_) => { - info!( + debug!( target: "rustfs::heal::manager", event = EVENT_HEAL_SCHEDULER_STATE, component = LOG_COMPONENT_HEAL, @@ -1429,7 +1449,7 @@ impl HealManager { heal_type = %task_type_label_for_spawn, set = %task_set_label_for_spawn, state = "task_completed", - "Heal scheduler finished task" + "Heal scheduler task completed" ); } Err(e) => { @@ -1443,7 +1463,7 @@ impl HealManager { set = %task_set_label_for_spawn, state = "task_failed", error = %e, - "Heal scheduler observed task failure" + "Heal scheduler task failed" ); } } @@ -1500,7 +1520,7 @@ impl HealManager { queue_len = remaining, active_tasks = active_heals_guard.len(), state = "backlog_high", - "Heal queue backlog summary recorded" + "Heal queue backlog high" ); } } diff --git a/crates/heal/src/heal/resume.rs b/crates/heal/src/heal/resume.rs index 84ce7a4cd..66bf41bfa 100644 --- a/crates/heal/src/heal/resume.rs +++ b/crates/heal/src/heal/resume.rs @@ -20,7 +20,7 @@ use std::path::Path; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::RwLock; -use tracing::{debug, info, warn}; +use tracing::{debug, warn}; use uuid::Uuid; const LOG_COMPONENT_HEAL: &str = "heal"; @@ -299,7 +299,7 @@ impl ResumeManager { let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await; } - info!( + debug!( target: "rustfs::heal::resume", event = EVENT_HEAL_RESUME_STATE, component = LOG_COMPONENT_HEAL, @@ -524,7 +524,7 @@ impl CheckpointManager { let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await; } - info!( + debug!( target: "rustfs::heal::resume", event = EVENT_HEAL_CHECKPOINT_STATE, component = LOG_COMPONENT_HEAL, @@ -649,7 +649,7 @@ impl ResumeUtils { let age_hours = (current_time - state.last_update) / 3600; if age_hours > max_age_hours { - info!( + debug!( target: "rustfs::heal::resume", event = EVENT_HEAL_RESUME_STATE, component = LOG_COMPONENT_HEAL, @@ -657,7 +657,7 @@ impl ResumeUtils { task_id, age_hours, state = "expired_cleanup_started", - "Heal resume state cleanup started" + "Heal resume cleanup started" ); if let Err(e) = resume_manager.cleanup().await { warn!( diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index d28f65ce3..40d708e9f 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -24,7 +24,7 @@ use rustfs_ecstore::{ use rustfs_madmin::heal_commands::HealResultItem; use rustfs_storage_api::{BucketInfo, DiskSetSelector, StorageAdminApi}; use std::sync::Arc; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, warn}; const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_STORAGE: &str = "storage"; @@ -206,7 +206,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket, object, result = "not_found", - "Heal storage request finished" + "Heal storage object metadata missing" ); Ok(None) } else { @@ -333,7 +333,7 @@ impl HealStorageAPI for ECStoreHealStorage { .await { Ok(_) => { - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_OBJECT_IO, component = LOG_COMPONENT_HEAL, @@ -342,7 +342,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket, object, result = "ok", - "Heal storage request finished" + "Heal storage object write completed" ); Ok(()) } @@ -378,7 +378,7 @@ impl HealStorageAPI for ECStoreHealStorage { match self.ecstore.delete_object(bucket, object, Default::default()).await { Ok(_) => { - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_OBJECT_IO, component = LOG_COMPONENT_HEAL, @@ -387,7 +387,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket, object, result = "ok", - "Heal storage request finished" + "Heal storage object delete completed" ); Ok(()) } @@ -447,7 +447,7 @@ impl HealStorageAPI for ECStoreHealStorage { let mut stream = reader.stream; match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await { Ok(_) => { - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, component = LOG_COMPONENT_HEAL, @@ -455,7 +455,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket, object, state = "ok", - "Heal storage object verification finished" + "Heal storage object verified" ); Ok(true) } @@ -544,7 +544,7 @@ impl HealStorageAPI for ECStoreHealStorage { // After healing, try to read the object data match self.get_object_data(bucket, object).await? { Some(data) => { - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_REPAIR_OP, component = LOG_COMPONENT_HEAL, @@ -554,7 +554,7 @@ impl HealStorageAPI for ECStoreHealStorage { object, bytes = data.len(), state = "ok", - "Heal storage repair finished" + "Heal storage EC decode rebuild completed" ); Ok(data) } @@ -608,7 +608,7 @@ impl HealStorageAPI for ECStoreHealStorage { // TODO: implement disk status check using ecstore // For now, return Ok status - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_ADMIN_OP, component = LOG_COMPONENT_HEAL, @@ -617,7 +617,7 @@ impl HealStorageAPI for ECStoreHealStorage { endpoint = ?endpoint, result = "ok", disk_status = "ok", - "Heal storage admin operation finished" + "Heal storage disk status resolved" ); Ok(DiskStatus::Ok) } @@ -640,7 +640,7 @@ impl HealStorageAPI for ECStoreHealStorage { if error.is_some() { return Err(Error::other(format!("Format failed: {error:?}"))); } - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_ADMIN_OP, component = LOG_COMPONENT_HEAL, @@ -648,7 +648,7 @@ impl HealStorageAPI for ECStoreHealStorage { operation = "format_disk", endpoint = ?endpoint, result = "ok", - "Heal storage admin operation finished" + "Heal storage disk format completed" ); Ok(()) } @@ -726,7 +726,7 @@ impl HealStorageAPI for ECStoreHealStorage { match self.heal_bucket(bucket, &heal_opts).await { Ok(_) => { - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_REPAIR_OP, component = LOG_COMPONENT_HEAL, @@ -734,7 +734,7 @@ impl HealStorageAPI for ECStoreHealStorage { operation = "heal_bucket_metadata", bucket, result = "ok", - "Heal storage repair finished" + "Heal storage bucket metadata repaired" ); Ok(()) } @@ -816,7 +816,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket, object, result = "not_found", - "Heal storage request finished" + "Heal storage object absence confirmed" ); Ok(false) } else if is_transient_object_exists_error(&e) { @@ -923,7 +923,7 @@ impl HealStorageAPI for ECStoreHealStorage { match self.ecstore.heal_object(bucket, object, version_id_str, opts).await { Ok((result, ecstore_error)) => { let error = ecstore_error.map(Error::other); - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_REPAIR_OP, component = LOG_COMPONENT_HEAL, @@ -935,7 +935,7 @@ impl HealStorageAPI for ECStoreHealStorage { drives_after = result.after.drives.len(), has_error = error.is_some(), result = "ok", - "Heal storage repair finished" + "Heal storage object repair completed" ); Ok((result, error)) } @@ -974,7 +974,7 @@ impl HealStorageAPI for ECStoreHealStorage { match self.ecstore.heal_bucket(bucket, opts).await { Ok(result) => { - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_REPAIR_OP, component = LOG_COMPONENT_HEAL, @@ -983,7 +983,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket, drives_after = result.after.drives.len(), result = "ok", - "Heal storage repair finished" + "Heal storage bucket repair completed" ); Ok(result) } @@ -1019,7 +1019,7 @@ impl HealStorageAPI for ECStoreHealStorage { match self.ecstore.heal_format(dry_run).await { Ok((result, ecstore_error)) => { let error = ecstore_error.map(Error::other); - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_REPAIR_OP, component = LOG_COMPONENT_HEAL, @@ -1028,7 +1028,7 @@ impl HealStorageAPI for ECStoreHealStorage { drives_after = result.after.drives.len(), has_error = error.is_some(), result = "ok", - "Heal storage repair finished" + "Heal storage format repair completed" ); Ok((result, error)) } @@ -1103,7 +1103,7 @@ impl HealStorageAPI for ECStoreHealStorage { } } - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_ADMIN_OP, component = LOG_COMPONENT_HEAL, @@ -1113,7 +1113,7 @@ impl HealStorageAPI for ECStoreHealStorage { prefix, object_count = all_objects.len(), result = "ok", - "Heal storage admin operation finished" + "Heal storage object listing completed" ); Ok(all_objects) } @@ -1175,12 +1175,12 @@ impl HealStorageAPI for ECStoreHealStorage { component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_STORAGE, operation = "list_objects_for_heal_page", - bucket, - prefix, - object_count = page_count, - is_truncated = list_info.is_truncated, - state = "page_loaded", - "Heal storage admin operation finished" + bucket, + prefix, + object_count = page_count, + is_truncated = list_info.is_truncated, + state = "page_loaded", + "Heal storage object listing page loaded" ); Ok((page_objects, list_info.next_continuation_token, list_info.is_truncated)) @@ -1210,7 +1210,7 @@ impl HealStorageAPI for ECStoreHealStorage { // Find the first available disk if let Some(disk_store) = disks.into_iter().flatten().next() { - info!( + debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_ADMIN_OP, component = LOG_COMPONENT_HEAL, @@ -1219,7 +1219,7 @@ impl HealStorageAPI for ECStoreHealStorage { set_disk_id, result = "ok", disk = ?disk_store, - "Heal storage admin operation finished" + "Heal storage resume disk resolved" ); return Ok(disk_store); } diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 7e80fe81e..a8148b5df 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -445,7 +445,7 @@ impl HealTask { heal_type = self.heal_type.log_kind(), state = "started", queue_delay = ?queue_delay, - "Heal task state updated" + "Heal task started" ); let result = match &self.heal_type { @@ -484,7 +484,7 @@ impl HealTask { task_id = %self.id, heal_type = self.heal_type.log_kind(), state = "completed", - "Heal task state updated" + "Heal task completed" ); } Err(Error::TaskCancelled) => { @@ -498,7 +498,7 @@ impl HealTask { task_id = %self.id, heal_type = self.heal_type.log_kind(), state = "cancelled", - "Heal task state updated" + "Heal task cancelled" ); } Err(Error::TaskTimeout) => { @@ -512,7 +512,7 @@ impl HealTask { task_id = %self.id, heal_type = self.heal_type.log_kind(), state = "timed_out", - "Heal task state updated" + "Heal task timed out" ); } Err(e) => { @@ -527,7 +527,7 @@ impl HealTask { heal_type = self.heal_type.log_kind(), state = "failed", error = %e, - "Heal task state updated" + "Heal task failed" ); } } @@ -539,7 +539,7 @@ impl HealTask { self.cancel_token.cancel(); let mut status = self.status.write().await; *status = HealTaskStatus::Cancelled; - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_TASK_STATE, component = LOG_COMPONENT_HEAL, @@ -548,7 +548,7 @@ impl HealTask { heal_type = self.heal_type.log_kind(), state = "cancelled", source = "manual", - "Heal task state updated" + "Heal task cancellation requested" ); Ok(()) } @@ -572,7 +572,7 @@ impl HealTask { // specific heal implementation method #[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))] async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_STAGE, component = LOG_COMPONENT_HEAL, @@ -582,7 +582,7 @@ impl HealTask { object, version_id = ?version_id, stage = "start", - "Heal object workflow started" + "Heal object started" ); // update progress @@ -625,7 +625,7 @@ impl HealTask { "Heal target object is missing" ); if self.options.recreate_missing { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_STAGE, component = LOG_COMPONENT_HEAL, @@ -634,7 +634,7 @@ impl HealTask { bucket, object, stage = "recreate_missing", - "Heal object recovery started" + "Heal object recreate requested" ); return self.recreate_missing_object(bucket, object, version_id).await; } else { @@ -690,7 +690,7 @@ impl HealTask { // Check if this is a "File not found" error during delete operations let error_msg = format!("{e}"); if error_msg.contains("File not found") || error_msg.contains("not found") { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_RESULT, component = LOG_COMPONENT_HEAL, @@ -699,7 +699,7 @@ impl HealTask { bucket, object, result = "treated_as_deleted", - "Heal object finished after target disappeared during repair" + "Heal missing object treated as deleted" ); { let mut progress = self.progress.write().await; @@ -723,7 +723,7 @@ impl HealTask { // If heal failed and remove_corrupted is enabled, delete the corrupted object if self.options.remove_corrupted { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_CLEANUP, component = LOG_COMPONENT_HEAL, @@ -737,7 +737,7 @@ impl HealTask { ); if !self.options.dry_run { self.await_with_control(self.storage.delete_object(bucket, object)).await?; - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_CLEANUP, component = LOG_COMPONENT_HEAL, @@ -747,7 +747,7 @@ impl HealTask { object, action = "delete_corrupted_object", result = "deleted", - "Heal object cleanup completed" + "Heal corrupted object deleted" ); } } @@ -775,7 +775,7 @@ impl HealTask { "Heal object stage entered" ); let object_size = result.object_size as u64; - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_RESULT, component = LOG_COMPONENT_HEAL, @@ -786,7 +786,7 @@ impl HealTask { object_size = object_size, drives_healed = result.after.drives.len(), result = "ok", - "Heal object completed" + "Heal object repaired" ); { @@ -806,7 +806,7 @@ impl HealTask { // Check if this is a "File not found" error during delete operations let error_msg = format!("{e}"); if error_msg.contains("File not found") || error_msg.contains("not found") { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_RESULT, component = LOG_COMPONENT_HEAL, @@ -815,7 +815,7 @@ impl HealTask { bucket, object, result = "treated_as_deleted", - "Heal object finished after target disappeared during repair" + "Heal missing object treated as deleted" ); { let mut progress = self.progress.write().await; @@ -839,7 +839,7 @@ impl HealTask { // If heal failed and remove_corrupted is enabled, delete the corrupted object if self.options.remove_corrupted { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_CLEANUP, component = LOG_COMPONENT_HEAL, @@ -853,7 +853,7 @@ impl HealTask { ); if !self.options.dry_run { self.await_with_control(self.storage.delete_object(bucket, object)).await?; - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_CLEANUP, component = LOG_COMPONENT_HEAL, @@ -863,7 +863,7 @@ impl HealTask { object, action = "delete_corrupted_object", result = "deleted", - "Heal object cleanup completed" + "Heal corrupted object deleted" ); } } @@ -882,7 +882,7 @@ impl HealTask { /// Recreate missing object (for EC decode scenarios) async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_STAGE, component = LOG_COMPONENT_HEAL, @@ -892,7 +892,7 @@ impl HealTask { object, version_id = ?version_id, stage = "recreate_missing", - "Heal object recovery started" + "Heal object recreate started" ); // Use ecstore's heal_object with recreate option @@ -932,7 +932,7 @@ impl HealTask { } let object_size = result.object_size as u64; - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_RESULT, component = LOG_COMPONENT_HEAL, @@ -942,7 +942,7 @@ impl HealTask { object, object_size, result = "recreated", - "Heal object recovery finished" + "Heal object recreated" ); { @@ -975,7 +975,7 @@ impl HealTask { } async fn heal_bucket(&self, bucket: &str) -> Result<()> { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_BUCKET_STAGE, component = LOG_COMPONENT_HEAL, @@ -984,7 +984,7 @@ impl HealTask { bucket, stage = "start", recursive = self.options.recursive, - "Heal bucket workflow started" + "Heal bucket started" ); // update progress @@ -1060,7 +1060,7 @@ impl HealTask { match heal_result { Ok(result) => { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_BUCKET_RESULT, component = LOG_COMPONENT_HEAL, @@ -1269,7 +1269,7 @@ impl HealTask { }); } - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_BUCKET_RESULT, component = LOG_COMPONENT_HEAL, @@ -1287,7 +1287,7 @@ impl HealTask { } async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_METADATA_STAGE, component = LOG_COMPONENT_HEAL, @@ -1296,7 +1296,7 @@ impl HealTask { bucket, object, stage = "start", - "Heal metadata workflow started" + "Heal metadata started" ); // update progress @@ -1400,7 +1400,7 @@ impl HealTask { }); } - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_METADATA_RESULT, component = LOG_COMPONENT_HEAL, @@ -1410,7 +1410,7 @@ impl HealTask { object, drives_healed = result.after.drives.len(), result = "ok", - "Heal metadata completed" + "Heal metadata repaired" ); { @@ -1447,7 +1447,7 @@ impl HealTask { } async fn heal_mrf(&self, meta_path: &str) -> Result<()> { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_MRF_STAGE, component = LOG_COMPONENT_HEAL, @@ -1455,7 +1455,7 @@ impl HealTask { task_id = %self.id, meta_path, stage = "start", - "Heal MRF workflow started" + "Heal MRF started" ); // update progress @@ -1530,7 +1530,7 @@ impl HealTask { }); } - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_MRF_RESULT, component = LOG_COMPONENT_HEAL, @@ -1541,7 +1541,7 @@ impl HealTask { object = %object, drives_healed = result.after.drives.len(), result = "ok", - "Heal MRF completed" + "Heal MRF repaired" ); { @@ -1579,7 +1579,7 @@ impl HealTask { } async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_EC_DECODE_STAGE, component = LOG_COMPONENT_HEAL, @@ -1589,7 +1589,7 @@ impl HealTask { object, version_id = ?version_id, stage = "start", - "Heal EC decode workflow started" + "Heal EC decode started" ); // update progress @@ -1694,7 +1694,7 @@ impl HealTask { } let object_size = result.object_size as u64; - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_EC_DECODE_RESULT, component = LOG_COMPONENT_HEAL, @@ -1705,7 +1705,7 @@ impl HealTask { object_size, drives_healed = result.after.drives.len(), result = "ok", - "Heal EC decode completed" + "Heal EC decode repaired" ); { @@ -1742,7 +1742,7 @@ impl HealTask { } async fn heal_erasure_set(&self, buckets: Vec, set_disk_id: String) -> Result<()> { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_ERASURE_SET_STAGE, component = LOG_COMPONENT_HEAL, @@ -1751,7 +1751,7 @@ impl HealTask { set_disk_id, bucket_count = buckets.len(), stage = "start", - "Heal erasure set workflow started" + "Heal erasure set started" ); // update progress @@ -1762,7 +1762,7 @@ impl HealTask { } let buckets = if buckets.is_empty() { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_ERASURE_SET_STAGE, component = LOG_COMPONENT_HEAL, @@ -1770,7 +1770,7 @@ impl HealTask { task_id = %self.id, set_disk_id, stage = "list_buckets", - "Heal erasure set resolved bucket list from storage" + "Heal erasure set bucket list resolved" ); let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?; bucket_infos.into_iter().map(|info| info.name).collect() @@ -1814,7 +1814,7 @@ impl HealTask { }); } - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_ERASURE_SET_RESULT, component = LOG_COMPONENT_HEAL, @@ -1823,7 +1823,7 @@ impl HealTask { set_disk_id, drives_healed = result.after.drives.len(), result = "format_ok", - "Heal erasure set format repair completed" + "Heal erasure set format repaired" ); } Err(Error::TaskCancelled) => return Err(Error::TaskCancelled), @@ -1970,7 +1970,7 @@ impl HealTask { match result { Ok(_) => { - info!( + debug!( target: "rustfs::heal::task", event = EVENT_HEAL_ERASURE_SET_RESULT, component = LOG_COMPONENT_HEAL, @@ -1979,7 +1979,7 @@ impl HealTask { set_disk_id, bucket_count = buckets.len(), result = "ok", - "Heal erasure set completed" + "Heal erasure set repaired" ); Ok(()) } diff --git a/crates/iam/src/lib.rs b/crates/iam/src/lib.rs index 31db6cc25..ede1e321f 100644 --- a/crates/iam/src/lib.rs +++ b/crates/iam/src/lib.rs @@ -19,7 +19,13 @@ use rustfs_ecstore::store::ECStore; use std::sync::{Arc, OnceLock}; use store::object::ObjectStore; use sys::IamSys; -use tracing::{error, info, instrument, warn}; +use tracing::{debug, error, info, instrument, warn}; + +const LOG_COMPONENT_IAM: &str = "iam"; +const LOG_SUBSYSTEM_RUNTIME: &str = "runtime"; +const LOG_SUBSYSTEM_OIDC: &str = "oidc"; +const EVENT_IAM_STATE: &str = "iam_state"; +const EVENT_OIDC_STATE: &str = "oidc_state"; pub mod cache; pub mod error; @@ -37,11 +43,23 @@ static OIDC_SYS: OnceLock> = OnceLock::new(); #[instrument(skip(ecstore))] pub async fn init_iam_sys(ecstore: Arc) -> Result<()> { if IAM_SYS.get().is_some() { - info!("IAM system already initialized, skipping."); + info!( + event = EVENT_IAM_STATE, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "already_initialized", + "IAM runtime already initialized" + ); return Ok(()); } - info!("Starting IAM system initialization sequence..."); + info!( + event = EVENT_IAM_STATE, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "starting", + "IAM runtime starting" + ); // 1. Create the persistent storage adapter let storage_adapter = ObjectStore::new(ecstore); @@ -55,11 +73,23 @@ pub async fn init_iam_sys(ecstore: Arc) -> Result<()> { // 4. Securely set the global singleton if IAM_SYS.set(iam_instance).is_err() { - error!("Critical: Race condition detected during IAM initialization!"); + error!( + event = EVENT_IAM_STATE, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "singleton_set_failed", + "IAM runtime singleton set failed" + ); return Err(Error::IamSysAlreadyInitialized); } - info!("IAM system initialization completed successfully."); + info!( + event = EVENT_IAM_STATE, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "ready", + "IAM runtime ready" + ); Ok(()) } @@ -84,29 +114,67 @@ pub fn get_global_iam_sys() -> Option>> { /// Initialize the global OIDC system. Non-fatal if no OIDC providers are configured. pub async fn init_oidc_sys() -> Result<()> { if OIDC_SYS.get().is_some() { - info!("OIDC system already initialized, skipping."); + debug!( + event = EVENT_OIDC_STATE, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + state = "already_initialized", + "OIDC runtime already initialized" + ); return Ok(()); } - info!("Starting OIDC system initialization..."); + debug!( + event = EVENT_OIDC_STATE, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + state = "starting", + "OIDC runtime starting" + ); let oidc_sys = match OidcSys::new().await { Ok(sys) => { if sys.has_providers() { - info!("OIDC system initialized with {} provider(s)", sys.list_providers().len()); + debug!( + event = EVENT_OIDC_STATE, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + provider_count = sys.list_providers().len(), + state = "ready", + "OIDC runtime ready" + ); } else { - info!("No OIDC providers configured"); + debug!( + event = EVENT_OIDC_STATE, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + state = "empty", + "OIDC runtime has no providers" + ); } sys } Err(e) => { - warn!("OIDC initialization failed (non-fatal): {}", e); + warn!( + event = EVENT_OIDC_STATE, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + state = "init_failed_non_fatal", + error = %e, + "OIDC runtime initialization failed" + ); OidcSys::empty().map_err(Error::StringError)? } }; if OIDC_SYS.set(Arc::new(oidc_sys)).is_err() { - warn!("Race condition during OIDC initialization (non-fatal)"); + warn!( + event = EVENT_OIDC_STATE, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + state = "singleton_set_race", + "OIDC runtime singleton set raced" + ); } Ok(()) diff --git a/crates/iam/src/manager.rs b/crates/iam/src/manager.rs index f6a6b8053..419e3b7c3 100644 --- a/crates/iam/src/manager.rs +++ b/crates/iam/src/manager.rs @@ -54,7 +54,7 @@ use tokio::{ }, }; use tracing::warn; -use tracing::{error, info}; +use tracing::{debug, error}; const IAM_FORMAT_FILE: &str = "format.json"; const IAM_FORMAT_VERSION_1: i32 = 1; @@ -153,10 +153,14 @@ where if let Err(e) = self.clone().load().await { if attempt == MAX_RETRIES - 1 { self.state.store(IamState::Error as u8, Ordering::SeqCst); - warn!("IAM failed to load initial data after {} attempts: {:?}", MAX_RETRIES, e); + warn!( + attempts = MAX_RETRIES, + error = ?e, + "IAM initial load failed" + ); load_error = Some(e); } else { - warn!("IAM load failed, retrying... attempt {}", attempt + 1); + warn!(attempt = attempt + 1, max_attempts = MAX_RETRIES, "IAM load retry scheduled"); tokio::time::sleep(INITIAL_LOAD_RETRY_DELAY).await; } } else { @@ -169,7 +173,7 @@ where } self.state.store(IamState::Ready as u8, Ordering::SeqCst); - info!("IAM System successfully initialized and marked as READY"); + debug!(state = "ready", "IAM manager ready"); // Background ticker for synchronization // Check if environment variable is set @@ -185,20 +189,20 @@ where loop { select! { _ = ticker.tick() => { - info!("iam load ticker"); + debug!(source = "ticker", "IAM reload tick"); if let Err(err) =s.clone().load().await{ - warn!("iam load err {:?}", err); + warn!(source = "ticker", error = ?err, "IAM reload failed"); } }, i = receiver.recv() => { - info!("iam load receiver"); + debug!(source = "receiver", "IAM reload signal received"); match i { Some(t) => { let last = s.last_timestamp.load(Ordering::Relaxed); if last <= t { - info!("iam load receiver load"); + debug!(source = "receiver", "IAM reload accepted"); if let Err(err) =s.clone().load().await{ - warn!("iam load err {:?}", err); + warn!(source = "receiver", error = ?err, "IAM reload failed"); } ticker.reset(); } @@ -1294,7 +1298,7 @@ where let cache = self.cache.snapshot(); let users = Arc::clone(&cache.users); if let Some(x) = users.get(access_key) { - warn!("user already exists: {:?}", x); + warn!(error = ?x, "IAM user already exists"); if x.credentials.is_temp() { return Err(Error::IAMActionNotAllowed); } diff --git a/crates/iam/src/oidc.rs b/crates/iam/src/oidc.rs index 9f8d167dc..23e4c3696 100644 --- a/crates/iam/src/oidc.rs +++ b/crates/iam/src/oidc.rs @@ -40,7 +40,7 @@ use std::pin::Pin; use std::sync::{LazyLock, Mutex, MutexGuard, RwLock}; use std::time::{Duration as StdDuration, Instant}; use tokio::time::sleep; -use tracing::{error, info, warn}; +use tracing::{debug, error, warn}; use url::Url; const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60); @@ -76,7 +76,7 @@ fn lock_oidc_plugin_authn_metrics<'a, T>(mutex: &'a Mutex, metric: &'static s match mutex.lock() { Ok(guard) => guard, Err(err) => { - warn!("recovering poisoned OIDC plugin authn metrics lock: {}", metric); + warn!(metric, "Recovering poisoned OIDC authn metrics lock"); err.into_inner() } } @@ -417,18 +417,18 @@ impl OidcSys { for sourced_config in parsed_configs { let config = sourced_config.config; if !config.enabled { - info!("OIDC provider '{}' is disabled, skipping", config.id); + debug!(provider = %config.id, "OIDC provider disabled"); continue; } match Self::discover_provider(&config, &http_client).await { Ok(state) => { - info!("OIDC provider '{}' discovered successfully", config.id); + debug!(provider = %config.id, "OIDC provider discovered"); provider_states.insert(config.id.clone(), state); configs.insert(config.id.clone(), config); } Err(e) => { - error!("Failed to discover OIDC provider '{}': {}", config.id, e); + error!(provider = %config.id, error = %e, "OIDC provider discovery failed"); } } } diff --git a/crates/iam/src/store/object.rs b/crates/iam/src/store/object.rs index 6507b31f6..4a367710f 100644 --- a/crates/iam/src/store/object.rs +++ b/crates/iam/src/store/object.rs @@ -41,7 +41,7 @@ use std::time::{Duration, Instant}; use std::{collections::HashMap, sync::Arc}; use tokio::sync::mpsc::{self, Sender}; use tokio_util::sync::CancellationToken; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, warn}; pub static IAM_CONFIG_PREFIX: LazyLock = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam")); pub static IAM_CONFIG_USERS_PREFIX: LazyLock = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam/users/")); @@ -307,11 +307,11 @@ impl ObjectStore { } Err(StorageError::PreconditionFailed) => { Self::complete_lazy_rewrite(path.as_str(), false); - debug!("iam lazy rewrite skipped due to stale etag, path: {}", path); + debug!(path = %path, state = "stale_etag", "IAM lazy rewrite skipped"); } Err(err) => { Self::complete_lazy_rewrite(path.as_str(), false); - warn!("iam lazy rewrite failed, path: {}, err: {}", path, err); + warn!(path = %path, error = %err, state = "rewrite_failed", "IAM lazy rewrite failed"); } } }); @@ -377,7 +377,7 @@ impl ObjectStore { bucket = Self::BUCKET_NAME, prefix = %path, error = %err, - "system path walk failed" + "IAM config walk failed" ); let _ = sender_on_error .send(StringOrErr { @@ -643,7 +643,7 @@ impl Store for ObjectStore { let outcome = match Self::decrypt_data_with_source(&data) { Ok(v) => v, Err(err) => { - warn!("config decrypt failed, keeping file: {}, path: {}", err, path_ref); + warn!(path = %path_ref, error = %err, "IAM config decrypt failed; keeping file"); // keep the config file when decrypt failed - do not delete return Err(Error::ConfigNotFound); } @@ -692,7 +692,7 @@ impl Store for ObjectStore { tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await; } Err(e) => { - error!("Final failure saving IAM config to {}: {:?}", path_ref, e); + error!(path = %path_ref, error = ?e, "IAM config save failed"); return Err(e.into()); } } @@ -717,7 +717,7 @@ impl Store for ObjectStore { debug!("Saving IAM identity to path: {}", path); self.save_iam_config(user_identity, path).await.map_err(|e| { - error!("ObjectStore save failure for {}: {:?}", name, e); + error!(name, error = ?e, "IAM identity save failed"); e }) } @@ -739,10 +739,10 @@ impl Store for ObjectStore { .await .map_err(|err| { if is_err_config_not_found(&err) { - warn!("load_user_identity failed: no such user, name: {name}, user_type: {user_type:?}"); + warn!(name, user_type = ?user_type, "IAM user identity missing"); Error::NoSuchUser(name.to_owned()) } else { - warn!("load_user_identity failed: {err:?}, name: {name}, user_type: {user_type:?}"); + warn!(name, user_type = ?user_type, error = ?err, "IAM user identity load failed"); err } })?; @@ -750,9 +750,7 @@ impl Store for ObjectStore { if u.credentials.is_expired() { let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await; let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await; - warn!( - "load_user_identity failed: user is expired, delete the user and mapped policy, name: {name}, user_type: {user_type:?}" - ); + warn!(name, user_type = ?user_type, "IAM user identity expired and was removed"); return Err(Error::NoSuchUser(name.to_owned())); } @@ -776,7 +774,7 @@ impl Store for ObjectStore { let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await; let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await; } - warn!("extract_jwt_claims failed: {err:?}, name: {name}, user_type: {user_type:?}"); + warn!(name, user_type = ?user_type, error = ?err, "IAM JWT claim extraction failed"); return Err(Error::NoSuchUser(name.to_owned())); } } @@ -804,7 +802,7 @@ impl Store for ObjectStore { while let Some(v) = rx.recv().await { if let Some(err) = v.err { - warn!("list_iam_config_items {:?}", err); + warn!(error = ?err, "IAM config item listing failed"); let _ = ctx.cancel(); return Err(err); @@ -866,7 +864,7 @@ impl Store for ObjectStore { while let Some(v) = rx.recv().await { if let Some(err) = v.err { - warn!("list_iam_config_items {:?}", err); + warn!(error = ?err, "IAM config item listing failed"); let _ = ctx.cancel(); return Err(err); @@ -934,7 +932,7 @@ impl Store for ObjectStore { while let Some(v) = rx.recv().await { if let Some(err) = v.err { - warn!("list_iam_config_items {:?}", err); + warn!(error = ?err, "IAM config item listing failed"); let _ = ctx.cancel(); return Err(err); @@ -1009,7 +1007,7 @@ impl Store for ObjectStore { while let Some(v) = rx.recv().await { if let Some(err) = v.err { - warn!("list_iam_config_items {:?}", err); + warn!(error = ?err, "IAM config item listing failed"); let _ = ctx.cancel(); return Err(err); @@ -1044,7 +1042,7 @@ impl Store for ObjectStore { let policy_name = rustfs_utils::path::dir(&policies_list[idx]); - info!("load policy: {}", policy_name); + debug!(policy = %policy_name, "IAM policy loaded"); policy_docs_cache.insert(policy_name, p); } @@ -1059,7 +1057,7 @@ impl Store for ObjectStore { } let policy_name = rustfs_utils::path::dir(&policies_list[idx]); - info!("load policy: {}", policy_name); + debug!(policy = %policy_name, "IAM policy loaded"); policy_docs_cache.insert(policy_name, p); } @@ -1083,7 +1081,7 @@ impl Store for ObjectStore { } let name = rustfs_utils::path::dir(&item_name_list[idx]); - info!("load reg user: {}", name); + debug!(user = %name, "IAM regular user loaded"); user_items_cache.insert(name, p); } break; @@ -1097,7 +1095,7 @@ impl Store for ObjectStore { } let name = rustfs_utils::path::dir(&item_name_list[idx]); - info!("load reg user: {}", name); + debug!(user = %name, "IAM regular user loaded"); user_items_cache.insert(name, p); } @@ -1112,7 +1110,7 @@ impl Store for ObjectStore { for item in item_name_list.iter() { let name = rustfs_utils::path::dir(item); - info!("load group: {}", name); + debug!(group = %name, "IAM group loaded"); if let Err(err) = self.load_group(&name, &mut items_cache).await { return Err(Error::other(format!("load group failed: {err}"))); }; @@ -1140,7 +1138,7 @@ impl Store for ObjectStore { } let name = item_name_list[idx].trim_end_matches(".json").to_owned(); - info!("load user policy: {}", name); + debug!(user = %name, "IAM user policy loaded"); items_cache.insert(name, p); } break; @@ -1156,7 +1154,7 @@ impl Store for ObjectStore { } let name = item_name_list[idx].trim_end_matches(".json").to_owned(); - info!("load user policy: {}", name); + debug!(user = %name, "IAM user policy loaded"); items_cache.insert(name, p); } @@ -1174,7 +1172,7 @@ impl Store for ObjectStore { for item in item_name_list.iter() { let name = item.trim_end_matches(".json"); - info!("load group policy: {}", name); + debug!(group = %name, "IAM group policy loaded"); if let Err(err) = self.load_mapped_policy(name, UserType::Reg, true, &mut items_cache).await && !is_err_no_such_policy(&err) { @@ -1193,7 +1191,7 @@ impl Store for ObjectStore { for item in item_name_list.iter() { let name = rustfs_utils::path::dir(item); - info!("load svc user: {}", name); + debug!(user = %name, "IAM service user loaded"); if let Err(err) = self.load_user(&name, UserType::Svc, &mut items_cache).await && !is_err_no_such_user(&err) { @@ -1204,7 +1202,7 @@ impl Store for ObjectStore { for (_, v) in items_cache.iter() { let parent = v.credentials.parent_user.clone(); if !user_items_cache.contains_key(&parent) { - info!("load sts user policy: {}", parent); + debug!(user = %parent, "IAM STS parent policy loaded"); if let Err(err) = self .load_mapped_policy(&parent, UserType::Sts, false, &mut sts_policies_cache) .await @@ -1223,12 +1221,12 @@ impl Store for ObjectStore { // sts users if let Some(item_name_list) = listed_config_items.get(STS_LIST_KEY) { for item in item_name_list.iter() { - info!("load sts user path: {}", item); + debug!(path = %item, "IAM STS user path discovered"); let name = rustfs_utils::path::dir(item); - info!("load sts user: {}", name); + debug!(user = %name, "IAM STS user loaded"); if let Err(err) = self.load_user(&name, UserType::Sts, &mut sts_items_cache).await { - info!("load sts user failed: {}", err); + debug!(user = %name, error = %err, "IAM STS user load failed"); }; } } @@ -1237,12 +1235,12 @@ impl Store for ObjectStore { if let Some(item_name_list) = listed_config_items.get(POLICY_DB_STS_USERS_LIST_KEY) { for item in item_name_list.iter() { let name = item.trim_end_matches(".json"); - info!("load sts user policy: {}", name); + debug!(user = %name, "IAM STS user policy loaded"); if let Err(err) = self .load_mapped_policy(name, UserType::Sts, false, &mut sts_policies_cache) .await { - info!("load sts user policy failed: {}", err); + debug!(user = %name, error = %err, "IAM STS user policy load failed"); }; } } @@ -1264,7 +1262,7 @@ impl Store for ObjectStore { cache.replace_sts_policies(sts_policies_cache); cache.build_user_group_memberships(); } else { - warn!("skip IAM full reload cache commit because one or more IAM caches changed during reload"); + warn!("IAM full reload cache commit skipped due to concurrent cache changes"); } }); diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index 2d8d76fd6..7e7648717 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -34,7 +34,7 @@ use std::path::PathBuf; use std::time::Duration; use tokio::fs; use tokio::sync::RwLock; -use tracing::{debug, info, warn}; +use tracing::{debug, warn}; /// Local KMS client that stores keys in local files pub struct LocalKmsClient { @@ -74,7 +74,7 @@ impl LocalKmsClient { // Create key directory if it doesn't exist if !config.key_dir.exists() { fs::create_dir_all(&config.key_dir).await?; - info!("Created KMS key directory: {:?}", config.key_dir); + debug!(path = ?config.key_dir, "KMS key directory created"); } // Initialize master cipher if master key is provided @@ -219,7 +219,7 @@ impl LocalKmsClient { fs::rename(&temp_path, &key_path).await?; - info!("Saved master key {} to {:?}", master_key.key_id, key_path); + debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved"); Ok(()) } @@ -278,7 +278,7 @@ impl KmsClient for LocalKmsClient { let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone()); - info!("Generated data key for master key: {}", request.master_key_id); + debug!(key_id = %request.master_key_id, "Local KMS data key generated"); Ok(data_key) } @@ -334,7 +334,7 @@ impl KmsClient for LocalKmsClient { .decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce) .await?; - info!("Successfully decrypted data"); + debug!("Local KMS data decrypted"); Ok(plaintext) } @@ -367,7 +367,7 @@ impl KmsClient for LocalKmsClient { let mut cache = self.key_cache.write().await; cache.insert(key_id.to_string(), master_key.clone()); - info!("Created master key: {}", key_id); + debug!(key_id, "Local KMS master key created"); Ok(master_key) } @@ -453,7 +453,7 @@ impl KmsClient for LocalKmsClient { let mut cache = self.key_cache.write().await; cache.insert(key_id.to_string(), master_key); - info!("Enabled key: {}", key_id); + debug!(key_id, "Local KMS key enabled"); Ok(()) } @@ -470,7 +470,7 @@ impl KmsClient for LocalKmsClient { let mut cache = self.key_cache.write().await; cache.insert(key_id.to_string(), master_key); - info!("Disabled key: {}", key_id); + debug!(key_id, "Local KMS key disabled"); Ok(()) } @@ -492,7 +492,7 @@ impl KmsClient for LocalKmsClient { let mut cache = self.key_cache.write().await; cache.insert(key_id.to_string(), master_key); - warn!("Scheduled key deletion: {}", key_id); + debug!(key_id, "Local KMS key deletion scheduled"); Ok(()) } @@ -509,7 +509,7 @@ impl KmsClient for LocalKmsClient { let mut cache = self.key_cache.write().await; cache.insert(key_id.to_string(), master_key); - info!("Canceled deletion for key: {}", key_id); + debug!(key_id, "Local KMS key deletion canceled"); Ok(()) } @@ -528,7 +528,7 @@ impl KmsClient for LocalKmsClient { let mut cache = self.key_cache.write().await; cache.insert(key_id.to_string(), master_key.clone()); - info!("Rotated key: {}", key_id); + debug!(key_id, "Local KMS key rotated"); Ok(master_key) } @@ -721,7 +721,7 @@ impl KmsBackend for LocalKmsBackend { let mut cache = self.client.key_cache.write().await; cache.remove(key_id); - info!("Immediately deleted key: {}", key_id); + debug!(key_id, "Local KMS key deleted immediately"); // Return success response for immediate deletion let key_metadata = KeyMetadata { diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index f3dce6a72..3983c304c 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -98,7 +98,7 @@ impl VaultKmsClient { let client = VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}")))?; - info!("Successfully connected to Vault at {}", config.address); + info!(address = %config.address, "Vault KMS backend connected"); Ok(Self { client, @@ -136,7 +136,7 @@ impl VaultKmsClient { // If encrypted_key_material is empty, generate and store it (fix for old keys) if key_data.encrypted_key_material.is_empty() { - warn!("Key {} has empty encrypted_key_material, generating and storing new key material", key_id); + warn!(key_id, "Vault KMS key material missing; regenerating"); let key_material = generate_key_material(&key_data.algorithm)?; key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?; // Store the updated key data back to Vault @@ -147,7 +147,7 @@ impl VaultKmsClient { let key_material = match self.decrypt_key_material(&key_data.encrypted_key_material).await { Ok(km) => km, Err(e) => { - warn!("Failed to decrypt key material for key {}: {}, generating new key material", key_id, e); + warn!(key_id, error = %e, "Vault KMS key material decrypt failed; regenerating"); let new_key_material = generate_key_material(&key_data.algorithm)?; key_data.encrypted_key_material = self.encrypt_key_material(&new_key_material).await?; // Store the updated key data back to Vault @@ -210,7 +210,7 @@ impl VaultKmsClient { // If encrypted_key_material is empty, generate it (this handles the case where // an old key was created without proper key material) if existing_key_data.encrypted_key_material.is_empty() { - warn!("Key {} has empty encrypted_key_material, generating new key material", key_id); + warn!(key_id, "Vault KMS key metadata missing encrypted key material"); let key_material = generate_key_material(&existing_key_data.algorithm)?; existing_key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?; } @@ -316,7 +316,7 @@ impl KmsClient for VaultKmsClient { let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone()); - info!("Generated data key for master key: {}", request.master_key_id); + debug!(key_id = %request.master_key_id, "Vault KMS data key generated"); Ok(data_key) } @@ -373,7 +373,7 @@ impl KmsClient for VaultKmsClient { .decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce) .await?; - info!("Successfully decrypted data"); + debug!("Vault KMS data decrypted"); Ok(plaintext) } @@ -418,7 +418,7 @@ impl KmsClient for VaultKmsClient { created_by: None, }; - info!("Successfully created master key: {}", key_id); + debug!(key_id, "Vault KMS master key created"); Ok(master_key) } @@ -486,7 +486,7 @@ impl KmsClient for VaultKmsClient { key_data.status = KeyStatus::Active; self.store_key_data(key_id, &key_data).await?; - info!("Enabled key: {}", key_id); + debug!(key_id, "Vault KMS key enabled"); Ok(()) } @@ -497,7 +497,7 @@ impl KmsClient for VaultKmsClient { key_data.status = KeyStatus::Disabled; self.store_key_data(key_id, &key_data).await?; - info!("Disabled key: {}", key_id); + debug!(key_id, "Vault KMS key disabled"); Ok(()) } @@ -513,7 +513,7 @@ impl KmsClient for VaultKmsClient { key_data.status = KeyStatus::PendingDeletion; self.store_key_data(key_id, &key_data).await?; - info!("Scheduled key deletion: {}", key_id); + debug!(key_id, "Vault KMS key deletion scheduled"); Ok(()) } @@ -524,7 +524,7 @@ impl KmsClient for VaultKmsClient { key_data.status = KeyStatus::Active; self.store_key_data(key_id, &key_data).await?; - info!("Canceled key deletion: {}", key_id); + debug!(key_id, "Vault KMS key deletion canceled"); Ok(()) } @@ -553,7 +553,7 @@ impl KmsClient for VaultKmsClient { created_by: None, }; - info!("Successfully rotated key: {}", key_id); + debug!(key_id, "Vault KMS key rotated"); Ok(master_key) } @@ -573,7 +573,7 @@ impl KmsClient for VaultKmsClient { debug!("Vault health check passed - 404 error is expected when no keys exist yet"); Ok(()) } else { - warn!("Vault health check failed: {}", e); + warn!(error = %e, "Vault KMS health check failed"); Err(e) } } diff --git a/crates/kms/src/service.rs b/crates/kms/src/service.rs index e2bd7e41b..4e7501328 100644 --- a/crates/kms/src/service.rs +++ b/crates/kms/src/service.rs @@ -24,7 +24,7 @@ use rand::random; use std::collections::HashMap; use std::io::Cursor; use tokio::io::{AsyncRead, AsyncReadExt}; -use tracing::{debug, info}; +use tracing::debug; use zeroize::Zeroize; /// Data key for object encryption @@ -209,7 +209,7 @@ impl ObjectEncryptionService { // Generate a unique random nonce for this data key // This ensures each object/part gets a unique base nonce for streaming encryption let nonce: [u8; 12] = random(); - tracing::info!("Generated random nonce for data key: {:02x?}", nonce); + tracing::debug!("Generated random nonce for data key"); let data_key = DataKey { plaintext_key: data_key_response @@ -302,7 +302,7 @@ impl ObjectEncryptionService { key_id: actual_key_id.to_string(), }; if let Err(KmsError::KeyNotFound { .. }) = self.kms_manager.describe_key(describe_req).await { - info!("Auto-creating SSE-S3 key: {}", actual_key_id); + debug!(key_id = %actual_key_id, "Auto-creating SSE-S3 key"); let create_req = CreateKeyRequest { key_name: Some(actual_key_id.to_string()), key_usage: KeyUsage::EncryptDecrypt, @@ -364,7 +364,13 @@ impl ObjectEncryptionService { encrypted_data_key: data_key.ciphertext_blob, }; - info!("Successfully encrypted object {}/{} ({} bytes)", bucket, object_key, original_size); + debug!( + bucket, + object = object_key, + original_size, + algorithm = %algorithm.as_str(), + "Object encrypted" + ); Ok(EncryptionResult { ciphertext, metadata }) } @@ -429,7 +435,13 @@ impl ObjectEncryptionService { // Decrypt the data let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?; - info!("Successfully decrypted object {}/{} ({} bytes)", bucket, object_key, plaintext.len()); + debug!( + bucket, + object = object_key, + plaintext_len = plaintext.len(), + algorithm = %metadata.algorithm, + "Object decrypted" + ); Ok(Box::new(Cursor::new(plaintext))) } @@ -507,7 +519,7 @@ impl ObjectEncryptionService { encrypted_data_key: Vec::new(), // Empty for SSE-C }; - info!( + debug!( "Successfully encrypted object {}/{} with SSE-C ({} bytes)", bucket, object_key, original_size ); @@ -568,7 +580,7 @@ impl ObjectEncryptionService { // Decrypt the data let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?; - info!( + debug!( "Successfully decrypted SSE-C object {}/{} ({} bytes)", bucket, object_key, diff --git a/crates/kms/src/service_manager.rs b/crates/kms/src/service_manager.rs index 387cffa4e..214d9e33d 100644 --- a/crates/kms/src/service_manager.rs +++ b/crates/kms/src/service_manager.rs @@ -25,7 +25,11 @@ use std::sync::{ atomic::{AtomicU64, Ordering}, }; use tokio::sync::{Mutex, RwLock}; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; + +const LOG_COMPONENT_KMS: &str = "kms"; +const LOG_SUBSYSTEM_SERVICE: &str = "service"; +const EVENT_KMS_SERVICE_STATE: &str = "kms_service_state"; /// KMS service status #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] @@ -106,7 +110,13 @@ impl KmsServiceManager { *status = KmsServiceStatus::Configured; } - info!("KMS configuration updated successfully"); + debug!( + event = EVENT_KMS_SERVICE_STATE, + component = LOG_COMPONENT_KMS, + subsystem = LOG_SUBSYSTEM_SERVICE, + state = "configured", + "KMS service configured" + ); Ok(()) } @@ -132,7 +142,14 @@ impl KmsServiceManager { } }; - info!("Starting KMS service with backend: {:?}", config.backend); + info!( + event = EVENT_KMS_SERVICE_STATE, + component = LOG_COMPONENT_KMS, + subsystem = LOG_SUBSYSTEM_SERVICE, + backend = ?config.backend, + state = "starting", + "KMS service starting" + ); match self.create_service_version(&config).await { Ok(service_version) => { @@ -146,7 +163,13 @@ impl KmsServiceManager { *status = KmsServiceStatus::Running; } - info!("KMS service started successfully"); + debug!( + event = EVENT_KMS_SERVICE_STATE, + component = LOG_COMPONENT_KMS, + subsystem = LOG_SUBSYSTEM_SERVICE, + state = "running", + "KMS service running" + ); Ok(()) } Err(e) => { @@ -170,7 +193,13 @@ impl KmsServiceManager { /// Internal stop implementation (called within lifecycle mutex) async fn stop_internal(&self) -> Result<()> { - info!("Stopping KMS service"); + debug!( + event = EVENT_KMS_SERVICE_STATE, + component = LOG_COMPONENT_KMS, + subsystem = LOG_SUBSYSTEM_SERVICE, + state = "stopping", + "KMS service stopping" + ); // Atomically clear current service version (lock-free, instant) // Note: Existing Arc references will keep the service alive until operations complete @@ -184,7 +213,13 @@ impl KmsServiceManager { } } - info!("KMS service stopped successfully (existing operations may continue)"); + debug!( + event = EVENT_KMS_SERVICE_STATE, + component = LOG_COMPONENT_KMS, + subsystem = LOG_SUBSYSTEM_SERVICE, + state = "configured", + "KMS service stopped" + ); Ok(()) } @@ -201,7 +236,13 @@ impl KmsServiceManager { pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> { let _guard = self.lifecycle_mutex.lock().await; - info!("Reconfiguring KMS service (zero-downtime)"); + debug!( + event = EVENT_KMS_SERVICE_STATE, + component = LOG_COMPONENT_KMS, + subsystem = LOG_SUBSYSTEM_SERVICE, + state = "reconfiguring", + "KMS service reconfiguring" + ); new_config.validate()?; // Configure with new config @@ -230,13 +271,22 @@ impl KmsServiceManager { if let Some(old_ver) = old_version { info!( - "KMS service reconfigured successfully: version {} -> {} (old service will be cleaned up when operations complete)", - old_ver, new_service_version.version + event = EVENT_KMS_SERVICE_STATE, + component = LOG_COMPONENT_KMS, + subsystem = LOG_SUBSYSTEM_SERVICE, + old_version = old_ver, + new_version = new_service_version.version, + state = "running", + "KMS service reconfigured" ); } else { info!( - "KMS service reconfigured successfully: version {} (service started)", - new_service_version.version + event = EVENT_KMS_SERVICE_STATE, + component = LOG_COMPONENT_KMS, + subsystem = LOG_SUBSYSTEM_SERVICE, + new_version = new_service_version.version, + state = "running", + "KMS service started from reconfigure" ); } Ok(()) diff --git a/crates/notify/src/bucket_config_manager.rs b/crates/notify/src/bucket_config_manager.rs index f06db42f8..1de84b4ea 100644 --- a/crates/notify/src/bucket_config_manager.rs +++ b/crates/notify/src/bucket_config_manager.rs @@ -69,7 +69,7 @@ impl NotifyBucketConfigManager { bucket = %bucket, region = %cfg.region, available_arn_count = arn_list.len(), - "Loaded available notify target ARNs for bucket config validation" + "notify bucket config validation" ); if let Err(e) = cfg.validate(&cfg.region, &arn_list) { @@ -81,7 +81,7 @@ impl NotifyBucketConfigManager { region = %cfg.region, error = %e, result = "validation_failed", - "Bucket notification config validation failed" + "notify bucket config validation" ); if !matches!(e, ParseConfigError::ArnNotFound(_)) { return Err(NotificationError::BucketNotification(e.to_string())); @@ -94,7 +94,7 @@ impl NotifyBucketConfigManager { region = %cfg.region, error = %e, result = "missing_target_arn", - "Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules" + "notify bucket config validation" ); } @@ -107,7 +107,7 @@ impl NotifyBucketConfigManager { bucket = %bucket, region = %cfg.region, rule_count = cfg.get_rules_map().inner().len(), - "Loaded bucket notification config" + "notify bucket config state" ); Ok(()) } diff --git a/crates/notify/src/config_manager.rs b/crates/notify/src/config_manager.rs index 9f0e2999d..6be90f866 100644 --- a/crates/notify/src/config_manager.rs +++ b/crates/notify/src/config_manager.rs @@ -91,7 +91,7 @@ impl NotifyConfigManager { component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_CONFIG, state = "initializing", - "Initializing notification system" + "notify runtime lifecycle" ); let config = { @@ -111,7 +111,7 @@ impl NotifyConfigManager { subsystem = LOG_SUBSYSTEM_CONFIG, state = "targets_created", target_count = targets.len(), - "Created notification targets" + "notify runtime lifecycle" ); if targets.is_empty() { debug!( @@ -121,7 +121,7 @@ impl NotifyConfigManager { state = "idle", reason = "no_targets_configured", hint = %notify_configuration_hint(), - "Notification runtime has no configured targets" + "notify runtime lifecycle" ); } @@ -132,7 +132,7 @@ impl NotifyConfigManager { component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_CONFIG, state = "initialized", - "Initialized notification system" + "notify runtime lifecycle" ); Ok(()) } @@ -156,13 +156,13 @@ impl NotifyConfigManager { if let Some(targets_of_type) = config.0.get_mut(&ttype) { if targets_of_type.remove(&tname).is_some() { info!( - event = EVENT_NOTIFY_CONFIG_UPDATE, - component = LOG_COMPONENT_NOTIFY, + event = EVENT_NOTIFY_CONFIG_UPDATE, + component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_CONFIG, action = "remove_target", target_id = %target_id, result = "removed", - "Removed notification target from configuration" + "notify config update" ); changed = true; } @@ -178,7 +178,7 @@ impl NotifyConfigManager { action = "remove_target", target_id = %target_id, result = "not_found", - "Notification target not found in configuration" + "notify config update" ); } changed @@ -246,7 +246,7 @@ impl NotifyConfigManager { target_type = %target_type, target_name = %target_name, result = "not_found", - "Notification target configuration not found" + "notify config update" ); } debug!( @@ -264,7 +264,7 @@ impl NotifyConfigManager { component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_CONFIG, state = "reloading", - "Reloading notification configuration" + "notify runtime lifecycle" ); self.update_config(new_config.clone()).await; @@ -281,7 +281,7 @@ impl NotifyConfigManager { subsystem = LOG_SUBSYSTEM_CONFIG, state = "targets_created", target_count = targets.len(), - "Created notification targets from reloaded configuration" + "notify runtime lifecycle" ); if targets.is_empty() { debug!( @@ -291,7 +291,7 @@ impl NotifyConfigManager { state = "idle", reason = "no_targets_configured", hint = %notify_configuration_hint(), - "Notification runtime has no configured targets after reload" + "notify runtime lifecycle" ); } @@ -302,7 +302,7 @@ impl NotifyConfigManager { component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_CONFIG, state = "reloaded", - "Reloaded notification configuration" + "notify runtime lifecycle" ); Ok(()) } @@ -333,7 +333,7 @@ impl NotifyConfigManager { subsystem = LOG_SUBSYSTEM_CONFIG, action = "reload_if_changed", result = "unchanged", - "Notification configuration unchanged; skipping reload" + "notify config update" ); return Ok(()); } @@ -348,7 +348,7 @@ impl NotifyConfigManager { subsystem = LOG_SUBSYSTEM_CONFIG, action = "reload_if_changed", result = "updated", - "Notification configuration updated; reloading runtime" + "notify config update" ); self.reload_config(new_config).await } diff --git a/crates/notify/src/global.rs b/crates/notify/src/global.rs index 059917fe7..894c51e30 100644 --- a/crates/notify/src/global.rs +++ b/crates/notify/src/global.rs @@ -23,6 +23,9 @@ use std::sync::{Arc, OnceLock}; use tracing::error; static NOTIFICATION_SYSTEM: OnceLock> = OnceLock::new(); +const LOG_COMPONENT_NOTIFY: &str = "notify"; +const LOG_SUBSYSTEM_GLOBAL: &str = "global"; +const EVENT_NOTIFY_GLOBAL_STATE: &str = "notify_global_state"; /// Initialize the global notification system with the given configuration. /// This function should only be called once throughout the application life cycle. @@ -102,7 +105,13 @@ pub mod notifier_global { // If the notification system itself cannot be retrieved, it will be returned directly Some(sys) => sys, None => { - error!("Notification system is not initialized."); + error!( + event = EVENT_NOTIFY_GLOBAL_STATE, + component = LOG_COMPONENT_NOTIFY, + subsystem = LOG_SUBSYSTEM_GLOBAL, + state = "uninitialized", + "notify global state" + ); return; } }; diff --git a/crates/notify/src/integration.rs b/crates/notify/src/integration.rs index 6db3aa9ef..d84a01838 100644 --- a/crates/notify/src/integration.rs +++ b/crates/notify/src/integration.rs @@ -372,7 +372,7 @@ impl Drop for NotificationSystem { component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_INTEGRATION, state = "dropping", - "Notification system instance is being dropped" + "notify system integration state" ); let snapshot = self.snapshot_metrics(); @@ -394,7 +394,7 @@ impl Drop for NotificationSystem { metric_name = name, metric_value = value, metric_kind = if is_gauge { "gauge" } else { "counter" }, - "Notification shutdown metric snapshot" + "notify system integration state" ); } @@ -406,7 +406,7 @@ impl Drop for NotificationSystem { subsystem = LOG_SUBSYSTEM_INTEGRATION, status_key = %key, status_value = %value, - "Notification system status snapshot" + "notify system integration state" ); } } diff --git a/crates/notify/src/notifier.rs b/crates/notify/src/notifier.rs index c6c62f34e..a7244dc73 100644 --- a/crates/notify/src/notifier.rs +++ b/crates/notify/src/notifier.rs @@ -96,7 +96,7 @@ impl EventNotifier { component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_DISPATCH, state = "targets_cleared", - "Removed all notify targets" + "notify runtime lifecycle" ); } @@ -136,7 +136,7 @@ impl EventNotifier { bucket = %bucket_name, object = %object_key, target_count = target_ids_len, - "Dispatching notify event" + "notify dispatch" ); for target_id in target_ids { // `get` now returns Option> @@ -166,7 +166,7 @@ impl EventNotifier { subsystem = LOG_SUBSYSTEM_DISPATCH, target_id = %target_for_task.id(), deferred = is_deferred, - "Prepared notify target dispatch" + "notify dispatch" ); // Use cloned data in closures to avoid borrowing conflicts // Create an EntityTarget from the event @@ -217,7 +217,7 @@ impl EventNotifier { subsystem = LOG_SUBSYSTEM_DISPATCH, target_id = %target_name_for_task, deferred = is_deferred, - "Completed notify target dispatch" + "notify dispatch" ); } }); @@ -256,7 +256,7 @@ impl EventNotifier { subsystem = LOG_SUBSYSTEM_DISPATCH, bucket = %bucket_name, target_count = target_ids_len, - "Finished notify dispatch fan-out" + "notify dispatch" ); } @@ -284,7 +284,7 @@ impl EventNotifier { subsystem = LOG_SUBSYSTEM_DISPATCH, state = "targets_initialized", target_count = target_list_guard.len(), - "Initialized notify runtime targets" + "notify runtime lifecycle" ); Ok(()) } diff --git a/crates/notify/src/rule_engine.rs b/crates/notify/src/rule_engine.rs index 792f091ca..96c0e3de5 100644 --- a/crates/notify/src/rule_engine.rs +++ b/crates/notify/src/rule_engine.rs @@ -69,7 +69,7 @@ impl NotifyRuleEngine { bucket = %bucket, state = "updated", event_count, - "Updated bucket notification rules" + "notify bucket rules state" ); } @@ -85,7 +85,7 @@ impl NotifyRuleEngine { subsystem = LOG_SUBSYSTEM_RULE_ENGINE, bucket = %bucket, state = "removed", - "Removed bucket notification rules" + "notify bucket rules state" ); } } diff --git a/crates/notify/src/runtime_facade.rs b/crates/notify/src/runtime_facade.rs index 098279d22..6624b5257 100644 --- a/crates/notify/src/runtime_facade.rs +++ b/crates/notify/src/runtime_facade.rs @@ -66,7 +66,7 @@ impl NotifyRuntimeFacade { subsystem = LOG_SUBSYSTEM_RUNTIME, target_id = %target_id, state = "replay_started", - "Started notify replay worker" + "notify runtime lifecycle" ); } else { debug!( @@ -76,7 +76,7 @@ impl NotifyRuntimeFacade { target_id = %target_id, state = "replay_skipped", reason = "no_store_configured", - "Skipped notify replay worker startup" + "notify runtime lifecycle" ); } }), @@ -122,7 +122,7 @@ impl NotifyRuntimeFacade { component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_RUNTIME, state = "stopping", - "Stopping notification runtime" + "notify runtime lifecycle" ); let active_targets = self.replay_workers.read().await.len(); @@ -132,7 +132,7 @@ impl NotifyRuntimeFacade { subsystem = LOG_SUBSYSTEM_RUNTIME, state = "replay_stopping", active_targets, - "Stopping notify replay workers" + "notify runtime lifecycle" ); { @@ -160,7 +160,7 @@ impl NotifyRuntimeFacade { component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_RUNTIME, state = "stopped", - "Stopped notification runtime" + "notify runtime lifecycle" ); } } diff --git a/crates/obs/src/telemetry/dial9.rs b/crates/obs/src/telemetry/dial9.rs index 723406ccf..ddd59b4b3 100644 --- a/crates/obs/src/telemetry/dial9.rs +++ b/crates/obs/src/telemetry/dial9.rs @@ -203,7 +203,7 @@ impl Dial9SessionGuard { component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_DIAL9, state = "shutdown_requested", - "dial9 state changed" + "dial9 state" ); // TelemetryGuard handles flushing automatically when dropped } @@ -219,7 +219,7 @@ impl Drop for Dial9SessionGuard { component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_DIAL9, state = "flushed", - "dial9 state changed" + "dial9 state" ); } } diff --git a/crates/obs/src/telemetry/local.rs b/crates/obs/src/telemetry/local.rs index 2c4198b65..d5a7e9e88 100644 --- a/crates/obs/src/telemetry/local.rs +++ b/crates/obs/src/telemetry/local.rs @@ -175,7 +175,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo output_format = "json", logger_level, is_production, - "local logging state changed" + "local logging state" ); OtelGuard { @@ -293,7 +293,7 @@ fn init_file_logging_internal( stdout_mirror_enabled = stdout_guard.is_some(), is_production, logger_level, - "local logging state changed" + "local logging state" ); Ok(OtelGuard { @@ -482,7 +482,7 @@ pub fn spawn_cleanup_task( zstd_level, zstd_fallback_to_gzip, zstd_workers, - "log cleaner state changed" + "log cleaner state" ); tokio::spawn(async move { diff --git a/crates/protocols/src/ftps/driver.rs b/crates/protocols/src/ftps/driver.rs index bd6fc6ae7..7d6723f8e 100644 --- a/crates/protocols/src/ftps/driver.rs +++ b/crates/protocols/src/ftps/driver.rs @@ -655,7 +655,7 @@ where state = "requested", username = %masked_username, path = %path_str, - "ftps object delete state changed" + "FTPS delete requested" ); let (bucket, key) = self @@ -749,7 +749,7 @@ where username = %masked_username, path = %path_str, bucket = %bucket, - "ftps directory state changed" + "FTPS directory created" ); Ok(()) } @@ -793,7 +793,7 @@ where state = "removed", path = %path_str, bucket = %bucket, - "ftps directory state changed" + "FTPS directory removed" ); Ok(()) } @@ -807,7 +807,7 @@ where subsystem = LOG_SUBSYSTEM_FTPS_DRIVER, state = "already_removed", bucket = %bucket, - "ftps directory state changed" + "FTPS directory already removed" ); Ok(()) } else { @@ -877,7 +877,7 @@ where username = %MaskedAccessKey(&user.username), from = %from_str, to = %to_str, - "ftps rename state changed" + "FTPS rename unsupported" ); Err(Error::new( diff --git a/crates/protocols/src/ftps/server.rs b/crates/protocols/src/ftps/server.rs index 893aaa061..f4d973602 100644 --- a/crates/protocols/src/ftps/server.rs +++ b/crates/protocols/src/ftps/server.rs @@ -107,7 +107,7 @@ where bind_addr = %self.config.bind_addr, tls_enabled = self.config.tls_enabled, ftps_required = self.config.ftps_required, - "ftps server state changed" + "FTPS server starting" ); let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false); @@ -121,14 +121,14 @@ where // Configure passive ports for data connections if let Some(passive_ports) = &self.config.passive_ports { let range = self.config.parse_passive_ports()?; - info!( + debug!( event = EVENT_FTPS_CONFIG_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_FTPS_SERVER, state = "passive_ports_configured", passive_ports = %passive_ports, passive_port_range = ?range, - "ftps config state changed" + "FTPS passive ports configured" ); server_builder = server_builder.passive_ports(range); } else { @@ -137,19 +137,19 @@ where component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_FTPS_SERVER, result = "system_assigned_passive_ports", - "ftps config state changed" + "FTPS passive ports defaulted" ); } // Configure external IP address for passive mode if let Some(ref external_ip) = self.config.external_ip { - info!( + debug!( event = EVENT_FTPS_CONFIG_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_FTPS_SERVER, state = "external_ip_configured", external_ip = %external_ip, - "ftps config state changed" + "FTPS external IP configured" ); server_builder = server_builder.passive_host(external_ip.as_str()); } @@ -157,13 +157,13 @@ where // Configure both active and passive mode support use libunftp::options::ActivePassiveMode; server_builder = server_builder.active_passive_mode(ActivePassiveMode::ActiveAndPassive); - info!( + debug!( event = EVENT_FTPS_CONFIG_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_FTPS_SERVER, state = "active_passive_mode_enabled", mode = "active_and_passive", - "ftps config state changed" + "FTPS active/passive mode configured" ); // Configure FTPS / TLS @@ -175,7 +175,7 @@ where subsystem = LOG_SUBSYSTEM_FTPS_SERVER, state = "enabled", cert_dir = %cert_dir, - "ftps tls state changed" + "FTPS TLS enabled" ); let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir) @@ -197,12 +197,12 @@ where server_builder = server_builder.ftps_manual::(Arc::new(server_config)); if self.config.ftps_required { - info!( + debug!( event = EVENT_FTPS_TLS_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_FTPS_SERVER, state = "required", - "ftps tls state changed" + "FTPS TLS required" ); server_builder = server_builder.ftps_required(FtpsRequired::All, FtpsRequired::All); } @@ -218,7 +218,7 @@ where subsystem = LOG_SUBSYSTEM_FTPS_SERVER, state = "disabled", mode = "plain_ftp", - "ftps tls state changed" + "FTPS TLS disabled" ); } @@ -248,13 +248,13 @@ where let _ = reload_shutdown_tx.send(true); match result { Ok(Ok(())) => { - info!( + debug!( event = EVENT_FTPS_SERVER_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_FTPS_SERVER, state = "stopped", result = "ok", - "ftps server state changed" + "FTPS server stopped" ); Ok(()) } @@ -288,7 +288,7 @@ where component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_FTPS_SERVER, state = "shutdown_requested", - "ftps server state changed" + "FTPS shutdown requested" ); let _ = reload_shutdown_tx.send(true); // libunftp listen() is not easily cancellable gracefully without dropping the future. @@ -330,7 +330,7 @@ impl UserDetailProvider for FtpsUserDetailProvider { result = "iam_unavailable", phase = "user_detail", error = %e, - "ftps auth state changed" + "FTPS user-detail IAM unavailable" ); UserDetailError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e))) })?; @@ -344,7 +344,7 @@ impl UserDetailProvider for FtpsUserDetailProvider { phase = "user_detail", username = %masked_username, error = %e, - "ftps auth state changed" + "FTPS user-detail key check failed" ); UserDetailError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e))) })?; @@ -357,7 +357,7 @@ impl UserDetailProvider for FtpsUserDetailProvider { result = "identity_missing", phase = "user_detail", username = %masked_username, - "ftps auth state changed" + "FTPS user-detail identity missing" ); UserDetailError::UserNotFound { username: principal.username.clone(), @@ -406,7 +406,7 @@ impl Authenticator for FtpsAuthenticator { result = "iam_unavailable", phase = "authenticate", error = %e, - "ftps auth state changed" + "FTPS auth IAM unavailable" ); AuthenticationError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e))) })?; @@ -433,7 +433,7 @@ impl Authenticator for FtpsAuthenticator { phase = "authenticate", username = %masked_username, error = %e, - "ftps auth state changed" + "FTPS auth key check failed" ); AuthenticationError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e))) })?; @@ -446,7 +446,7 @@ impl Authenticator for FtpsAuthenticator { result = "invalid_access_key", phase = "authenticate", username = %masked_username, - "ftps auth state changed" + "FTPS auth rejected access key" ); return Err(AuthenticationError::BadUser); } @@ -459,7 +459,7 @@ impl Authenticator for FtpsAuthenticator { result = "identity_missing", phase = "authenticate", username = %masked_username, - "ftps auth state changed" + "FTPS auth identity missing" ); AuthenticationError::BadUser })?; @@ -472,7 +472,7 @@ impl Authenticator for FtpsAuthenticator { result = "invalid_secret_key", phase = "authenticate", username = %masked_username, - "ftps auth state changed" + "FTPS auth rejected secret key" ); return Err(AuthenticationError::BadPassword); } @@ -484,7 +484,7 @@ impl Authenticator for FtpsAuthenticator { result = "authenticated", phase = "authenticate", username = %masked_username, - "ftps auth state changed" + "FTPS auth accepted" ); Ok(Principal { username: username.to_string(), diff --git a/crates/protocols/src/sftp/driver.rs b/crates/protocols/src/sftp/driver.rs index 9b4259cf0..b1833009d 100644 --- a/crates/protocols/src/sftp/driver.rs +++ b/crates/protocols/src/sftp/driver.rs @@ -188,7 +188,7 @@ impl SftpDriver { peer = %self.session_context.source_ip, user = %MaskedAccessKey(&self.session_context.principal.user_identity.credentials.access_key), result = "read_only_rejected", - "sftp driver state changed" + "SFTP write rejected by read-only mode" ); return Err(SftpError::code(StatusCode::PermissionDenied)); } @@ -250,7 +250,7 @@ impl SftpDriver { op = op, timeout_secs = self.backend_op_timeout_secs, result = "timeout", - "sftp backend state changed" + "SFTP backend operation timed out" ); Err(SftpError::code(StatusCode::Failure)) } diff --git a/crates/protocols/src/sftp/server.rs b/crates/protocols/src/sftp/server.rs index 33e03fe26..dfbf8a1b3 100644 --- a/crates/protocols/src/sftp/server.rs +++ b/crates/protocols/src/sftp/server.rs @@ -255,7 +255,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc, state = "enabled", host_key_dir = %config.host_key_dir.display(), interval_secs, - "sftp host key reload state changed" + "SFTP host-key reload enabled" ); tokio::spawn(async move { @@ -265,13 +265,13 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc, loop { tokio::select! { _ = shutdown_token.cancelled() => { - info!( + debug!( event = EVENT_SFTP_HOST_KEY_RELOAD_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SFTP_SERVER, state = "stopped", host_key_dir = %config.host_key_dir.display(), - "sftp host key reload state changed" + "SFTP host-key reload stopped" ); break; } @@ -280,14 +280,14 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc, match holder.reload_from_config(&config).await { Ok(Some(host_key_count)) => { - info!( + debug!( event = EVENT_SFTP_HOST_KEY_RELOAD_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SFTP_SERVER, result = "reloaded", host_key_dir = %config.host_key_dir.display(), host_key_count, - "sftp host key reload state changed" + "SFTP host keys reloaded" ); } Ok(None) => { @@ -297,7 +297,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc, subsystem = LOG_SUBSYSTEM_SFTP_SERVER, result = "unchanged", host_key_dir = %config.host_key_dir.display(), - "sftp host key reload state changed" + "SFTP host keys unchanged" ); } Err(err) => { @@ -308,7 +308,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc, result = "reload_failed", host_key_dir = %config.host_key_dir.display(), err = %err, - "sftp host key reload state changed" + "SFTP host-key reload failed" ); } } @@ -398,7 +398,7 @@ where state = "listening", bind_addr = %self.config.bind_addr, read_only = self.config.read_only, - "sftp server state changed" + "SFTP server listening" ); let mut sessions: JoinSet<()> = JoinSet::new(); diff --git a/crates/protocols/src/sftp/write.rs b/crates/protocols/src/sftp/write.rs index e0072deca..d6b8a1a4b 100644 --- a/crates/protocols/src/sftp/write.rs +++ b/crates/protocols/src/sftp/write.rs @@ -75,7 +75,7 @@ pub(super) fn write_dispatch_byte_count(phase: &WritePhase, part_size: u64, satu component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SFTP_WRITE, result = "byte_count_overflow", - "sftp write state changed" + "SFTP write byte-count overflow" ); Err(SftpError::code(StatusCode::Failure)) } @@ -90,7 +90,7 @@ pub(super) fn write_dispatch_byte_count(phase: &WritePhase, part_size: u64, satu component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SFTP_WRITE, result = "handle_poisoned", - "sftp write state changed" + "SFTP write handle poisoned" ); Err(SftpError::code(StatusCode::Failure)) } @@ -115,7 +115,7 @@ pub(super) fn write_dispatch_append_bytes(phase: &mut WritePhase, data: &[u8]) - component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SFTP_WRITE, result = "append_on_failed_handle", - "sftp write state changed" + "SFTP write append rejected on failed handle" ); Err(SftpError::code(StatusCode::Failure)) } @@ -368,7 +368,7 @@ impl SftpDriver { key = %sanitise_control_bytes(key), attempt = attempt, state = "retrying_put_object", - "sftp write state changed", + "SFTP put_object retry scheduled", ); } @@ -414,7 +414,7 @@ impl SftpDriver { bucket = %sanitise_control_bytes(bucket), key = %sanitise_control_bytes(key), result = "retry_loop_fell_through", - "sftp write state changed", + "SFTP put_object retry loop fell through", ); Err(SftpError::code(StatusCode::Failure)) } @@ -459,7 +459,7 @@ impl SftpDriver { upload_id = %upload_id, part_number = part_number, result = "etag_missing", - "sftp multipart state changed" + "SFTP multipart part missing etag" ); SftpError::code(StatusCode::Failure) })?; @@ -543,7 +543,7 @@ impl SftpDriver { bucket = %sanitise_control_bytes(bucket), key = %sanitise_control_bytes(key), result = "upload_id_missing", - "sftp multipart state changed" + "SFTP multipart upload missing upload_id" ); SftpError::code(StatusCode::Failure) })?; @@ -708,7 +708,7 @@ impl SftpDriver { component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SFTP_WRITE, result = "buffering_phase_lost", - "sftp multipart state changed" + "SFTP multipart buffering phase lost" ); return Err(SftpError::code(StatusCode::Failure)); } @@ -772,7 +772,7 @@ impl SftpDriver { key = %key, limit = S3_MAX_MULTIPART_PARTS, result = "parts_limit_exceeded", - "sftp multipart state changed", + "SFTP multipart parts limit exceeded", ); let upload_id_for_fail = upload_id.clone(); let abort_authorized_for_fail = *abort_authorized; @@ -791,7 +791,7 @@ impl SftpDriver { component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SFTP_WRITE, result = "flush_without_streaming", - "sftp multipart state changed" + "SFTP multipart flush requested without streaming state" ); return Err(SftpError::code(StatusCode::Failure)); } @@ -817,7 +817,7 @@ impl SftpDriver { component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SFTP_WRITE, result = "post_upload_phase_missing", - "sftp multipart state changed" + "SFTP multipart post-upload phase missing" ); Err(SftpError::code(StatusCode::Failure)) } @@ -887,7 +887,7 @@ impl SftpDriver { err = ?abort_err, context = context, result = "abort_failed", - "sftp abort state changed", + "SFTP multipart abort failed", ); } } else { @@ -901,7 +901,7 @@ impl SftpDriver { access_key = %MaskedAccessKey(self.access_key()), context = context, result = "abort_skipped_unauthorized", - "sftp abort state changed", + "SFTP multipart abort skipped by policy", ); } } @@ -959,7 +959,7 @@ impl SftpDriver { upload_id = %upload_id, limit = S3_MAX_MULTIPART_PARTS, result = "final_part_limit_exceeded", - "sftp multipart state changed", + "SFTP multipart final-part limit exceeded", ); self.close_abort_or_skip(bucket, key, &upload_id, abort_authorized, "parts-limit breach") .await; @@ -1080,7 +1080,7 @@ impl SftpDriver { upload_id = %mp.upload_id, part_number = part_number, result = "etag_missing", - "sftp multipart copy state changed" + "SFTP multipart copy part missing etag" ); SftpError::code(StatusCode::Failure) })?; diff --git a/crates/protocols/src/webdav/driver.rs b/crates/protocols/src/webdav/driver.rs index 4c8707ebd..3b6b67891 100644 --- a/crates/protocols/src/webdav/driver.rs +++ b/crates/protocols/src/webdav/driver.rs @@ -423,7 +423,7 @@ where bucket = %bucket, object = %key, file_size, - "webdav object write state changed" + "WebDAV object flush completed" ); // Buffer already cleared by std::mem::take above Ok(()) @@ -438,7 +438,7 @@ where object = %key, file_size, error = %e, - "webdav object write state changed" + "WebDAV object flush failed" ); Err(FsError::GeneralFailure) } @@ -1219,7 +1219,7 @@ where result = "not_found", bucket = %bucket, error = %e, - "webdav bucket metadata state changed" + "WebDAV bucket metadata listed" ); Err(FsError::NotFound) } @@ -1279,7 +1279,7 @@ where state = "created", bucket = %bucket, object = %dir_key, - "webdav directory state changed" + "WebDAV directory marker created" ); return Ok(()); } @@ -1292,7 +1292,7 @@ where bucket = %bucket, object = %dir_key, error = %e, - "webdav directory state changed" + "WebDAV directory marker create failed" ); return Err(FsError::GeneralFailure); } @@ -1320,7 +1320,7 @@ where subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER, state = "bucket_created", bucket = %bucket, - "webdav directory state changed" + "WebDAV bucket created" ); Ok(()) } @@ -1332,7 +1332,7 @@ where state = "bucket_create_failed", bucket = %bucket, error = %e, - "webdav directory state changed" + "WebDAV bucket create failed" ); Err(FsError::GeneralFailure) } @@ -1465,7 +1465,7 @@ where state = "deleted", bucket = %bucket, object = %key, - "webdav object delete state changed" + "WebDAV object deleted" ); Ok(()) } @@ -1478,7 +1478,7 @@ where bucket = %bucket, object = %key, error = %e, - "webdav object delete state changed" + "WebDAV object delete failed" ); Err(FsError::GeneralFailure) } @@ -1529,7 +1529,7 @@ where dst_bucket = %dst_bucket, dst_object = %dst_key, error = %e, - "webdav rename state changed" + "WebDAV rename source delete failed" ); FsError::GeneralFailure })?; @@ -1543,7 +1543,7 @@ where src_object = %src_key, dst_bucket = %dst_bucket, dst_object = %dst_key, - "webdav rename state changed" + "WebDAV file renamed" ); return Ok(()); } @@ -1603,7 +1603,7 @@ where dst_bucket = %dst_bucket, dst_prefix = %dst_prefix, error = %e, - "webdav rename state changed" + "WebDAV rename directory listing failed" ); FsError::GeneralFailure })?; @@ -1652,7 +1652,7 @@ where src_object = %src_key, dst_bucket = %dst_bucket, dst_object = %dst_key, - "webdav rename state changed" + "WebDAV rename source not found" ); return Err(FsError::NotFound); } @@ -1666,7 +1666,7 @@ where src_object = %src_key, dst_bucket = %dst_bucket, dst_object = %dst_key, - "webdav rename state changed" + "WebDAV directory renamed" ); Ok(()) } diff --git a/crates/protocols/src/webdav/server.rs b/crates/protocols/src/webdav/server.rs index 64bee71e6..007ab349d 100644 --- a/crates/protocols/src/webdav/server.rs +++ b/crates/protocols/src/webdav/server.rs @@ -86,7 +86,7 @@ where bind_addr = %self.config.bind_addr, tls_enabled = self.config.tls_enabled, max_body_size = self.config.max_body_size, - "webdav server state changed" + "WebDAV server starting" ); let listener = TcpListener::bind(self.config.bind_addr).await?; @@ -96,7 +96,7 @@ where subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER, state = "listening", bind_addr = %self.config.bind_addr, - "webdav server state changed" + "WebDAV server listening" ); let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false); @@ -109,7 +109,7 @@ where subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER, state = "enabled", cert_dir = %cert_dir, - "webdav tls state changed" + "WebDAV TLS enabled" ); let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir) @@ -211,7 +211,7 @@ where component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER, state = "shutdown_requested", - "webdav server state changed" + "WebDAV shutdown requested" ); let _ = reload_shutdown_tx.send(true); break; @@ -225,7 +225,7 @@ where component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER, state = "stopped", - "webdav server state changed" + "WebDAV server stopped" ); Ok(()) } @@ -376,7 +376,7 @@ where result = "iam_unavailable", source_ip = %source_ip, error = %e, - "webdav auth state changed" + "WebDAV auth IAM unavailable" ); WebDavInitError::Server("Internal authentication service unavailable".to_string()) })?; @@ -403,7 +403,7 @@ where source_ip = %source_ip, access_key = %masked_access_key, error = %e, - "webdav auth state changed" + "WebDAV auth key check failed" ); WebDavInitError::Server("Authentication verification failed".to_string()) })?; @@ -416,7 +416,7 @@ where result = "invalid_access_key", source_ip = %source_ip, access_key = %masked_access_key, - "webdav auth state changed" + "WebDAV auth rejected access key" ); return Err(WebDavInitError::Server("Invalid credentials".to_string())); } @@ -429,7 +429,7 @@ where result = "identity_missing", source_ip = %source_ip, access_key = %masked_access_key, - "webdav auth state changed" + "WebDAV auth identity missing" ); WebDavInitError::Server("User not found".to_string()) })?; @@ -442,7 +442,7 @@ where result = "invalid_secret_key", source_ip = %source_ip, access_key = %masked_access_key, - "webdav auth state changed" + "WebDAV auth rejected secret key" ); return Err(WebDavInitError::Server("Invalid credentials".to_string())); } @@ -454,7 +454,7 @@ where result = "authenticated", source_ip = %source_ip, access_key = %masked_access_key, - "webdav auth state changed" + "WebDAV auth accepted" ); Ok(SessionContext::new( diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index a2d363ee4..65a4df30a 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -97,7 +97,7 @@ fn resolve_scanner_runtime_config() -> crate::runtime_config::ScannerRuntimeConf subsystem = LOG_SUBSYSTEM_RUNTIME, state = "resolve_failed", error = %err, - "Scanner runtime config resolution failed; using last applied config" + "Scanner runtime config fallback applied" ); current_scanner_runtime_config() } @@ -179,7 +179,7 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc) -> b path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), state = "startup_inspect_failed", error = %err, - "Scanner startup cache inspection failed; keeping configured startup delay" + "Scanner startup cache inspection failed" ); return false; } @@ -198,7 +198,7 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc) -> b path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), state = "startup_decode_failed", error = %err, - "Scanner startup cache decode failed; skipping startup delay" + "Scanner startup cache decode failed" ); true } @@ -222,7 +222,7 @@ async fn initial_scanner_startup_usage_state(storeapi: &Arc) -> (bool, subsystem = LOG_SUBSYSTEM_RUNTIME, state = "startup_bucket_inspect_failed", error = %err, - "Scanner startup bucket inspection failed; keeping configured startup delay" + "Scanner startup bucket inspection failed" ); false } @@ -418,7 +418,7 @@ async fn configure_scanner_defaults(storeapi: &Arc) { replication_active = features.replication, feature_inspection_failed = features.inspection_failed, state = "single_disk_defaults_applied", - "Scanner defaults updated for single-disk deployment" + "Scanner defaults applied" ); } else { set_scanner_default_speed(ScannerSpeed::Default); @@ -562,7 +562,7 @@ pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHeal path = %&*BACKGROUND_HEAL_INFO_PATH, state = "decode_failed", error = %e, - "Scanner background heal state decode failed" + "Scanner background heal decode failed" ); BackgroundHealInfo::default() }), @@ -577,7 +577,7 @@ pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHeal path = %&*BACKGROUND_HEAL_INFO_PATH, state = "read_failed", error = %e, - "Scanner background heal state read failed" + "Scanner background heal read failed" ); } BackgroundHealInfo::default() @@ -605,7 +605,7 @@ pub async fn save_background_heal_info(storeapi: Arc, info: BackgroundH path = %&*BACKGROUND_HEAL_INFO_PATH, state = "encode_failed", error = %e, - "Scanner background heal state encode failed" + "Scanner background heal encode failed" ); return; } @@ -621,7 +621,7 @@ pub async fn save_background_heal_info(storeapi: Arc, info: BackgroundH path = %&*BACKGROUND_HEAL_INFO_PATH, state = "save_failed", error = %e, - "Scanner background heal state save failed" + "Scanner background heal save failed" ); } } @@ -650,7 +650,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc subsystem = LOG_SUBSYSTEM_RUNTIME, state = "refresh_failed", error = %err, - "Scanner runtime config refresh failed; using last applied config" + "Scanner runtime config refresh failed" ); } let configured_cycle_interval = scanner_cycle_interval(); @@ -685,7 +685,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc cycle = cycle_info.current, scan_mode = ?scan_mode, state = "started", - "Scanner cycle state updated" + "Scanner cycle started" ); let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode); if let Some(new_heal_info) = background_heal_info_for_scan_start( @@ -730,7 +730,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc max_objects = ?cycle_budget.max_objects(), max_directories = ?cycle_budget.max_directories(), state = "budget_reached", - "Scanner cycle stopped after reaching budget" + "Scanner cycle budget reached" ); let budget_reason = cycle_budget.reason(); emit_scan_cycle_partial_with_source( @@ -751,7 +751,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc state = "failed", duration = ?now.elapsed(), error = %e, - "Scanner cycle state updated" + "Scanner cycle failed" ); emit_scan_cycle_complete(false, cycle_start.elapsed()); if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) { @@ -772,7 +772,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc max_objects = ?cycle_budget.max_objects(), max_directories = ?cycle_budget.max_directories(), state = "budget_reached", - "Scanner cycle stopped after reaching budget" + "Scanner cycle budget reached" ); global_metrics().finish_scan_cycle_work(cycle_work_start); let budget_reason = cycle_budget.reason(); @@ -806,7 +806,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc state = "completed", duration = ?now.elapsed(), cycles_total = cycle_info.cycle_completed.len(), - "Scanner cycle state updated" + "Scanner cycle completed" ); retain_recent_cycle_completions(&mut cycle_info.cycle_completed); @@ -830,14 +830,14 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc "Scanner state persistence failed" ); } else { - info!( + debug!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, path = %&*DATA_USAGE_BLOOM_NAME_PATH, state = "saved", - "Scanner state persisted" + "Scanner state saved" ); } } @@ -854,7 +854,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc) -> subsystem = LOG_SUBSYSTEM_RUNTIME, lock_name = "leader.lock", state = "acquired", - "Scanner leader lock state updated" + "Scanner leader lock acquired" ); guard } @@ -867,7 +867,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc) -> lock_name = "leader.lock", state = "contended", error = ?e, - "Scanner leader lock state updated" + "Scanner leader lock contended" ); return Ok(()); } @@ -881,7 +881,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc) -> lock_name = "leader.lock", state = "create_failed", error = %e, - "Scanner leader lock state updated" + "Scanner leader lock creation failed" ); return Ok(()); } @@ -977,7 +977,7 @@ pub async fn store_data_usage_in_backend( && let (Some(new_ts), Some(existing_ts)) = (data_usage_info.last_update, existing.last_update) && new_ts <= existing_ts { - info!( + debug!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, @@ -986,7 +986,7 @@ pub async fn store_data_usage_in_backend( incoming_last_update = ?new_ts, existing_last_update = ?existing_ts, state = "skip_stale_update", - "Scanner data usage persistence skipped stale update" + "Scanner stale data usage update skipped" ); continue; } @@ -1003,7 +1003,7 @@ pub async fn store_data_usage_in_backend( path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), state = "encode_failed", error = %e, - "Scanner data usage persistence encode failed" + "Scanner data usage encode failed" ); continue; } @@ -1040,7 +1040,7 @@ pub async fn store_data_usage_in_backend( path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), state = "save_failed", error = %e, - "Scanner data usage persistence failed" + "Scanner data usage save failed" ); } else { rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await; diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index cca58ab22..745ce52bf 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -66,7 +66,7 @@ use time::OffsetDateTime; use tokio::select; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, warn}; const LOG_COMPONENT_SCANNER: &str = "scanner"; const LOG_SUBSYSTEM_FOLDER: &str = "folder"; @@ -639,7 +639,7 @@ impl ScannerItem { subsystem = LOG_SUBSYSTEM_LIFECYCLE, object_path = %self.object_path(), state = "started", - "Scanner lifecycle action evaluation started" + "Scanner lifecycle evaluation started" ); let versioning_config = match BucketVersioningSys::get(&self.bucket).await { @@ -990,7 +990,7 @@ impl ScannerItem { let age = now - mod_time; age.whole_seconds().max(0) }); - info!( + debug!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_HEAL_ADMISSION, component = LOG_COMPONENT_SCANNER, @@ -1003,7 +1003,7 @@ impl ScannerItem { original_scan_mode = %HealScanMode::Deep.as_str(), effective_scan_mode = %scan_mode.as_str(), state = "downgraded_to_normal", - "Scanner heal admission downgraded deep scan during cooldown" + "Scanner heal deep scan downgraded" ); } @@ -1688,14 +1688,14 @@ impl FolderScanner { if found_objects && is_erasure().await { // If we found an object in erasure mode, we skip subdirs (only datadirs)... - info!( + debug!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_FOLDER_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_FOLDER, folder = %folder.name, state = "erasure_object_found", - "Scanner folder stopped descending after finding erasure object" + "Scanner folder descent stopped after erasure object" ); break; } diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index a9df682ea..de16828d2 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -900,7 +900,7 @@ impl ScannerIOCache for SetDisks { subsystem = LOG_SUBSYSTEM_IO, bucket = %bucket.name, state = "scan_started", - "Scanner disk bucket state updated" + "Scanner disk bucket scan started" ); let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]); @@ -910,13 +910,13 @@ impl ScannerIOCache for SetDisks { error!( target: "rustfs::scanner::io", event = EVENT_SCANNER_DISK_BUCKET_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - bucket = %bucket.name, - cache_name = %cache_name, - state = "cache_load_failed", - error = %e, - "Scanner disk bucket state updated" + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "cache_load_failed", + error = %e, + "Scanner disk bucket cache load failed" ); } @@ -942,7 +942,7 @@ impl ScannerIOCache for SetDisks { bucket = %bucket.name, cache_name = ?cache.info.name, state = "cache_ready", - "Scanner disk bucket state updated" + "Scanner disk bucket cache ready" ); let (updates_tx, mut updates_rx) = mpsc::channel::(1); @@ -997,7 +997,7 @@ impl ScannerIOCache for SetDisks { bucket = %bucket.name, state = "cancelled", error = %e, - "Scanner disk bucket state updated" + "Scanner disk bucket scan cancelled" ); } else { error!( @@ -1008,7 +1008,7 @@ impl ScannerIOCache for SetDisks { bucket = %bucket.name, state = "scan_failed", error = %e, - "Scanner disk bucket state updated" + "Scanner disk bucket scan failed" ); } @@ -1104,7 +1104,7 @@ impl ScannerIOCache for SetDisks { bucket = %bucket.name, cache_name = %cache.info.name, state = "scan_completed", - "Scanner disk bucket state updated" + "Scanner disk bucket scan completed" ); if let Err(e) = update_fut.await { @@ -1132,7 +1132,7 @@ impl ScannerIOCache for SetDisks { bucket = %bucket.name, cache_name = %cache.info.name, state = "send_root_entry", - "Scanner data usage stream progress updated" + "Scanner root entry publish started" ); if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await { @@ -1182,7 +1182,7 @@ impl ScannerIOCache for SetDisks { component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_IO, state = "set_scan_completed", - "Scanner set-level disk scan completed" + "Scanner set scan completed" ); Ok(()) diff --git a/crates/targets/src/store.rs b/crates/targets/src/store.rs index b568f9ba8..5b8ae9d5a 100644 --- a/crates/targets/src/store.rs +++ b/crates/targets/src/store.rs @@ -30,6 +30,10 @@ use std::{ use tracing::{debug, warn}; use uuid::Uuid; +const LOG_COMPONENT_TARGETS: &str = "targets"; +const LOG_SUBSYSTEM_STORE: &str = "store"; +const EVENT_TARGET_STORE_STATE: &str = "target_store_state"; + fn resolve_queue_store_compression_from_env_value(value: Option<&str>) -> bool { value .and_then(|value| value.parse::().ok().map(|state| state.is_enabled())) @@ -83,7 +87,14 @@ impl std::fmt::Display for Key { /// Parses a string into a Key pub fn parse_key(s: &str) -> Key { - debug!("Parsing key: {}", s); + debug!( + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + action = "parse_key", + key = %s, + "target store state" + ); let mut name = s.to_string(); let mut extension = String::new(); @@ -111,8 +122,16 @@ pub fn parse_key(s: &str) -> Key { } debug!( - "Parsed key - name: {}, extension: {}, item_count: {}, compress: {}", - name, extension, item_count, compress + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + action = "parse_key", + result = "parsed", + key_name = %name, + extension = %extension, + item_count, + compressed = compress, + "target store state" ); Key { @@ -276,7 +295,15 @@ impl QueueStore { .read() .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?; let path = self.file_path(key); - debug!("Reading file for key: {},path: {}", key, path.display()); + debug!( + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + action = "read_file", + key = %key, + path = %path.display(), + "target store state" + ); let data = std::fs::read(&path).map_err(|e| { if e.kind() == std::io::ErrorKind::NotFound { StoreError::NotFound @@ -342,7 +369,14 @@ impl QueueStore { std::fs::write(&path, data).map_err(StoreError::Io)?; } let modified = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64; - debug!("Wrote event to store: {}", key); + debug!( + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + action = "write_file", + key = %key, + "target store state" + ); Ok(modified) } @@ -409,7 +443,15 @@ where } } - debug!("Opened store at: {:?}", self.directory); + debug!( + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + state = "opened", + store_dir = ?self.directory, + entry_count = entries_map.len(), + "target store state" + ); Ok(()) } @@ -471,7 +513,14 @@ where } fn get_multiple(&self, key: &Self::Key) -> Result, Self::Error> { - debug!("Reading items from store for key: {}", key); + debug!( + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + action = "read_batch", + key = %key, + "target store state" + ); let data = self.get_raw(key)?; if data.is_empty() { return Err(StoreError::Deserialization("Cannot deserialize empty data".to_string())); @@ -500,10 +549,15 @@ where if items.len() < key.item_count && !items.is_empty() { // Partial read warn!( - "Expected {} items for key {}, but only found {}. Possible data corruption or incorrect item_count.", - key.item_count, - key, - items.len() + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + action = "read_batch", + key = %key, + expected_items = key.item_count, + actual_items = items.len(), + reason = "partial_batch_read", + "target store state" ); // Depending on strictness, this could be an error. } else if items.is_empty() { @@ -538,7 +592,15 @@ where Ok(()) => {} Err(e) if e.kind() == std::io::ErrorKind::NotFound => { // File already gone — still clean up the entries map to avoid stale keys. - warn!("File not found for key {} during del, cleaning up entries map.", key); + warn!( + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + action = "delete", + key = %key, + result = "file_missing", + "target store state" + ); } Err(e) => return Err(StoreError::Io(e)), } @@ -550,9 +612,25 @@ where .map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?; if entries.remove(&key.to_key_string()).is_none() { - debug!("Key {} not found in entries map during del, might have been already removed.", key); + debug!( + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + action = "delete", + key = %key, + result = "entry_missing", + "target store state" + ); } - debug!("Deleted event from store: {}", key.to_string()); + debug!( + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + action = "delete", + key = %key, + result = "deleted", + "target store state" + ); Ok(()) } @@ -580,7 +658,14 @@ where let entries = match self.entries.read() { Ok(entries) => entries, Err(_) => { - debug!("Failed to acquire read lock on entries for listing"); + debug!( + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + action = "list", + result = "lock_unavailable", + "target store state" + ); return Vec::new(); } }; @@ -597,7 +682,14 @@ where match self.entries.read() { Ok(entries) => entries.len(), Err(_) => { - debug!("Failed to acquire read lock on entries for len"); + debug!( + event = EVENT_TARGET_STORE_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_STORE, + action = "len", + result = "lock_unavailable", + "target store state" + ); 0 } } diff --git a/crates/targets/src/target/mqtt.rs b/crates/targets/src/target/mqtt.rs index e99ad4b18..0113882a6 100644 --- a/crates/targets/src/target/mqtt.rs +++ b/crates/targets/src/target/mqtt.rs @@ -60,6 +60,10 @@ const DEFAULT_MQTT_TLS_PORT: u16 = 8883; const DEFAULT_MQTT_WSS_PORT: u16 = 443; const MAX_MQTT_PACKET_SIZE_BYTES: u32 = 100 * 1024 * 1024; const DEFAULT_MQTT_WS_PATH_ALLOWLIST: &[&str] = &["/", "/mqtt"]; +const LOG_COMPONENT_TARGETS: &str = "targets"; +const LOG_SUBSYSTEM_MQTT: &str = "mqtt"; +const EVENT_MQTT_TARGET_STATE: &str = "mqtt_target_state"; +const EVENT_MQTT_DELIVERY_STATE: &str = "mqtt_delivery_state"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MQTTTlsPolicy { @@ -574,7 +578,14 @@ where Some(MAX_MQTT_PACKET_SIZE_BYTES), )?; - info!(target_id = %target_id, "MQTT target created"); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "created", + "mqtt target state" + ); Ok(MQTTTarget:: { id: target_id, args, @@ -593,7 +604,14 @@ where #[instrument(skip(self), fields(target_id = %self.id))] async fn init(&self) -> Result<(), TargetError> { if self.connected.load(Ordering::SeqCst) { - debug!(target_id = %self.id, "Already connected."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "already_connected", + "mqtt target state" + ); return Ok(()); } @@ -607,7 +625,14 @@ where let _ = bg_task_manager .init_cell .get_or_try_init(|| async { - debug!(target_id = %target_id_clone, "Initializing MQTT background task."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id_clone, + state = "background_task_initializing", + "mqtt target state" + ); // Use the latest MqttOptions (may have been updated by TLS reload coordinator). let mqtt_options: MqttOptions = (**pending_mqtt_options.load()).clone(); @@ -615,30 +640,67 @@ where let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build(); if let Err(e) = new_client.subscribe(&args_clone.topic, args_clone.qos).await { - error!(target_id = %target_id_clone, error = %e, "Failed to subscribe to MQTT topic during init"); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id_clone, + state = "subscribe_failed", + error = %e, + "mqtt target state" + ); return Err(TargetError::Network(format!("MQTT subscribe failed: {e}"))); } let mut rx_guard = bg_task_manager.initial_cancel_rx.lock().await; let cancel_rx = rx_guard.take().ok_or_else(|| { - error!(target_id = %target_id_clone, "MQTT cancel receiver already taken for task."); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id_clone, + state = "cancel_receiver_unavailable", + "mqtt target state" + ); TargetError::Configuration("MQTT cancel receiver already taken for task".to_string()) })?; drop(rx_guard); *client_arc.lock().await = Some(new_client.clone()); - info!(target_id = %target_id_clone, "Spawning MQTT event loop task."); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id_clone, + state = "event_loop_spawning", + "mqtt target state" + ); let task_handle = tokio::spawn(run_mqtt_event_loop(eventloop, connected_arc.clone(), target_id_clone.clone(), cancel_rx)); Ok(task_handle) }) .await .map_err(|e: TargetError| { - error!(target_id = %self.id, error = %e, "Failed to initialize MQTT background task"); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "background_task_init_failed", + error = %e, + "mqtt target state" + ); e })?; - debug!(target_id = %self.id, "MQTT background task initialized successfully."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "background_task_initialized", + "mqtt target state" + ); match tokio::time::timeout(DEFAULT_CONNECTION_TIMEOUT, async { while !self.connected.load(Ordering::SeqCst) { @@ -646,23 +708,51 @@ where && handle.is_finished() && !self.connected.load(Ordering::SeqCst) { - error!(target_id = %self.id, "MQTT background task exited prematurely before connection was established."); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "background_task_exited_before_connect", + "mqtt target state" + ); return Err(TargetError::Network("MQTT background task exited prematurely".to_string())); } tokio::time::sleep(Duration::from_millis(100)).await; } - debug!(target_id = %self.id, "MQTT target connected successfully."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "connected", + "mqtt target state" + ); Ok(()) }) .await { Ok(Ok(_)) => { - info!(target_id = %self.id, "MQTT target initialized and connected."); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "ready", + "mqtt target state" + ); Ok(()) } Ok(Err(e)) => Err(e), Err(_) => { - error!(target_id = %self.id, "Timeout waiting for MQTT connection after task spawn."); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "connect_timeout", + "mqtt target state" + ); Err(TargetError::Network("Timeout waiting for MQTT connection".to_string())) } } @@ -680,12 +770,16 @@ where .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?; debug!( - target = %self.id, + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, bucket = %meta.bucket_name, object = %meta.object_name, event = %meta.event_name, payload_len = body.len(), - "Sending MQTT payload" + state = "publishing", + "mqtt delivery state" ); client @@ -693,7 +787,16 @@ where .await .map_err(|e| { if e.to_string().contains("Connection") || e.to_string().contains("Timeout") { - warn!(target_id = %self.id, error = %e, "Publish failed due to connection issue, marking as not connected."); + warn!( + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "publish_failed", + reason = "connectivity_error", + error = %e, + "mqtt delivery state" + ); let err = TargetError::NotConnected; mark_target_disconnected_on_connectivity_error(&self.connected, &err); err @@ -702,7 +805,15 @@ where } })?; - debug!(target_id = %self.id, topic = %self.args.topic, "Event published to MQTT topic"); + debug!( + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + topic = %self.args.topic, + state = "published", + "mqtt delivery state" + ); self.delivery_counters.record_success(); Ok(()) } @@ -781,14 +892,28 @@ async fn run_mqtt_event_loop( target_id: TargetID, mut cancel_rx: mpsc::Receiver<()>, ) { - info!(target_id = %target_id, "MQTT event loop task started."); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "event_loop_started", + "mqtt target state" + ); let mut initial_connection_established = false; loop { tokio::select! { biased; _ = cancel_rx.recv() => { - info!(target_id = %target_id, "MQTT event loop task received cancellation signal. Shutting down."); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "cancellation_received", + "mqtt target state" + ); break; } polled_event_result = async { @@ -796,7 +921,14 @@ async fn run_mqtt_event_loop( match tokio::time::timeout(EVENT_LOOP_POLL_TIMEOUT, eventloop.poll()).await { Ok(result) => Some(result), Err(_) => { - debug!(target_id = %target_id, "MQTT poll timed out (EVENT_LOOP_POLL_TIMEOUT) while not connected or status pending."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "poll_timeout", + "mqtt target state" + ); connected_status.store(false, Ordering::SeqCst); None } @@ -810,15 +942,38 @@ async fn run_mqtt_event_loop( trace!(target_id = %target_id, event = ?notification, "Received MQTT event"); match notification { rumqttc::Event::Incoming(Incoming::ConnAck(_conn_ack)) => { - info!(target_id = %target_id, "MQTT connected (ConnAck)."); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "connack_received", + "mqtt target state" + ); connected_status.store(true, Ordering::SeqCst); initial_connection_established = true; } rumqttc::Event::Incoming(Incoming::Publish(publish)) => { - debug!(target_id = %target_id, topic = ?publish.topic, payload_len = publish.payload.len(), "Received message on subscribed topic."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "publish_received", + topic = ?publish.topic, + payload_len = publish.payload.len(), + "mqtt target state" + ); } rumqttc::Event::Incoming(Incoming::Disconnect(_)) => { - info!(target_id = %target_id, "Received Disconnect packet from broker. MQTT connection lost."); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "broker_disconnected", + "mqtt target state" + ); connected_status.store(false, Ordering::SeqCst); } rumqttc::Event::Incoming(Incoming::PingResp(_)) => { @@ -832,7 +987,14 @@ async fn run_mqtt_event_loop( } // Process other incoming packet types as needed (PubRec, PubRel, PubComp, UnsubAck) rumqttc::Event::Outgoing(Outgoing::Disconnect) => { - info!(target_id = %target_id, "MQTT outgoing disconnect initiated by client."); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "client_disconnect_requested", + "mqtt target state" + ); connected_status.store(false, Ordering::SeqCst); } rumqttc::Event::Outgoing(Outgoing::PingReq) => { @@ -848,7 +1010,15 @@ async fn run_mqtt_event_loop( } Some(Err(e)) => { connected_status.store(false, Ordering::SeqCst); - error!(target_id = %target_id, error = %e, "Error from MQTT event loop poll"); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "poll_failed", + error = %e, + "mqtt target state" + ); if matches!(e, ConnectionError::Io(_) | @@ -856,12 +1026,28 @@ async fn run_mqtt_event_loop( ConnectionError::ConnectionRefused(_) | ConnectionError::Tls(_) ) { - warn!(target_id = %target_id, error = %e, "MQTT connection error. Relying on rumqttc for reconnection if applicable."); + warn!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "reconnect_pending", + error = %e, + "mqtt target state" + ); } // Here you can decide whether to break loops based on the error type. // For example, for some unrecoverable errors. if is_fatal_mqtt_error(&e) { - error!(target_id = %target_id, error = %e, "Fatal MQTT error, terminating event loop."); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "fatal_error", + error = %e, + "mqtt target state" + ); break; } // rumqttc's eventloop.poll() may return Err and terminate after some errors, @@ -871,7 +1057,14 @@ async fn run_mqtt_event_loop( tokio::time::sleep(Duration::from_secs(1)).await; } None => { - warn!(target_id = %target_id, "Timeout during initial poll or pending state, will retry."); + warn!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "poll_retry_scheduled", + "mqtt target state" + ); continue; } } @@ -879,7 +1072,14 @@ async fn run_mqtt_event_loop( } } connected_status.store(false, Ordering::SeqCst); - info!(target_id = %target_id, "MQTT event loop task finished."); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %target_id, + state = "event_loop_finished", + "mqtt target state" + ); } /// Check whether the given MQTT connection error should be considered a fatal error, @@ -938,26 +1138,61 @@ where #[instrument(skip(self), fields(target_id = %self.id))] async fn is_active(&self) -> Result { - debug!(target_id = %self.id, "Checking if MQTT target is active."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "activity_check", + "mqtt target state" + ); if self.client.lock().await.is_none() && !self.connected.load(Ordering::SeqCst) { // Check if the background task is running and has not panicked if let Some(handle) = self.bg_task_manager.init_cell.get() && handle.is_finished() { - error!(target_id = %self.id, "MQTT background task has finished, possibly due to an error. Target is not active."); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "inactive_background_task_finished", + "mqtt target state" + ); return Err(TargetError::Network("MQTT background task terminated".to_string())); } - debug!(target_id = %self.id, "MQTT client not yet initialized or task not running/connected."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "inactive_client_unavailable", + "mqtt target state" + ); return Err(TargetError::Configuration( "MQTT client not available or not initialized/connected".to_string(), )); } if self.connected.load(Ordering::SeqCst) { - debug!(target_id = %self.id, "MQTT target is active (connected flag is true)."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "active", + "mqtt target state" + ); Ok(true) } else { - debug!(target_id = %self.id, "MQTT target is not connected (connected flag is false)."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "inactive_not_connected", + "mqtt target state" + ); Err(TargetError::NotConnected) } } @@ -973,14 +1208,36 @@ where }; if let Some(store) = &self.store { - debug!(target_id = %self.id, "Event saved to store start"); + debug!( + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "store_enqueue_started", + "mqtt delivery state" + ); match persist_queued_payload_to_store(store.as_ref(), &queued) { Ok(_) => { - debug!(target_id = %self.id, "Event saved to store for MQTT target successfully."); + debug!( + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "store_enqueued", + "mqtt delivery state" + ); Ok(()) } Err(e) => { - error!(target_id = %self.id, error = %e, "Failed to save event to store"); + error!( + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "store_enqueue_failed", + error = %e, + "mqtt delivery state" + ); self.delivery_counters.record_final_failure(); Err(e) } @@ -991,18 +1248,47 @@ where } if !self.connected.load(Ordering::SeqCst) { - warn!(target_id = %self.id, "Attempting to send directly but not connected; trying to init."); + warn!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "direct_send_requires_init", + "mqtt target state" + ); // Call the struct's init method, not the trait's default match MQTTTarget::::init(self).await { - Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."), + Ok(_) => debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "init_completed", + "mqtt target state" + ), Err(e) => { - error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target."); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "init_failed", + error = %e, + "mqtt target state" + ); self.delivery_counters.record_final_failure(); return Err(TargetError::NotConnected); } } if !self.connected.load(Ordering::SeqCst) { - error!(target_id = %self.id, "Cannot save (send directly) as target is not active after init attempt."); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "init_completed_not_connected", + "mqtt target state" + ); self.delivery_counters.record_final_failure(); return Err(TargetError::NotConnected); } @@ -1017,50 +1303,143 @@ where #[instrument(skip(self, body, meta), fields(target_id = %self.id))] async fn send_raw_from_store(&self, key: Key, body: Vec, meta: QueuedPayloadMeta) -> Result<(), TargetError> { - debug!(target_id = %self.id, ?key, "Attempting to send queued payload from store."); + debug!( + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + ?key, + state = "store_replay_started", + "mqtt delivery state" + ); if !self.is_enabled() { return Err(TargetError::Disabled); } if !self.connected.load(Ordering::SeqCst) { - warn!(target_id = %self.id, "Not connected; trying to init before sending from store."); + warn!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "store_replay_requires_init", + "mqtt target state" + ); match MQTTTarget::::init(self).await { - Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."), + Ok(_) => debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "init_completed", + "mqtt target state" + ), Err(e) => { - error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target."); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "init_failed", + error = %e, + "mqtt target state" + ); return Err(TargetError::NotConnected); } } if !self.connected.load(Ordering::SeqCst) { - error!(target_id = %self.id, "Cannot send from store as target is not active after init attempt."); + error!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "init_completed_not_connected", + "mqtt target state" + ); return Err(TargetError::NotConnected); } } - debug!(target_id = %self.id, ?key, "Sending event from store."); + debug!( + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + ?key, + state = "store_replay_publishing", + "mqtt delivery state" + ); if let Err(e) = self.send_body(body, &meta).await { if matches!(e, TargetError::NotConnected) { - warn!(target_id = %self.id, "Failed to send event from store: Not connected. Event remains in store."); + warn!( + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + ?key, + state = "store_replay_deferred", + reason = "not_connected", + "mqtt delivery state" + ); return Err(TargetError::NotConnected); } - error!(target_id = %self.id, error = %e, "Failed to send event from store with an unexpected error."); + error!( + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + ?key, + state = "store_replay_failed", + error = %e, + "mqtt delivery state" + ); return Err(e); } - debug!(target_id = %self.id, ?key, "Event sent from store successfully."); + debug!( + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + ?key, + state = "store_replay_published", + "mqtt delivery state" + ); Ok(()) } async fn close(&self) -> Result<(), TargetError> { - info!(target_id = %self.id, "Attempting to close MQTT target."); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "closing", + "mqtt target state" + ); if let Err(e) = self.bg_task_manager.cancel_tx.send(()).await { - warn!(target_id = %self.id, error = %e, "Failed to send cancel signal to MQTT background task. It might have already exited."); + warn!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "cancel_signal_failed", + error = %e, + "mqtt target state" + ); } // Wait for the task to finish if it was initialized if let Some(_task_handle) = self.bg_task_manager.init_cell.get() { - debug!(target_id = %self.id, "Waiting for MQTT background task to complete..."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "waiting_for_event_loop", + "mqtt target state" + ); // It's tricky to await here if close is called from a sync context or Drop // For async close, this is fine. Consider a timeout. // let _ = tokio::time::timeout(Duration::from_secs(5), task_handle.await).await; @@ -1069,9 +1448,24 @@ where } if let Some(client_instance) = self.client.lock().await.take() { - info!(target_id = %self.id, "Disconnecting MQTT client."); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "disconnecting_client", + "mqtt target state" + ); if let Err(e) = client_instance.disconnect().await { - warn!(target_id = %self.id, error = %e, "Error during MQTT client disconnect."); + warn!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "disconnect_failed", + error = %e, + "mqtt target state" + ); } } @@ -1083,7 +1477,14 @@ where } self.connected.store(false, Ordering::SeqCst); - info!(target_id = %self.id, "MQTT target close method finished."); + info!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "closed", + "mqtt target state" + ); Ok(()) } @@ -1097,7 +1498,14 @@ where async fn init(&self) -> Result<(), TargetError> { if !self.is_enabled() { - debug!(target_id = %self.id, "Target is disabled, skipping init."); + debug!( + event = EVENT_MQTT_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "disabled", + "mqtt target state" + ); return Ok(()); } // Call the internal init logic diff --git a/crates/targets/src/target/webhook.rs b/crates/targets/src/target/webhook.rs index 07edd269c..bd0a67415 100644 --- a/crates/targets/src/target/webhook.rs +++ b/crates/targets/src/target/webhook.rs @@ -45,6 +45,11 @@ use std::{ use tokio::sync::mpsc; use tracing::{debug, error, info, instrument, warn}; +const LOG_COMPONENT_TARGETS: &str = "targets"; +const LOG_SUBSYSTEM_WEBHOOK: &str = "webhook"; +const EVENT_WEBHOOK_TARGET_STATE: &str = "webhook_target_state"; +const EVENT_WEBHOOK_DELIVERY_STATE: &str = "webhook_delivery_state"; + /// Arguments for configuring a Webhook target #[derive(Clone)] pub struct WebhookArgs { @@ -190,7 +195,14 @@ where // Create a cancel channel let (cancel_sender, _) = mpsc::channel(1); - info!(target_id = %target_id.id, "Webhook target created"); + info!( + event = EVENT_WEBHOOK_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %target_id.id, + state = "created", + "webhook target state" + ); Ok(WebhookTarget:: { id: target_id, args, @@ -220,8 +232,13 @@ where // DANGEROUS: For testing only, skip all certificate verification client_builder = client_builder.danger_accept_invalid_certs(true); warn!( - "Webhook target '{}' is configured to skip TLS verification. This is insecure and should not be used in production.", - args.endpoint + event = EVENT_WEBHOOK_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + endpoint = %args.endpoint, + state = "tls_verification_skipped", + fallback = "danger_accept_invalid_certs", + "webhook target state" ); } else if !args.client_ca.is_empty() { // Use user-provided custom CA certificate @@ -301,10 +318,14 @@ where match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await { Ok(Ok(resp)) => { debug!( - target = %self.id, + event = EVENT_WEBHOOK_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id, status = %resp.status(), health_check_url = %health_check_url, - "Webhook health check request succeeded" + state = "reachability_probe_succeeded", + "webhook target state" ); Ok(true) } @@ -337,7 +358,15 @@ where // behavior matches real delivery while avoiding path-specific false negatives. match self.probe_reachability().await { Ok(true) => { - debug!("Webhook target {} reachability probe succeeded via {:?}", self.id, self.health_check_url); + debug!( + event = EVENT_WEBHOOK_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id, + health_check_url = ?self.health_check_url, + state = "reachable", + "webhook target state" + ); } Ok(false) => { return Err(TargetError::NotConnected); @@ -348,7 +377,14 @@ where } self.initialized.store(true, Ordering::SeqCst); - info!("Webhook target {} initialized", self.id); + info!( + event = EVENT_WEBHOOK_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id, + state = "initialized", + "webhook target state" + ); Ok(()) } @@ -357,14 +393,17 @@ where } async fn send_body(&self, body: Vec, meta: &QueuedPayloadMeta) -> Result<(), TargetError> { - info!("Webhook sending queued payload to target: {}", self.id); debug!( - target = %self.id, + event = EVENT_WEBHOOK_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id, bucket = %meta.bucket_name, object = %meta.object_name, - event = %meta.event_name, + payload_event = %meta.event_name, payload_len = body.len(), - "Sending webhook payload" + state = "sending", + "webhook delivery state" ); // When a TLS reload adapter is attached, it drives client rebuilds in @@ -406,7 +445,15 @@ where let status = resp.status(); if status.is_success() { - debug!("Event sent to webhook target: {}", self.id); + debug!( + event = EVENT_WEBHOOK_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id, + status = %status, + state = "sent", + "webhook delivery state" + ); self.delivery_counters.record_success(); Ok(()) } else if status == StatusCode::FORBIDDEN { @@ -454,13 +501,28 @@ where self.delivery_counters.record_final_failure(); return Err(e); } - debug!("Event saved to store for target: {}", self.id); + debug!( + event = EVENT_WEBHOOK_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id, + state = "store_enqueued", + "webhook delivery state" + ); Ok(()) } else { match self.init().await { Ok(_) => (), Err(e) => { - error!("Failed to initialize Webhook target {}: {}", self.id.id, e); + error!( + event = EVENT_WEBHOOK_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id.id, + state = "init_failed", + error = %e, + "webhook target state" + ); self.delivery_counters.record_final_failure(); return Err(TargetError::NotConnected); } @@ -474,13 +536,27 @@ where } async fn send_raw_from_store(&self, key: Key, body: Vec, meta: QueuedPayloadMeta) -> Result<(), TargetError> { - debug!("Sending queued payload from store for target: {}, key: {}", self.id, key); + debug!( + event = EVENT_WEBHOOK_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id, + key = %key, + state = "store_replay_started", + "webhook delivery state" + ); match self.init().await { - Ok(_) => { - debug!("Event sent to store for target: {}", self.name()); - } + Ok(_) => {} Err(e) => { - error!("Failed to initialize Webhook target {}: {}", self.id.id, e); + error!( + event = EVENT_WEBHOOK_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id.id, + state = "init_failed", + error = %e, + "webhook target state" + ); return Err(TargetError::NotConnected); } } @@ -492,7 +568,15 @@ where return Err(e); } - debug!("Event sent from store and deleted for target: {}", self.id); + debug!( + event = EVENT_WEBHOOK_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id, + key = %key, + state = "store_replay_sent", + "webhook delivery state" + ); Ok(()) } @@ -500,7 +584,14 @@ where // Send cancel signal to background tasks let _ = self.cancel_sender.try_send(()); // Adapter cleanup is done by the coordinator; no local state to reset. - info!("Webhook target closed: {}", self.id); + info!( + event = EVENT_WEBHOOK_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id, + state = "closed", + "webhook target state" + ); Ok(()) } @@ -515,7 +606,14 @@ where async fn init(&self) -> Result<(), TargetError> { if !self.is_enabled() { - debug!("Webhook target {} is disabled, skipping initialization", self.id); + debug!( + event = EVENT_WEBHOOK_TARGET_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_WEBHOOK, + target_id = %self.id, + state = "disabled", + "webhook target state" + ); return Ok(()); } self.init_inner().await diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index f885b9aae..28bc629de 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -65,6 +65,10 @@ use time::OffsetDateTime; use tracing::warn; use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_BUCKET_META: &str = "bucket_meta"; +const EVENT_ADMIN_BUCKET_META_STATE: &str = "admin_bucket_meta_state"; + #[derive(Debug, Default, serde::Deserialize)] pub struct ExportBucketMetadataQuery { pub bucket: String, @@ -106,7 +110,7 @@ impl Operation for ExportBucketMetadata { let query = { if let Some(query) = req.uri.query() { let input: ExportBucketMetadataQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { ExportBucketMetadataQuery::default() @@ -114,7 +118,7 @@ impl Operation for ExportBucketMetadata { }; let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -131,19 +135,19 @@ impl Operation for ExportBucketMetadata { .await?; let Some(store) = new_object_layer_fn() else { - return Err(s3_error!(InvalidRequest, "object store not init")); + return Err(s3_error!(InternalError, "object store is not initialized")); }; let buckets = if query.bucket.is_empty() { store .list_bucket(&BucketOptions::default()) .await - .map_err(|e| s3_error!(InternalError, "list buckets failed: {e}"))? + .map_err(|e| s3_error!(InternalError, "failed to list buckets: {e}"))? } else { let bucket = store .get_bucket_info(&query.bucket, &BucketOptions::default()) .await - .map_err(|e| s3_error!(InternalError, "get bucket failed: {e}"))?; + .map_err(|e| s3_error!(InternalError, "failed to load bucket: {e}"))?; vec![bucket] }; @@ -173,17 +177,17 @@ impl Operation for ExportBucketMetadata { if e == StorageError::ConfigNotFound { continue; } - return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); + return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}")); } }; - let config_json = - serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; + let config_json = serde_json::to_vec(&config) + .map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?; zip_writer .start_file(conf_path, SimpleFileOptions::default()) - .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?; + .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?; zip_writer .write_all(&config_json) - .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?; + .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?; } BUCKET_NOTIFICATION_CONFIG => { let config: s3s::dto::NotificationConfiguration = @@ -215,18 +219,18 @@ impl Operation for ExportBucketMetadata { if e == StorageError::ConfigNotFound { continue; } - return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); + return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}")); } }; let config_xml = - serialize(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; + serialize(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?; zip_writer .start_file(conf_path, SimpleFileOptions::default()) - .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?; + .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?; zip_writer .write_all(&config_xml) - .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?; + .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?; } BUCKET_TAGGING_CONFIG => { let config: Tagging = match metadata_sys::get_tagging_config(&bucket.name).await { @@ -235,18 +239,18 @@ impl Operation for ExportBucketMetadata { if e == StorageError::ConfigNotFound { continue; } - return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); + return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}")); } }; let config_xml = - serialize(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; + serialize(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?; zip_writer .start_file(conf_path, SimpleFileOptions::default()) - .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?; + .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?; zip_writer .write_all(&config_xml) - .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?; + .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?; } BUCKET_QUOTA_CONFIG_FILE => { let config: BucketQuota = match metadata_sys::get_quota_config(&bucket.name).await { @@ -376,7 +380,7 @@ impl Operation for ExportBucketMetadata { let zip_bytes = zip_writer .finish() - .map_err(|e| s3_error!(InternalError, "finish zip failed: {e}"))?; + .map_err(|e| s3_error!(InternalError, "failed to finalize export archive: {e}"))?; let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/zip".parse().unwrap()); header.insert(CONTENT_DISPOSITION, "attachment; filename=bucket-meta.zip".parse().unwrap()); @@ -399,7 +403,7 @@ impl Operation for ImportBucketMetadata { let _query = { if let Some(query) = req.uri.query() { let input: ImportBucketMetadataQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { ImportBucketMetadataQuery::default() @@ -407,7 +411,7 @@ impl Operation for ImportBucketMetadata { }; let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -427,24 +431,33 @@ impl Operation for ImportBucketMetadata { let body = match input.store_all_limited(MAX_BUCKET_METADATA_IMPORT_SIZE).await { Ok(b) => b, Err(e) => { - warn!("get body failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "body_read_failed", + error = ?e, + "admin bucket meta state" + ); return Err(s3_error!(InvalidRequest, "bucket metadata import body too large or failed to read")); } }; - let mut zip_reader = ZipArchive::new(Cursor::new(body)).map_err(|e| s3_error!(InternalError, "get body failed: {e}"))?; + let mut zip_reader = + ZipArchive::new(Cursor::new(body)).map_err(|e| s3_error!(InternalError, "failed to read import archive: {e}"))?; // First pass: read all file contents into memory let mut file_contents = Vec::new(); for i in 0..zip_reader.len() { let mut file = zip_reader .by_index(i) - .map_err(|e| s3_error!(InternalError, "get file failed: {e}"))?; + .map_err(|e| s3_error!(InternalError, "failed to read archive entry: {e}"))?; let file_path = file.name().to_string(); let mut content = Vec::new(); file.read_to_end(&mut content) - .map_err(|e| s3_error!(InternalError, "read file failed: {e}"))?; + .map_err(|e| s3_error!(InternalError, "failed to read archive entry content: {e}"))?; file_contents.push((file_path, content)); } @@ -455,7 +468,15 @@ impl Operation for ImportBucketMetadata { let file_path_split = file_path.split(SLASH_SEPARATOR).collect::>(); if file_path_split.len() < 2 { - warn!("file path is invalid: {}", file_path); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "invalid_file_path", + file_path = %file_path, + "admin bucket meta state" + ); continue; } @@ -474,17 +495,33 @@ impl Operation for ImportBucketMetadata { } Err(e) => { if e == StorageError::ConfigNotFound { - warn!("bucket metadata not found: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "bucket_metadata_missing", + error = %e, + "admin bucket meta state" + ); continue; } - warn!("get bucket metadata failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "bucket_metadata_load_failed", + error = %e, + "admin bucket meta state" + ); continue; } }; } let Some(store) = new_object_layer_fn() else { - return Err(s3_error!(InvalidRequest, "object store not init")); + return Err(s3_error!(InternalError, "object store is not initialized")); }; let update_at = OffsetDateTime::now_utc(); @@ -494,7 +531,15 @@ impl Operation for ImportBucketMetadata { let file_path_split = file_path.split(SLASH_SEPARATOR).collect::>(); if file_path_split.len() < 2 { - warn!("file path is invalid: {}", file_path); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "invalid_file_path", + file_path = %file_path, + "admin bucket meta state" + ); continue; } @@ -513,7 +558,16 @@ impl Operation for ImportBucketMetadata { ) .await { - warn!("create bucket failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "bucket_create_failed", + bucket = %bucket_name, + error = %e, + "admin bucket meta state" + ); continue; } @@ -527,7 +581,17 @@ impl Operation for ImportBucketMetadata { let config: BucketPolicy = match serde_json::from_slice(&content) { Ok(config) => config, Err(e) => { - warn!("deserialize config failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "config_deserialize_failed", + bucket = %bucket_name, + config_name = %conf_name, + error = %e, + "admin bucket meta state" + ); continue; } }; @@ -542,7 +606,17 @@ impl Operation for ImportBucketMetadata { } BUCKET_NOTIFICATION_CONFIG => { if let Err(e) = deserialize::(&content) { - warn!("deserialize config failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "config_deserialize_failed", + bucket = %bucket_name, + config_name = %conf_name, + error = %e, + "admin bucket meta state" + ); continue; } @@ -553,7 +627,17 @@ impl Operation for ImportBucketMetadata { BUCKET_LIFECYCLE_CONFIG => { if let Err(e) = deserialize::(&content) { - warn!("deserialize config failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "config_deserialize_failed", + bucket = %bucket_name, + config_name = %conf_name, + error = %e, + "admin bucket meta state" + ); continue; } @@ -564,7 +648,17 @@ impl Operation for ImportBucketMetadata { BUCKET_SSECONFIG => { if let Err(e) = deserialize::(&content) { - warn!("deserialize config failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "config_deserialize_failed", + bucket = %bucket_name, + config_name = %conf_name, + error = %e, + "admin bucket meta state" + ); continue; } @@ -575,7 +669,17 @@ impl Operation for ImportBucketMetadata { BUCKET_TAGGING_CONFIG => { if let Err(e) = deserialize::(&content) { - warn!("deserialize config failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "config_deserialize_failed", + bucket = %bucket_name, + config_name = %conf_name, + error = %e, + "admin bucket meta state" + ); continue; } @@ -586,7 +690,17 @@ impl Operation for ImportBucketMetadata { BUCKET_QUOTA_CONFIG_FILE => { if let Err(e) = serde_json::from_slice::(&content) { - warn!("deserialize config failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "config_deserialize_failed", + bucket = %bucket_name, + config_name = %conf_name, + error = %e, + "admin bucket meta state" + ); continue; } @@ -597,7 +711,17 @@ impl Operation for ImportBucketMetadata { OBJECT_LOCK_CONFIG => { if let Err(e) = deserialize::(&content) { - warn!("deserialize config failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "config_deserialize_failed", + bucket = %bucket_name, + config_name = %conf_name, + error = %e, + "admin bucket meta state" + ); continue; } @@ -608,7 +732,17 @@ impl Operation for ImportBucketMetadata { BUCKET_VERSIONING_CONFIG => { if let Err(e) = deserialize::(&content) { - warn!("deserialize config failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "config_deserialize_failed", + bucket = %bucket_name, + config_name = %conf_name, + error = %e, + "admin bucket meta state" + ); continue; } @@ -619,7 +753,17 @@ impl Operation for ImportBucketMetadata { BUCKET_REPLICATION_CONFIG => { if let Err(e) = deserialize::(&content) { - warn!("deserialize config failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "config_deserialize_failed", + bucket = %bucket_name, + config_name = %conf_name, + error = %e, + "admin bucket meta state" + ); continue; } @@ -630,7 +774,17 @@ impl Operation for ImportBucketMetadata { BUCKET_TARGETS_FILE => { if let Err(e) = serde_json::from_slice::(&content) { - warn!("deserialize config failed: {e}"); + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "import_bucket_metadata", + result = "config_deserialize_failed", + bucket = %bucket_name, + config_name = %conf_name, + error = %e, + "admin bucket meta state" + ); continue; } diff --git a/rustfs/src/admin/handlers/group.rs b/rustfs/src/admin/handlers/group.rs index ac1f5660f..f1f2d00ff 100644 --- a/rustfs/src/admin/handlers/group.rs +++ b/rustfs/src/admin/handlers/group.rs @@ -41,6 +41,10 @@ use serde::Deserialize; use serde_urlencoded::from_bytes; use tracing::warn; +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_GROUP: &str = "group"; +const EVENT_ADMIN_GROUP_STATE: &str = "admin_group_state"; + pub fn register_group_management_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( Method::GET, @@ -85,10 +89,17 @@ pub struct ListGroups {} #[async_trait::async_trait] impl Operation for ListGroups { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - warn!("handle ListGroups"); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "list_groups", + state = "requested", + "admin group state" + ); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -104,14 +115,24 @@ impl Operation for ListGroups { ) .await?; - let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InternalError, "iam is not initialized")); + }; let groups = iam_store.list_groups_load().await.map_err(|e| { - warn!("list groups failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "list_groups", + result = "load_failed", + error = ?e, + "admin group state" + ); S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) })?; - let body = serde_json::to_vec(&groups).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; + let body = serde_json::to_vec(&groups).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?; let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); @@ -124,10 +145,17 @@ pub struct GetGroup {} #[async_trait::async_trait] impl Operation for GetGroup { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - warn!("handle GetGroup"); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "get_group", + state = "requested", + "admin group state" + ); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -146,20 +174,31 @@ impl Operation for GetGroup { let query = { if let Some(query) = req.uri.query() { let input: GroupQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { GroupQuery::default() } }; - let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InternalError, "iam is not initialized")); + }; let g = iam_store.get_group_description(&query.group).await.map_err(|e| { - warn!("get group failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "get_group", + group = %query.group, + result = "load_failed", + error = ?e, + "admin group state" + ); iam_error_to_s3_error(e) })?; - let body = serde_json::to_vec(&g).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; + let body = serde_json::to_vec(&g).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?; let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); @@ -189,10 +228,17 @@ pub struct DeleteGroup {} #[async_trait::async_trait] impl Operation for DeleteGroup { async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - warn!("handle DeleteGroup"); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "delete_group", + state = "requested", + "admin group state" + ); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -210,22 +256,33 @@ impl Operation for DeleteGroup { let group = decode_delete_group_name(¶ms)?; - let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InternalError, "iam is not initialized")); + }; let updated_at = iam_store.remove_users_from_group(&group, vec![]).await.map_err(|e| { - warn!("delete group failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "delete_group", + group = %group, + result = "delete_failed", + error = ?e, + "admin group state" + ); match e { rustfs_iam::error::Error::GroupNotEmpty => { s3_error!(InvalidRequest, "group is not empty") } rustfs_iam::error::Error::InvalidArgument => { - s3_error!(InvalidArgument, "{e}") + s3_error!(InvalidArgument, "invalid group request: {e}") } _ => { if is_err_no_such_group(&e) { iam_error_to_s3_error(e) } else { - s3_error!(InternalError, "{e}") + s3_error!(InternalError, "failed to delete group: {e}") } } } @@ -248,7 +305,15 @@ impl Operation for DeleteGroup { }) .await { - warn!("site replication group delete hook failed, err: {err}"); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "delete_group", + result = "site_replication_hook_failed", + error = %err, + "admin group state" + ); } let mut header = HeaderMap::new(); @@ -289,10 +354,17 @@ pub struct SetGroupStatus {} #[async_trait::async_trait] impl Operation for SetGroupStatus { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - warn!("handle SetGroupStatus"); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "set_group_status", + state = "requested", + "admin group state" + ); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -311,7 +383,7 @@ impl Operation for SetGroupStatus { let query = { if let Some(query) = req.uri.query() { let input: GroupQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { GroupQuery::default() @@ -322,16 +394,38 @@ impl Operation for SetGroupStatus { return Err(s3_error!(InvalidArgument, "group is required")); } - let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InternalError, "iam is not initialized")); + }; let updated_at = if let Some(status) = query.status.as_deref() { match status { "enabled" => iam_store.set_group_status(&query.group, true).await.map_err(|e| { - warn!("enable group failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "set_group_status", + group = %query.group, + status = "enabled", + result = "update_failed", + error = ?e, + "admin group state" + ); iam_error_to_s3_error(e) })?, "disabled" => iam_store.set_group_status(&query.group, false).await.map_err(|e| { - warn!("enable group failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "set_group_status", + group = %query.group, + status = "disabled", + result = "update_failed", + error = ?e, + "admin group state" + ); iam_error_to_s3_error(e) })?, _ => { @@ -363,7 +457,15 @@ impl Operation for SetGroupStatus { }) .await { - warn!("site replication group status hook failed, err: {err}"); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "set_group_status", + result = "site_replication_hook_failed", + error = %err, + "admin group state" + ); } let mut header = HeaderMap::new(); @@ -377,10 +479,17 @@ pub struct UpdateGroupMembers {} #[async_trait::async_trait] impl Operation for UpdateGroupMembers { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - warn!("handle UpdateGroupMembers"); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "update_group_members", + state = "requested", + "admin group state" + ); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -400,17 +509,37 @@ impl Operation for UpdateGroupMembers { let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await { Ok(b) => b, Err(e) => { - warn!("get body failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "update_group_members", + result = "body_read_failed", + error = ?e, + "admin group state" + ); return Err(s3_error!(InvalidRequest, "group configuration body too large or failed to read")); } }; let args: GroupAddRemove = serde_json::from_slice(&body) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?; + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?; - warn!("UpdateGroupMembers args {:?}", args); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "update_group_members", + group = %args.group, + member_count = args.members.len(), + remove = args.is_remove, + state = "decoded", + "admin group state" + ); - let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InternalError, "iam is not initialized")); + }; for member in args.members.iter() { match iam_store.is_temp_user(member).await { @@ -433,7 +562,10 @@ impl Operation for UpdateGroupMembers { Ok(()) }) .unwrap_or_else(|| { - Err(S3Error::with_message(S3ErrorCode::InternalError, "get global cred failed".to_string())) + Err(S3Error::with_message( + S3ErrorCode::InternalError, + "failed to load global credentials".to_string(), + )) })?; } Err(e) => { @@ -445,29 +577,65 @@ impl Operation for UpdateGroupMembers { } let updated_at = if args.is_remove { - warn!("remove group members"); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "remove_group_members", + group = %args.group, + member_count = args.members.len(), + state = "requested", + "admin group state" + ); iam_store .remove_users_from_group(&args.group, args.members.clone()) .await .map_err(|e| { - warn!("remove group members failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "remove_group_members", + group = %args.group, + result = "update_failed", + error = ?e, + "admin group state" + ); iam_error_to_s3_error(e) })? } else { - warn!("add group members"); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "add_group_members", + group = %args.group, + member_count = args.members.len(), + state = "requested", + "admin group state" + ); if let Err(err) = iam_store.get_group_description(&args.group).await && is_err_no_such_group(&err) && has_space_be(&args.group) { - return Err(s3_error!(InvalidArgument, "not such group")); + return Err(s3_error!(InvalidArgument, "group not found")); } iam_store .add_users_to_group(&args.group, args.members.clone()) .await .map_err(|e| { - warn!("add group members failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "add_group_members", + group = %args.group, + result = "update_failed", + error = ?e, + "admin group state" + ); iam_error_to_s3_error(e) })? }; @@ -484,7 +652,15 @@ impl Operation for UpdateGroupMembers { }) .await { - warn!("site replication group membership hook failed, err: {err}"); + warn!( + event = EVENT_ADMIN_GROUP_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_GROUP, + action = "update_group_members", + result = "site_replication_hook_failed", + error = %err, + "admin group state" + ); } let mut header = HeaderMap::new(); diff --git a/rustfs/src/admin/handlers/kms_dynamic.rs b/rustfs/src/admin/handlers/kms_dynamic.rs index 990189253..a65b1b140 100644 --- a/rustfs/src/admin/handlers/kms_dynamic.rs +++ b/rustfs/src/admin/handlers/kms_dynamic.rs @@ -36,6 +36,7 @@ use tracing::{error, info, instrument, warn}; const KMS_CONFIG_PATH: &str = "config/kms_config.json"; const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_SUBSYSTEM_KMS: &str = "kms"; +const EVENT_ADMIN_KMS_DYNAMIC_STATE: &str = "admin_kms_dynamic_state"; fn kms_service_manager_from_context() -> std::sync::Arc { resolve_kms_runtime_service_manager().unwrap_or_else(|| { @@ -43,7 +44,8 @@ fn kms_service_manager_from_context() -> std::sync::Arc Result<(), String> { subsystem = LOG_SUBSYSTEM_KMS, event = "kms_config_persisted", storage_path = KMS_CONFIG_PATH, - "Persisted KMS configuration" + state = "config_persisted", + "admin kms dynamic state" ); Ok(()) } @@ -131,7 +134,8 @@ pub async fn load_kms_config() -> Option { subsystem = LOG_SUBSYSTEM_KMS, event = "kms_config_load_skipped", reason = "storage_uninitialized", - "Skipped KMS configuration load" + result = "config_load_skipped", + "admin kms dynamic state" ); return None; }; @@ -144,7 +148,8 @@ pub async fn load_kms_config() -> Option { subsystem = LOG_SUBSYSTEM_KMS, event = "kms_config_loaded", storage_path = KMS_CONFIG_PATH, - "Loaded persisted KMS configuration" + state = "config_loaded", + "admin kms dynamic state" ); Some(config) } @@ -154,8 +159,9 @@ pub async fn load_kms_config() -> Option { subsystem = LOG_SUBSYSTEM_KMS, event = "kms_config_deserialize_failed", storage_path = KMS_CONFIG_PATH, + result = "config_deserialize_failed", error = %e, - "Failed to deserialize KMS configuration" + "admin kms dynamic state" ); None } @@ -169,7 +175,7 @@ pub async fn load_kms_config() -> Option { event = "kms_config_loaded", state = "not_found", storage_path = KMS_CONFIG_PATH, - "Persisted KMS configuration not found" + "admin kms dynamic state" ); } else { warn!( @@ -177,8 +183,9 @@ pub async fn load_kms_config() -> Option { subsystem = LOG_SUBSYSTEM_KMS, event = "kms_config_load_failed", storage_path = KMS_CONFIG_PATH, + result = "config_load_failed", error = %e, - "Failed to load KMS configuration" + "admin kms dynamic state" ); } None @@ -263,8 +270,9 @@ impl Operation for ConfigureKmsHandler { subsystem = LOG_SUBSYSTEM_KMS, event = "kms_request_decode_failed", operation = "configure", + result = "request_decode_failed", error = %e, - "Failed to decode KMS admin request" + "admin kms dynamic state" ); return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}"))))); } @@ -277,7 +285,7 @@ impl Operation for ConfigureKmsHandler { event = "kms_service_state", operation = "configure", state = "requested", - "KMS service state changed" + "admin kms dynamic state" ); let service_manager = kms_service_manager_from_context(); @@ -303,7 +311,7 @@ impl Operation for ConfigureKmsHandler { operation = "configure", state = "persist_failed", error = %e, - "KMS service state changed" + "admin kms dynamic state" ); let status = service_manager.get_status().await; (false, error_msg, status) @@ -316,7 +324,7 @@ impl Operation for ConfigureKmsHandler { operation = "configure", state = "configured", status = ?status, - "KMS service state changed" + "admin kms dynamic state" ); (true, "KMS configured successfully".to_string(), status) } @@ -330,7 +338,7 @@ impl Operation for ConfigureKmsHandler { operation = "configure", state = "configure_failed", error = %e, - "KMS service state changed" + "admin kms dynamic state" ); let status = service_manager.get_status().await; (false, error_msg, status) @@ -346,7 +354,15 @@ impl Operation for ConfigureKmsHandler { let json_response = match serde_json::to_string(&response) { Ok(json) => json, Err(e) => { - error!("Failed to serialize response: {}", e); + error!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = EVENT_ADMIN_KMS_DYNAMIC_STATE, + operation = "configure", + result = "response_serialize_failed", + error = %e, + "admin kms dynamic state" + ); return Ok(S3Response::new(( StatusCode::INTERNAL_SERVER_ERROR, Body::from("Serialization error".to_string()), @@ -398,8 +414,9 @@ impl Operation for StartKmsHandler { subsystem = LOG_SUBSYSTEM_KMS, event = "kms_request_decode_failed", operation = "start", + result = "request_decode_failed", error = %e, - "Failed to decode KMS admin request" + "admin kms dynamic state" ); return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}"))))); } @@ -413,7 +430,7 @@ impl Operation for StartKmsHandler { operation = "start", state = "requested", force = start_request.force.unwrap_or(false), - "KMS service state changed" + "admin kms dynamic state" ); let service_manager = kms_service_manager_from_context(); @@ -427,7 +444,7 @@ impl Operation for StartKmsHandler { event = "kms_service_state", operation = "start", state = "already_running", - "KMS service state changed" + "admin kms dynamic state" ); let response = StartKmsResponse { success: false, @@ -437,7 +454,15 @@ impl Operation for StartKmsHandler { let json_response = match serde_json::to_string(&response) { Ok(json) => json, Err(e) => { - error!("Failed to serialize response: {}", e); + error!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = EVENT_ADMIN_KMS_DYNAMIC_STATE, + operation = "start", + result = "response_serialize_failed", + error = %e, + "admin kms dynamic state" + ); return Ok(S3Response::new(( StatusCode::INTERNAL_SERVER_ERROR, Body::from("Serialization error".to_string()), @@ -462,7 +487,7 @@ impl Operation for StartKmsHandler { operation = "restart", state = "running", status = ?status, - "KMS service state changed" + "admin kms dynamic state" ); (true, "KMS service restarted successfully".to_string(), status) } @@ -475,7 +500,7 @@ impl Operation for StartKmsHandler { operation = "restart", state = "start_failed", error = %e, - "KMS service state changed" + "admin kms dynamic state" ); let status = service_manager.get_status().await; (false, error_msg, status) @@ -490,7 +515,7 @@ impl Operation for StartKmsHandler { operation = "restart", state = "stop_failed", error = %e, - "KMS service state changed" + "admin kms dynamic state" ); let status = service_manager.get_status().await; (false, error_msg, status) @@ -508,7 +533,7 @@ impl Operation for StartKmsHandler { operation = "start", state = "running", status = ?status, - "KMS service state changed" + "admin kms dynamic state" ); (true, "KMS service started successfully".to_string(), status) } @@ -521,7 +546,7 @@ impl Operation for StartKmsHandler { operation = "start", state = "start_failed", error = %e, - "KMS service state changed" + "admin kms dynamic state" ); let status = service_manager.get_status().await; (false, error_msg, status) @@ -538,7 +563,15 @@ impl Operation for StartKmsHandler { let json_response = match serde_json::to_string(&response) { Ok(json) => json, Err(e) => { - error!("Failed to serialize response: {}", e); + error!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = EVENT_ADMIN_KMS_DYNAMIC_STATE, + operation = "start", + result = "response_serialize_failed", + error = %e, + "admin kms dynamic state" + ); return Ok(S3Response::new(( StatusCode::INTERNAL_SERVER_ERROR, Body::from("Serialization error".to_string()), @@ -579,7 +612,7 @@ impl Operation for StopKmsHandler { event = "kms_service_state", operation = "stop", state = "requested", - "KMS service state changed" + "admin kms dynamic state" ); let service_manager = kms_service_manager_from_context(); @@ -594,7 +627,7 @@ impl Operation for StopKmsHandler { operation = "stop", state = "stopped", status = ?status, - "KMS service state changed" + "admin kms dynamic state" ); (true, "KMS service stopped successfully".to_string(), status) } @@ -607,7 +640,7 @@ impl Operation for StopKmsHandler { operation = "stop", state = "stop_failed", error = %e, - "KMS service state changed" + "admin kms dynamic state" ); let status = service_manager.get_status().await; (false, error_msg, status) @@ -623,7 +656,15 @@ impl Operation for StopKmsHandler { let json_response = match serde_json::to_string(&response) { Ok(json) => json, Err(e) => { - error!("Failed to serialize response: {}", e); + error!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = EVENT_ADMIN_KMS_DYNAMIC_STATE, + operation = "stop", + result = "response_serialize_failed", + error = %e, + "admin kms dynamic state" + ); return Ok(S3Response::new(( StatusCode::INTERNAL_SERVER_ERROR, Body::from("Serialization error".to_string()), @@ -662,7 +703,8 @@ impl Operation for GetKmsStatusHandler { component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_KMS, event = "kms_status_requested", - "KMS status requested" + state = "status_requested", + "admin kms dynamic state" ); let service_manager = kms_service_manager_from_context(); @@ -699,13 +741,22 @@ impl Operation for GetKmsStatusHandler { backend_type = ?response.backend_type, healthy = response.healthy, has_config_summary = response.config_summary.is_some(), - "KMS status resolved" + state = "status_resolved", + "admin kms dynamic state" ); let json_response = match serde_json::to_string(&response) { Ok(json) => json, Err(e) => { - error!("Failed to serialize response: {}", e); + error!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = EVENT_ADMIN_KMS_DYNAMIC_STATE, + operation = "status", + result = "response_serialize_failed", + error = %e, + "admin kms dynamic state" + ); return Ok(S3Response::new(( StatusCode::INTERNAL_SERVER_ERROR, Body::from("Serialization error".to_string()), @@ -760,8 +811,9 @@ impl Operation for ReconfigureKmsHandler { subsystem = LOG_SUBSYSTEM_KMS, event = "kms_request_decode_failed", operation = "reconfigure", + result = "request_decode_failed", error = %e, - "Failed to decode KMS admin request" + "admin kms dynamic state" ); return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}"))))); } @@ -774,7 +826,7 @@ impl Operation for ReconfigureKmsHandler { event = "kms_service_state", operation = "reconfigure", state = "requested", - "KMS service state changed" + "admin kms dynamic state" ); let service_manager = kms_service_manager_from_context(); @@ -800,7 +852,7 @@ impl Operation for ReconfigureKmsHandler { operation = "reconfigure", state = "persist_failed", error = %e, - "KMS service state changed" + "admin kms dynamic state" ); let status = service_manager.get_status().await; (false, error_msg, status) @@ -813,7 +865,7 @@ impl Operation for ReconfigureKmsHandler { operation = "reconfigure", state = "reconfigured", status = ?status, - "KMS service state changed" + "admin kms dynamic state" ); (true, "KMS reconfigured and restarted successfully".to_string(), status) } @@ -827,7 +879,7 @@ impl Operation for ReconfigureKmsHandler { operation = "reconfigure", state = "reconfigure_failed", error = %e, - "KMS service state changed" + "admin kms dynamic state" ); let status = service_manager.get_status().await; (false, error_msg, status) @@ -843,7 +895,15 @@ impl Operation for ReconfigureKmsHandler { let json_response = match serde_json::to_string(&response) { Ok(json) => json, Err(e) => { - error!("Failed to serialize response: {}", e); + error!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = EVENT_ADMIN_KMS_DYNAMIC_STATE, + operation = "reconfigure", + result = "response_serialize_failed", + error = %e, + "admin kms dynamic state" + ); return Ok(S3Response::new(( StatusCode::INTERNAL_SERVER_ERROR, Body::from("Serialization error".to_string()), diff --git a/rustfs/src/admin/handlers/kms_keys.rs b/rustfs/src/admin/handlers/kms_keys.rs index dc61980c7..0b01325c8 100644 --- a/rustfs/src/admin/handlers/kms_keys.rs +++ b/rustfs/src/admin/handlers/kms_keys.rs @@ -33,6 +33,10 @@ use std::collections::HashMap; use tracing::{error, info}; use urlencoding; +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_KMS_KEYS: &str = "kms_keys"; +const EVENT_ADMIN_KMS_KEYS_STATE: &str = "admin_kms_keys_state"; + #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct CreateKmsKeyRequest { @@ -227,7 +231,7 @@ impl Operation for CreateKeyHandler { }; let Some(service) = kms_encryption_service_from_context().await else { - return Err(s3_error!(InternalError, "KMS service not initialized")); + return Err(s3_error!(InternalError, "kms service is not initialized")); }; // Extract key name from tags if provided @@ -259,7 +263,15 @@ impl Operation for CreateKeyHandler { Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) } Err(e) => { - error!("Failed to create KMS key: {}", e); + error!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "create_key", + result = "failed", + error = %e, + "admin kms keys state" + ); Err(s3_error!(InternalError, "failed to create key: {}", e)) } } @@ -290,11 +302,11 @@ impl Operation for DescribeKeyHandler { .await?; let Some(key_id) = extract_key_id(&req.uri) else { - return Err(s3_error!(InvalidRequest, "missing keyId parameter")); + return Err(s3_error!(InvalidRequest, "missing required parameter: 'keyId'")); }; let Some(service) = kms_encryption_service_from_context().await else { - return Err(s3_error!(InternalError, "KMS service not initialized")); + return Err(s3_error!(InternalError, "kms service is not initialized")); }; let request = DescribeKeyRequest { key_id: key_id.clone() }; @@ -314,7 +326,16 @@ impl Operation for DescribeKeyHandler { Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) } Err(e) => { - error!("Failed to describe KMS key {}: {}", key_id, e); + error!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "describe_key", + key_id = %key_id, + result = "failed", + error = %e, + "admin kms keys state" + ); Err(s3_error!(InternalError, "failed to describe key: {}", e)) } } @@ -451,7 +472,7 @@ impl Operation for ListKeysHandler { let marker = query_params.get("marker").cloned(); let Some(service) = kms_encryption_service_from_context().await else { - return Err(s3_error!(InternalError, "KMS service not initialized")); + return Err(s3_error!(InternalError, "kms service is not initialized")); }; let request = ListKeysRequest { @@ -478,7 +499,15 @@ impl Operation for ListKeysHandler { Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) } Err(e) => { - error!("Failed to list KMS keys: {}", e); + error!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "list_keys", + result = "failed", + error = %e, + "admin kms keys state" + ); Err(s3_error!(InternalError, "failed to list keys: {}", e)) } } @@ -544,7 +573,15 @@ impl Operation for GenerateDataKeyHandler { Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) } Err(e) => { - error!("Failed to generate data key: {}", e); + error!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "generate_data_key", + result = "failed", + error = %e, + "admin kms keys state" + ); Err(s3_error!(InternalError, "failed to generate data key: {}", e)) } } @@ -593,7 +630,7 @@ impl Operation for CreateKmsKeyHandler { let Some(service_manager) = kms_service_manager_from_context() else { let response = CreateKmsKeyResponse { success: false, - message: "KMS service manager not initialized".to_string(), + message: "kms service manager is not initialized".to_string(), key_id: "".to_string(), key_metadata: None, }; @@ -607,7 +644,7 @@ impl Operation for CreateKmsKeyHandler { let Some(manager) = service_manager.get_manager().await else { let response = CreateKmsKeyResponse { success: false, - message: "KMS service not running".to_string(), + message: "kms service is not running".to_string(), key_id: "".to_string(), key_metadata: None, }; @@ -633,10 +670,18 @@ impl Operation for CreateKmsKeyHandler { match manager.create_key(kms_request).await { Ok(kms_response) => { - info!("Created KMS key: {}", kms_response.key_id); + info!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "create_key", + key_id = %kms_response.key_id, + state = "completed", + "admin kms keys state" + ); let response = CreateKmsKeyResponse { success: true, - message: "Key created successfully".to_string(), + message: "key created successfully".to_string(), key_id: kms_response.key_id, key_metadata: Some(kms_response.key_metadata), }; @@ -650,10 +695,18 @@ impl Operation for CreateKmsKeyHandler { Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) } Err(e) => { - error!("Failed to create KMS key: {}", e); + error!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "create_key", + result = "failed", + error = %e, + "admin kms keys state" + ); let response = CreateKmsKeyResponse { success: false, - message: format!("Failed to create key: {e}"), + message: format!("failed to create key: {e}"), key_id: "".to_string(), key_metadata: None, }; @@ -720,7 +773,7 @@ impl Operation for DeleteKmsKeyHandler { let Some(key_id) = query_params.get("keyId") else { let response = DeleteKmsKeyResponse { success: false, - message: "missing keyId parameter".to_string(), + message: "missing required parameter: 'keyId'".to_string(), key_id: "".to_string(), deletion_date: None, }; @@ -747,7 +800,7 @@ impl Operation for DeleteKmsKeyHandler { let Some(service_manager) = kms_service_manager_from_context() else { let response = DeleteKmsKeyResponse { success: false, - message: "KMS service manager not initialized".to_string(), + message: "kms service manager is not initialized".to_string(), key_id: request.key_id, deletion_date: None, }; @@ -761,7 +814,7 @@ impl Operation for DeleteKmsKeyHandler { let Some(manager) = service_manager.get_manager().await else { let response = DeleteKmsKeyResponse { success: false, - message: "KMS service not running".to_string(), + message: "kms service is not running".to_string(), key_id: request.key_id, deletion_date: None, }; @@ -780,10 +833,18 @@ impl Operation for DeleteKmsKeyHandler { match manager.delete_key(kms_request).await { Ok(kms_response) => { - info!("Successfully deleted KMS key: {}", kms_response.key_id); + info!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "delete_key", + key_id = %kms_response.key_id, + state = "completed", + "admin kms keys state" + ); let response = DeleteKmsKeyResponse { success: true, - message: "Key deleted successfully".to_string(), + message: "key deleted successfully".to_string(), key_id: kms_response.key_id, deletion_date: kms_response.deletion_date, }; @@ -797,7 +858,16 @@ impl Operation for DeleteKmsKeyHandler { Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) } Err(e) => { - error!("Failed to delete KMS key {}: {}", request.key_id, e); + error!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "delete_key", + key_id = %request.key_id, + result = "failed", + error = %e, + "admin kms keys state" + ); let status = match &e { KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND, KmsError::InvalidOperation { .. } | KmsError::ValidationError { .. } => StatusCode::BAD_REQUEST, @@ -870,7 +940,7 @@ impl Operation for CancelKmsKeyDeletionHandler { let Some(key_id) = query_params.get("keyId") else { let response = CancelKmsKeyDeletionResponse { success: false, - message: "missing keyId parameter".to_string(), + message: "missing required parameter: 'keyId'".to_string(), key_id: "".to_string(), key_metadata: None, }; @@ -888,7 +958,7 @@ impl Operation for CancelKmsKeyDeletionHandler { let Some(service_manager) = kms_service_manager_from_context() else { let response = CancelKmsKeyDeletionResponse { success: false, - message: "KMS service manager not initialized".to_string(), + message: "kms service manager is not initialized".to_string(), key_id: request.key_id, key_metadata: None, }; @@ -902,7 +972,7 @@ impl Operation for CancelKmsKeyDeletionHandler { let Some(manager) = service_manager.get_manager().await else { let response = CancelKmsKeyDeletionResponse { success: false, - message: "KMS service not running".to_string(), + message: "kms service is not running".to_string(), key_id: request.key_id, key_metadata: None, }; @@ -919,10 +989,18 @@ impl Operation for CancelKmsKeyDeletionHandler { match manager.cancel_key_deletion(kms_request).await { Ok(kms_response) => { - info!("Cancelled deletion for KMS key: {}", kms_response.key_id); + info!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "cancel_delete_key", + key_id = %kms_response.key_id, + state = "completed", + "admin kms keys state" + ); let response = CancelKmsKeyDeletionResponse { success: true, - message: "Key deletion cancelled successfully".to_string(), + message: "key deletion cancelled successfully".to_string(), key_id: kms_response.key_id, key_metadata: Some(kms_response.key_metadata), }; @@ -936,7 +1014,16 @@ impl Operation for CancelKmsKeyDeletionHandler { Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) } Err(e) => { - error!("Failed to cancel deletion for KMS key {}: {}", request.key_id, e); + error!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "cancel_delete_key", + key_id = %request.key_id, + result = "failed", + error = %e, + "admin kms keys state" + ); let response = CancelKmsKeyDeletionResponse { success: false, message: format!("Failed to cancel key deletion: {e}"), @@ -995,7 +1082,7 @@ impl Operation for ListKmsKeysHandler { let Some(service_manager) = kms_service_manager_from_context() else { let response = ListKmsKeysResponse { success: false, - message: "KMS service manager not initialized".to_string(), + message: "kms service manager is not initialized".to_string(), keys: vec![], truncated: false, next_marker: None, @@ -1010,7 +1097,7 @@ impl Operation for ListKmsKeysHandler { let Some(manager) = service_manager.get_manager().await else { let response = ListKmsKeysResponse { success: false, - message: "KMS service not running".to_string(), + message: "kms service is not running".to_string(), keys: vec![], truncated: false, next_marker: None, @@ -1031,10 +1118,18 @@ impl Operation for ListKmsKeysHandler { match manager.list_keys(kms_request).await { Ok(kms_response) => { - info!("Listed {} KMS keys", kms_response.keys.len()); + info!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "list_keys", + state = "completed", + key_count = kms_response.keys.len(), + "admin kms keys state" + ); let response = ListKmsKeysResponse { success: true, - message: "Keys listed successfully".to_string(), + message: "keys listed successfully".to_string(), keys: kms_response.keys, truncated: kms_response.truncated, next_marker: kms_response.next_marker, @@ -1049,10 +1144,18 @@ impl Operation for ListKmsKeysHandler { Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) } Err(e) => { - error!("Failed to list KMS keys: {}", e); + error!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "list_keys", + result = "failed", + error = %e, + "admin kms keys state" + ); let response = ListKmsKeysResponse { success: false, - message: format!("Failed to list keys: {e}"), + message: format!("failed to list keys: {e}"), keys: vec![], truncated: false, next_marker: None, @@ -1103,7 +1206,7 @@ impl Operation for DescribeKmsKeyHandler { let Some(key_id) = params.get("key_id") else { let response = DescribeKmsKeyResponse { success: false, - message: "missing keyId parameter".to_string(), + message: "missing required parameter: 'keyId'".to_string(), key_metadata: None, }; let data = @@ -1116,7 +1219,7 @@ impl Operation for DescribeKmsKeyHandler { let Some(service_manager) = kms_service_manager_from_context() else { let response = DescribeKmsKeyResponse { success: false, - message: "KMS service manager not initialized".to_string(), + message: "kms service manager is not initialized".to_string(), key_metadata: None, }; let data = @@ -1129,7 +1232,7 @@ impl Operation for DescribeKmsKeyHandler { let Some(manager) = service_manager.get_manager().await else { let response = DescribeKmsKeyResponse { success: false, - message: "KMS service not running".to_string(), + message: "kms service is not running".to_string(), key_metadata: None, }; let data = @@ -1145,7 +1248,15 @@ impl Operation for DescribeKmsKeyHandler { match manager.describe_key(kms_request).await { Ok(kms_response) => { - info!("Described KMS key: {}", key_id); + info!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "describe_key", + key_id = %key_id, + state = "completed", + "admin kms keys state" + ); let response = DescribeKmsKeyResponse { success: true, message: "Key described successfully".to_string(), @@ -1161,7 +1272,16 @@ impl Operation for DescribeKmsKeyHandler { Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) } Err(e) => { - error!("Failed to describe KMS key {}: {}", key_id, e); + error!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = "describe_key", + key_id = %key_id, + result = "failed", + error = %e, + "admin kms keys state" + ); let status = match &e { KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND, KmsError::InvalidOperation { .. } => StatusCode::BAD_REQUEST, diff --git a/rustfs/src/admin/handlers/oidc.rs b/rustfs/src/admin/handlers/oidc.rs index cf33f5d48..178af0819 100644 --- a/rustfs/src/admin/handlers/oidc.rs +++ b/rustfs/src/admin/handlers/oidc.rs @@ -39,6 +39,10 @@ use time::OffsetDateTime; use tracing::{error, info, warn}; use url::Url; +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_OIDC: &str = "oidc"; +const EVENT_ADMIN_OIDC_STATE: &str = "admin_oidc_state"; + const OIDC_PUBLIC_PROVIDERS_SUFFIX: &str = "/v3/oidc/providers"; const OIDC_AUTHORIZE_SUFFIX: &str = "/v3/oidc/authorize/"; const OIDC_CALLBACK_SUFFIX: &str = "/v3/oidc/callback/"; @@ -448,7 +452,14 @@ impl Operation for OidcAuthorizeHandler { .await .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("authorize failed: {e}")))?; - info!("OIDC authorize redirect for provider '{}' to IdP", provider_id); + info!( + event = EVENT_ADMIN_OIDC_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, + provider_id = %provider_id, + state = "authorize_redirect", + "admin oidc state" + ); // Return 302 redirect let mut resp = S3Response::new((StatusCode::FOUND, Body::empty())); @@ -486,7 +497,15 @@ impl Operation for OidcCallbackHandler { // Check for error response from IdP if let Some(error) = extract_query_param(&req.uri, "error") { let desc = extract_query_param(&req.uri, "error_description").unwrap_or_default(); - warn!("OIDC callback received error from IdP: {} - {}", error, desc); + warn!( + event = EVENT_ADMIN_OIDC_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "idp_callback_error", + error_code = %error, + error_description = %desc, + "admin oidc state" + ); return Err(S3Error::with_message( S3ErrorCode::AccessDenied, format!("OIDC authentication failed: {error} - {desc}"), @@ -500,21 +519,38 @@ impl Operation for OidcCallbackHandler { // Exchange authorization code for tokens and extract claims let (claims, actual_provider_id, session, id_token) = oidc_sys.exchange_code(&state, &code, &redirect_uri).await.map_err(|e| { - error!("OIDC code exchange failed: {}", e); + error!( + event = EVENT_ADMIN_OIDC_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "code_exchange_failed", + error = %e, + "admin oidc state" + ); S3Error::with_message(S3ErrorCode::AccessDenied, format!("code exchange failed: {e}")) })?; info!( - "OIDC login successful: username='{}', email='{}', sub='{}' (provider: {})", - claims.username, claims.email, claims.sub, actual_provider_id + event = EVENT_ADMIN_OIDC_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, + provider_id = %actual_provider_id, + state = "authentication_succeeded", + "admin oidc state" ); // Map claims to policies and groups let (policies, groups) = oidc_sys.map_claims_to_policies(&actual_provider_id, &claims); info!( - "OIDC claim mapping: user='{}', policies={:?}, groups={:?}", - claims.username, policies, groups + event = EVENT_ADMIN_OIDC_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, + provider_id = %actual_provider_id, + policy_count = policies.len(), + group_count = groups.len(), + state = "claims_mapped", + "admin oidc state" ); // Generate STS credentials using the shared helper. @@ -568,7 +604,14 @@ impl Operation for OidcLogoutHandler { Ok(Some(url)) => url, Ok(None) => fallback_location.clone(), Err(err) => { - warn!("OIDC logout fallback triggered: {}", err); + warn!( + event = EVENT_ADMIN_OIDC_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "logout_fallback_triggered", + error = %err, + "admin oidc state" + ); fallback_location.clone() } }, diff --git a/rustfs/src/admin/handlers/policies.rs b/rustfs/src/admin/handlers/policies.rs index a8decfd29..f04c7387e 100644 --- a/rustfs/src/admin/handlers/policies.rs +++ b/rustfs/src/admin/handlers/policies.rs @@ -52,6 +52,7 @@ use url::form_urlencoded; const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_SUBSYSTEM_POLICY: &str = "policy"; +const EVENT_ADMIN_POLICY_STATE: &str = "admin_policy_state"; pub fn register_iam_policy_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( @@ -117,7 +118,7 @@ pub struct ListCannedPolicies {} impl Operation for ListCannedPolicies { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -136,23 +137,26 @@ impl Operation for ListCannedPolicies { let query = { if let Some(query) = req.uri.query() { let input: BucketQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { BucketQuery::default() } }; - let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InternalError, "iam is not initialized")); + }; let policies = iam_store.list_polices(&query.bucket).await.map_err(|e| { warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_list_failed", + event = EVENT_ADMIN_POLICY_STATE, bucket = %query.bucket, + result = "list_failed", error = ?e, - "Failed to list canned policies" + "admin policy state" ); S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) })?; @@ -162,7 +166,7 @@ impl Operation for ListCannedPolicies { .filter(|(_, v)| serde_json::to_string(v).is_ok()) .collect(); - let body = serde_json::to_vec(&kvs).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; + let body = serde_json::to_vec(&kvs).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?; let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); @@ -181,7 +185,7 @@ pub struct AddCannedPolicy {} impl Operation for AddCannedPolicy { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -200,7 +204,7 @@ impl Operation for AddCannedPolicy { let query = { if let Some(query) = req.uri.query() { let input: PolicyNameQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { PolicyNameQuery::default() @@ -208,11 +212,11 @@ impl Operation for AddCannedPolicy { }; if query.name.is_empty() { - return Err(s3_error!(InvalidArgument, "policy name is empty")); + return Err(s3_error!(InvalidArgument, "policy name is required")); } if has_space_be(&query.name) { - return Err(s3_error!(InvalidArgument, "policy name has space")); + return Err(s3_error!(InvalidArgument, "policy name contains spaces")); } let mut input = req.input; @@ -222,10 +226,11 @@ impl Operation for AddCannedPolicy { warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_body_read_failed", + event = EVENT_ADMIN_POLICY_STATE, policy = %query.name, + result = "body_read_failed", error = ?e, - "Failed to read policy request body" + "admin policy state" ); return Err(s3_error!(InvalidRequest, "policy configuration body too large or failed to read")); } @@ -235,27 +240,31 @@ impl Operation for AddCannedPolicy { warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_parse_failed", + event = EVENT_ADMIN_POLICY_STATE, policy = %query.name, + result = "parse_failed", error = ?e, - "Failed to parse canned policy" + "admin policy state" ); S3Error::with_message(S3ErrorCode::InvalidRequest, e.to_string()) })?; if policy.version.is_empty() { - return Err(s3_error!(InvalidArgument, "policy version is empty")); + return Err(s3_error!(InvalidArgument, "policy version is required")); } - let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InternalError, "iam is not initialized")); + }; let updated_at = iam_store.set_policy(&query.name, policy.clone()).await.map_err(|e| { warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_set_failed", + event = EVENT_ADMIN_POLICY_STATE, policy = %query.name, + result = "persist_failed", error = ?e, - "Failed to persist canned policy" + "admin policy state" ); S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) })?; @@ -272,7 +281,16 @@ impl Operation for AddCannedPolicy { }) .await { - warn!(policy = %query.name, error = ?err, "site replication policy add hook failed"); + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_POLICY, + event = EVENT_ADMIN_POLICY_STATE, + policy = %query.name, + action = "add_policy", + result = "site_replication_hook_failed", + error = ?err, + "admin policy state" + ); } let mut header = HeaderMap::new(); @@ -287,7 +305,7 @@ pub struct InfoCannedPolicy {} impl Operation for InfoCannedPolicy { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -306,7 +324,7 @@ impl Operation for InfoCannedPolicy { let query = { if let Some(query) = req.uri.query() { let input: PolicyNameQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { PolicyNameQuery::default() @@ -314,7 +332,7 @@ impl Operation for InfoCannedPolicy { }; if query.name.is_empty() { - return Err(s3_error!(InvalidArgument, "policy name is empty")); + return Err(s3_error!(InvalidArgument, "policy name is required")); } let policies = MappedPolicy::new(&query.name).to_slice(); @@ -322,21 +340,24 @@ impl Operation for InfoCannedPolicy { return Err(s3_error!(InvalidArgument, "too many policies")); } - let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InternalError, "iam is not initialized")); + }; let pd = iam_store.info_policy(&query.name).await.map_err(|e| { warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_info_failed", + event = EVENT_ADMIN_POLICY_STATE, policy = %query.name, + result = "info_failed", error = ?e, - "Failed to load canned policy" + "admin policy state" ); S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) })?; - let body = serde_json::to_vec(&pd).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; + let body = serde_json::to_vec(&pd).map_err(|e| s3_error!(InternalError, "failed to serialize response: {:?}", e))?; let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); @@ -350,7 +371,7 @@ pub struct RemoveCannedPolicy {} impl Operation for RemoveCannedPolicy { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -369,7 +390,7 @@ impl Operation for RemoveCannedPolicy { let query = { if let Some(query) = req.uri.query() { let input: PolicyNameQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { PolicyNameQuery::default() @@ -377,19 +398,22 @@ impl Operation for RemoveCannedPolicy { }; if query.name.is_empty() { - return Err(s3_error!(InvalidArgument, "policy name is empty")); + return Err(s3_error!(InvalidArgument, "policy name is required")); } - let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InternalError, "iam is not initialized")); + }; iam_store.delete_policy(&query.name, true).await.map_err(|e| { warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_delete_failed", + event = EVENT_ADMIN_POLICY_STATE, policy = %query.name, + result = "delete_failed", error = ?e, - "Failed to delete canned policy" + "admin policy state" ); S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) })?; @@ -403,7 +427,16 @@ impl Operation for RemoveCannedPolicy { }) .await { - warn!(policy = %query.name, error = ?err, "site replication policy delete hook failed"); + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_POLICY, + event = EVENT_ADMIN_POLICY_STATE, + policy = %query.name, + action = "delete_policy", + result = "site_replication_hook_failed", + error = ?err, + "admin policy state" + ); } let mut header = HeaderMap::new(); @@ -429,7 +462,7 @@ pub struct SetPolicyForUserOrGroup {} impl Operation for SetPolicyForUserOrGroup { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -448,7 +481,7 @@ impl Operation for SetPolicyForUserOrGroup { let query = { if let Some(query) = req.uri.query() { let input: SetPolicyForUserOrGroupQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { SetPolicyForUserOrGroupQuery::default() @@ -456,10 +489,12 @@ impl Operation for SetPolicyForUserOrGroup { }; if query.user_or_group.is_empty() { - return Err(s3_error!(InvalidArgument, "user or group is empty")); + return Err(s3_error!(InvalidArgument, "user or group is required")); } - let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InternalError, "iam is not initialized")); + }; if !query.is_group { match iam_store.is_temp_user(&query.user_or_group).await { @@ -473,11 +508,12 @@ impl Operation for SetPolicyForUserOrGroup { warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_target_validation_failed", + event = EVENT_ADMIN_POLICY_STATE, target = %query.user_or_group, check = "is_temp_user", + result = "target_validation_failed", error = ?err, - "Failed to validate policy target" + "admin policy state" ); return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string())); } @@ -485,28 +521,29 @@ impl Operation for SetPolicyForUserOrGroup { }; let Some(sys_cred) = get_global_action_cred() else { - return Err(s3_error!(InternalError, "get global action cred failed")); + return Err(s3_error!(InternalError, "failed to load global credentials")); }; if query.user_or_group == sys_cred.access_key { - return Err(s3_error!(InvalidArgument, "can't set policy for system user")); + return Err(s3_error!(InvalidArgument, "cannot set a policy for the system user")); } } if !query.is_group { if iam_store.get_user(&query.user_or_group).await.is_none() { - return Err(s3_error!(InvalidArgument, "user not exist")); + return Err(s3_error!(InvalidArgument, "user not found")); } } else { iam_store.get_group_description(&query.user_or_group).await.map_err(|e| { warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_target_validation_failed", + event = EVENT_ADMIN_POLICY_STATE, target = %query.user_or_group, check = "group_description", + result = "target_validation_failed", error = ?e, - "Failed to validate policy target" + "admin policy state" ); iam_error_to_s3_error(e) })?; @@ -519,12 +556,13 @@ impl Operation for SetPolicyForUserOrGroup { warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_mapping_set_failed", + event = EVENT_ADMIN_POLICY_STATE, target = %query.user_or_group, policy = %query.policy_name, is_group = query.is_group, + result = "mapping_set_failed", error = ?e, - "Failed to update policy mapping" + "admin policy state" ); S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) })?; @@ -546,7 +584,16 @@ impl Operation for SetPolicyForUserOrGroup { }) .await { - warn!(target = %query.user_or_group, error = ?err, "site replication policy mapping hook failed"); + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_POLICY, + event = EVENT_ADMIN_POLICY_STATE, + target = %query.user_or_group, + action = "set_policy_mapping", + result = "site_replication_hook_failed", + error = ?err, + "admin policy state" + ); } let mut header = HeaderMap::new(); @@ -923,11 +970,12 @@ async fn handle_builtin_policy_association(req: S3Request, is_attach: bool warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_target_validation_failed", + event = EVENT_ADMIN_POLICY_STATE, target = %assoc_req.user, check = "is_temp_user", + result = "target_validation_failed", error = ?err, - "Failed to validate policy association target" + "admin policy state" ); return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string())); } @@ -950,10 +998,11 @@ async fn handle_builtin_policy_association(req: S3Request, is_attach: bool warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_user_info_failed", + event = EVENT_ADMIN_POLICY_STATE, target = %assoc_req.user, + result = "user_info_failed", error = ?e, - "Failed to load policy association target" + "admin policy state" ); iam_error_to_s3_error(e) })?; @@ -964,10 +1013,11 @@ async fn handle_builtin_policy_association(req: S3Request, is_attach: bool warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_group_lookup_failed", + event = EVENT_ADMIN_POLICY_STATE, group = %assoc_req.group, + result = "group_lookup_failed", error = ?e, - "Failed to load policy association target" + "admin policy state" ); iam_error_to_s3_error(e) })?; @@ -988,12 +1038,13 @@ async fn handle_builtin_policy_association(req: S3Request, is_attach: bool warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, - event = "policy_mapping_set_failed", + event = EVENT_ADMIN_POLICY_STATE, target = %target_name, policy_count = updated_policies.len(), is_group, + result = "mapping_set_failed", error = ?e, - "Failed to update policy association" + "admin policy state" ); S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) })?; @@ -1015,7 +1066,16 @@ async fn handle_builtin_policy_association(req: S3Request, is_attach: bool }) .await { - warn!(target = %target_name, error = ?err, "site replication policy association hook failed"); + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_POLICY, + event = EVENT_ADMIN_POLICY_STATE, + target = %target_name, + action = if is_attach { "attach_policy" } else { "detach_policy" }, + result = "site_replication_hook_failed", + error = ?err, + "admin policy state" + ); } let policies_attached = if is_attach { changed_policies.clone() } else { Vec::new() }; diff --git a/rustfs/src/admin/handlers/quota.rs b/rustfs/src/admin/handlers/quota.rs index e690c66a3..b37719a28 100644 --- a/rustfs/src/admin/handlers/quota.rs +++ b/rustfs/src/admin/handlers/quota.rs @@ -33,6 +33,10 @@ use tokio::sync::RwLock; use tracing::{debug, info, warn}; use url::form_urlencoded; +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_QUOTA: &str = "quota"; +const EVENT_ADMIN_QUOTA_STATE: &str = "admin_quota_state"; + #[derive(Debug, Deserialize)] pub struct SetBucketQuotaRequest { pub quota: Option, @@ -231,7 +235,14 @@ pub fn register_quota_route(r: &mut S3Router) -> std::io::Result impl Operation for SetBucketQuotaHandler { #[tracing::instrument(skip_all)] async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { - warn!("handle SetBucketQuota"); + warn!( + event = EVENT_ADMIN_QUOTA_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_QUOTA, + action = "set_bucket_quota", + state = "requested", + "admin quota state" + ); let Some(ref cred) = req.credentials else { return Err(s3_error!(InvalidRequest, "authentication required")); @@ -276,7 +287,7 @@ impl Operation for SetBucketQuotaHandler { quota_checker .set_quota_config(&bucket, quota.clone()) .await - .map_err(|e| s3_error!(InternalError, "Failed to set quota: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to set quota: {}", e))?; // Get real-time usage from data usage system let current_usage = current_usage_from_context(&bucket).await; @@ -291,7 +302,7 @@ impl Operation for SetBucketQuotaHandler { quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(), }; - serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))? + serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))? }; Ok(S3Response::new((StatusCode::OK, Body::from(json)))) @@ -302,7 +313,14 @@ impl Operation for SetBucketQuotaHandler { impl Operation for GetBucketQuotaHandler { #[tracing::instrument(skip_all)] async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - warn!("handle GetBucketQuota"); + warn!( + event = EVENT_ADMIN_QUOTA_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_QUOTA, + action = "get_bucket_quota", + state = "requested", + "admin quota state" + ); let Some(ref cred) = req.credentials else { return Err(s3_error!(InvalidRequest, "authentication required")); @@ -327,21 +345,21 @@ impl Operation for GetBucketQuotaHandler { ) .await?; - let metadata_sys_lock = - bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?; + let metadata_sys_lock = bucket_metadata_from_context() + .ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?; let quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); let (quota, current_usage) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e { QuotaError::ConfigNotFound { .. } => { - s3_error!(NoSuchBucket, "Bucket not found: {}", bucket) + s3_error!(NoSuchBucket, "bucket not found: {}", bucket) } - _ => s3_error!(InternalError, "Failed to get quota: {}", e), + _ => s3_error!(InternalError, "failed to get quota: {}", e), })?; let json = if is_compat_get_bucket_quota_path(req.uri.path()) { serde_json::to_string(&compat_bucket_quota_response("a)) - .map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))? + .map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))? } else { let response = BucketQuotaResponse { bucket, @@ -350,7 +368,7 @@ impl Operation for GetBucketQuotaHandler { quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(), }; - serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))? + serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))? }; Ok(S3Response::new((StatusCode::OK, Body::from(json)))) @@ -361,7 +379,14 @@ impl Operation for GetBucketQuotaHandler { impl Operation for ClearBucketQuotaHandler { #[tracing::instrument(skip_all)] async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - warn!("handle ClearBucketQuota"); + warn!( + event = EVENT_ADMIN_QUOTA_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_QUOTA, + action = "clear_bucket_quota", + state = "requested", + "admin quota state" + ); let Some(ref cred) = req.credentials else { return Err(s3_error!(InvalidRequest, "authentication required")); @@ -385,10 +410,18 @@ impl Operation for ClearBucketQuotaHandler { return Err(s3_error!(InvalidRequest, "bucket name is required")); } - info!("Clearing quota for bucket: {}", bucket); + info!( + event = EVENT_ADMIN_QUOTA_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_QUOTA, + action = "clear_bucket_quota", + bucket = %bucket, + state = "clearing", + "admin quota state" + ); - let metadata_sys_lock = - bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?; + let metadata_sys_lock = bucket_metadata_from_context() + .ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?; let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); @@ -397,9 +430,17 @@ impl Operation for ClearBucketQuotaHandler { quota_checker .set_quota_config(&bucket, quota.clone()) .await - .map_err(|e| s3_error!(InternalError, "Failed to clear quota: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to clear quota: {}", e))?; - info!("Successfully cleared quota for bucket: {}", bucket); + info!( + event = EVENT_ADMIN_QUOTA_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_QUOTA, + action = "clear_bucket_quota", + bucket = %bucket, + state = "cleared", + "admin quota state" + ); // Get real-time usage from data usage system let current_usage = current_usage_from_context(&bucket).await; @@ -412,7 +453,7 @@ impl Operation for ClearBucketQuotaHandler { }; let json = - serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?; + serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?; Ok(S3Response::new((StatusCode::OK, Body::from(json)))) } @@ -422,7 +463,14 @@ impl Operation for ClearBucketQuotaHandler { impl Operation for GetBucketQuotaStatsHandler { #[tracing::instrument(skip_all)] async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - warn!("handle GetBucketQuotaStats"); + warn!( + event = EVENT_ADMIN_QUOTA_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_QUOTA, + action = "get_bucket_quota_stats", + state = "requested", + "admin quota state" + ); let Some(ref cred) = req.credentials else { return Err(s3_error!(InvalidRequest, "authentication required")); @@ -447,16 +495,16 @@ impl Operation for GetBucketQuotaStatsHandler { ) .await?; - let metadata_sys_lock = - bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?; + let metadata_sys_lock = bucket_metadata_from_context() + .ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?; let quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); let (quota, current_usage_opt) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e { QuotaError::ConfigNotFound { .. } => { - s3_error!(NoSuchBucket, "Bucket not found: {}", bucket) + s3_error!(NoSuchBucket, "bucket not found: {}", bucket) } - _ => s3_error!(InternalError, "Failed to get quota stats: {}", e), + _ => s3_error!(InternalError, "failed to get quota stats: {}", e), })?; let current_usage = current_usage_opt.unwrap_or(0); @@ -479,7 +527,7 @@ impl Operation for GetBucketQuotaStatsHandler { }; let json = - serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?; + serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?; Ok(S3Response::new((StatusCode::OK, Body::from(json)))) } @@ -489,7 +537,14 @@ impl Operation for GetBucketQuotaStatsHandler { impl Operation for CheckBucketQuotaHandler { #[tracing::instrument(skip_all)] async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { - warn!("handle CheckBucketQuota"); + warn!( + event = EVENT_ADMIN_QUOTA_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_QUOTA, + action = "check_bucket_quota", + state = "requested", + "admin quota state" + ); let Some(ref cred) = req.credentials else { return Err(s3_error!(InvalidRequest, "authentication required")); @@ -527,12 +582,19 @@ impl Operation for CheckBucketQuotaHandler { }; debug!( - "Checking quota for bucket: {}, operation: {}, size: {}", - bucket, request.operation_type, request.operation_size + event = EVENT_ADMIN_QUOTA_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_QUOTA, + action = "check_bucket_quota", + bucket = %bucket, + operation = %request.operation_type, + operation_size = request.operation_size, + state = "checking", + "admin quota state" ); - let metadata_sys_lock = - bucket_metadata_from_context().ok_or_else(|| s3_error!(InternalError, "Bucket metadata system not initialized"))?; + let metadata_sys_lock = bucket_metadata_from_context() + .ok_or_else(|| s3_error!(InternalError, "bucket metadata system is not initialized"))?; let quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); @@ -547,7 +609,7 @@ impl Operation for CheckBucketQuotaHandler { let result = quota_checker .check_quota_with_usage_reporting(&bucket, operation, request.operation_size, true) .await - .map_err(|e| s3_error!(InternalError, "Failed to check quota: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to check quota: {}", e))?; let response = CheckQuotaResponse { bucket, @@ -560,7 +622,7 @@ impl Operation for CheckBucketQuotaHandler { }; let json = - serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?; + serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?; Ok(S3Response::new((StatusCode::OK, Body::from(json)))) } diff --git a/rustfs/src/admin/handlers/rebalance.rs b/rustfs/src/admin/handlers/rebalance.rs index d9cd43cac..f4ec14823 100644 --- a/rustfs/src/admin/handlers/rebalance.rs +++ b/rustfs/src/admin/handlers/rebalance.rs @@ -41,7 +41,11 @@ use s3s::{ use serde::{Deserialize, Serialize}; use std::time::Duration; use time::OffsetDateTime; -use tracing::warn; +use tracing::info; + +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_REBALANCE: &str = "rebalance"; +const EVENT_ADMIN_REBALANCE_STATE: &str = "admin_rebalance_state"; pub fn register_rebalance_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( @@ -219,10 +223,17 @@ pub struct RebalanceStart {} impl Operation for RebalanceStart { #[tracing::instrument(skip_all)] async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - warn!("handle RebalanceStart"); + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "start", + state = "requested", + "admin rebalance state" + ); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "Failed to start rebalance: missing credentials")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -239,7 +250,7 @@ impl Operation for RebalanceStart { .await?; let Some(store) = new_object_layer_fn() else { - return Err(s3_error!(InternalError, "Failed to start rebalance: object layer not initialized")); + return Err(s3_error!(InternalError, "object layer is not initialized")); }; if store.pools.len() == 1 { @@ -247,47 +258,74 @@ impl Operation for RebalanceStart { } if store.is_decommission_running().await { - return Err(s3_error!( - InvalidRequest, - "Rebalance cannot be started, decommission is already in progress" - )); + return Err(s3_error!(InvalidRequest, "cannot start rebalance while decommission is in progress")); } if store.is_rebalance_conflicting_with_decommission().await { - return Err(s3_error!(OperationAborted, "Rebalance already in progress")); + return Err(s3_error!(OperationAborted, "rebalance is already in progress")); } let bucket_infos = store .list_bucket(&BucketOptions::default()) .await - .map_err(|e| s3_error!(InternalError, "Failed to list buckets for rebalance: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to list buckets for rebalance: {}", e))?; let buckets: Vec = bucket_infos.into_iter().map(|bucket| bucket.name).collect(); let id = match store.init_rebalance_meta(buckets).await { Ok(id) => id, Err(e) => { - return Err(s3_error!(InternalError, "Failed to initialize rebalance metadata: {}", e)); + return Err(s3_error!(InternalError, "failed to initialize rebalance metadata: {}", e)); } }; store .start_rebalance() .await - .map_err(|e| s3_error!(InternalError, "Failed to start rebalance: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to start rebalance: {}", e))?; - warn!("Rebalance started with id: {}", id); + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "start", + state = "started", + rebalance_id = %id, + "admin rebalance state" + ); if let Some(notification_sys) = get_global_notification_sys() { - warn!("RebalanceStart Loading rebalance meta start"); + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "start", + state = "propagation_started", + "admin rebalance state" + ); if let Err(err) = notification_sys.load_rebalance_meta(true).await { - warn!("rebalance start propagation failed after local state update: {err}"); + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "start", + result = "propagation_failed", + error = %err, + "admin rebalance state" + ); } - warn!("RebalanceStart Loading rebalance meta done"); + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "start", + state = "propagation_completed", + "admin rebalance state" + ); } let resp = RebalanceResp { id }; let data = serde_json::to_string(&resp) - .map_err(|e| s3_error!(InternalError, "Failed to serialize rebalance start response: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to serialize rebalance start response: {}", e))?; let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); @@ -303,10 +341,17 @@ pub struct RebalanceStatus {} impl Operation for RebalanceStatus { #[tracing::instrument(skip_all)] async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - warn!("handle RebalanceStatus"); + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "status", + state = "requested", + "admin rebalance state" + ); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "Failed to load rebalance status: missing credentials")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -323,26 +368,26 @@ impl Operation for RebalanceStatus { .await?; let Some(store) = new_object_layer_fn() else { - return Err(s3_error!(InternalError, "Failed to load rebalance status: object layer not initialized")); + return Err(s3_error!(InternalError, "object layer is not initialized")); }; if store.pools.is_empty() { - return Err(s3_error!(InternalError, "Failed to load rebalance status: no storage pools available")); + return Err(s3_error!(InternalError, "no storage pools are available")); } let first_pool = store .pools .first() .cloned() - .ok_or_else(|| s3_error!(InternalError, "Failed to load rebalance status: no storage pools available"))?; + .ok_or_else(|| s3_error!(InternalError, "no storage pools are available"))?; let mut meta = RebalanceMeta::new(); if let Err(err) = meta.load(first_pool).await { if err == StorageError::ConfigNotFound { - return Err(s3_error!(NoSuchResource, "Pool rebalance is not started")); + return Err(s3_error!(NoSuchResource, "pool rebalance is not started")); } - return Err(s3_error!(InternalError, "Failed to load rebalance metadata from pool 0: {}", err)); + return Err(s3_error!(InternalError, "failed to load rebalance metadata from pool 0: {}", err)); } // Compute disk usage percentage @@ -366,7 +411,7 @@ impl Operation for RebalanceStatus { }; let data = serde_json::to_string(&admin_status) - .map_err(|e| s3_error!(InternalError, "Failed to serialize rebalance status response: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to serialize rebalance status response: {}", e))?; let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); @@ -381,10 +426,17 @@ pub struct RebalanceStop {} impl Operation for RebalanceStop { #[tracing::instrument(skip_all)] async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - warn!("handle RebalanceStop"); + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "stop", + state = "requested", + "admin rebalance state" + ); let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "Failed to stop rebalance: missing credentials")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -401,37 +453,66 @@ impl Operation for RebalanceStop { .await?; let Some(store) = new_object_layer_fn() else { - return Err(s3_error!(InternalError, "Failed to stop rebalance: object layer not initialized")); + return Err(s3_error!(InternalError, "object layer is not initialized")); }; if !store.is_rebalance_conflicting_with_decommission().await { - return Err(s3_error!(NoSuchResource, "Pool rebalance is not started")); + return Err(s3_error!(NoSuchResource, "pool rebalance is not started")); } if let Some(notification_sys) = get_global_notification_sys() { notification_sys .stop_rebalance() .await - .map_err(|e| s3_error!(InternalError, "Failed to stop rebalance via notification system: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to stop rebalance via notification system: {}", e))?; } else { store .stop_rebalance() .await - .map_err(|e| s3_error!(InternalError, "Failed to stop rebalance: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to stop rebalance: {}", e))?; store .save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt) .await - .map_err(|e| s3_error!(InternalError, "Failed to persist rebalance stop metadata: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to persist rebalance stop metadata: {}", e))?; } - warn!("handle RebalanceStop save_rebalance_stats done "); + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "stop", + state = "local_stop_persisted", + "admin rebalance state" + ); if let Some(notification_sys) = get_global_notification_sys() { - warn!("handle RebalanceStop notification_sys load_rebalance_meta"); + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "stop", + state = "propagation_started", + "admin rebalance state" + ); if let Err(err) = notification_sys.load_rebalance_meta(false).await { - warn!("rebalance stop propagation failed after local state update: {err}"); + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "stop", + result = "propagation_failed", + error = %err, + "admin rebalance state" + ); } - warn!("handle RebalanceStop notification_sys load_rebalance_meta done"); + info!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "stop", + state = "propagation_completed", + "admin rebalance state" + ); } let mut header = HeaderMap::new(); diff --git a/rustfs/src/admin/handlers/service_account.rs b/rustfs/src/admin/handlers/service_account.rs index c440e0f3c..2e9e1620f 100644 --- a/rustfs/src/admin/handlers/service_account.rs +++ b/rustfs/src/admin/handlers/service_account.rs @@ -50,6 +50,7 @@ use url::form_urlencoded; const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_SUBSYSTEM_SERVICE_ACCOUNT: &str = "service_account"; +const EVENT_ADMIN_SERVICE_ACCOUNT_STATE: &str = "admin_service_account_state"; fn sr_session_policy_from_value(value: Option<&serde_json::Value>) -> S3Result { let Some(value) = value else { @@ -110,10 +111,11 @@ fn map_service_account_lookup_error(err: rustfs_iam::error::Error, action: &str) debug!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, - event = "service_account_lookup_failed", + event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE, action, + result = "lookup_failed", error = ?err, - "Service account lookup failed" + "admin service account state" ); if is_err_no_such_service_account(&err) { iam_error_to_s3_error(err) @@ -126,10 +128,11 @@ fn map_temp_account_lookup_error(err: rustfs_iam::error::Error, action: &str) -> debug!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, - event = "temporary_account_lookup_failed", + event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE, action, + result = "temporary_lookup_failed", error = ?err, - "Temporary account lookup failed" + "admin service account state" ); if is_err_no_such_temp_account(&err) { iam_error_to_s3_error(err) @@ -144,9 +147,10 @@ fn parse_service_account_policy(policy: &serde_json::Value) -> S3Result debug!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, - event = "service_account_policy_parse_failed", + event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE, + result = "policy_parse_failed", error = ?e, - "Failed to parse service account policy" + "admin service account state" ); match e { rustfs_policy::error::Error::PolicyError(rustfs_policy::policy::Error::NonResource) => { @@ -372,10 +376,11 @@ impl Operation for AddServiceAccount { debug!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, - event = "service_account_create_failed", + event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE, target_user = %target_user, + result = "create_failed", error = ?e, - "Failed to create service account" + "admin service account state" ); s3_error!(InternalError, "create service account failed, e: {:?}", e) })?; @@ -410,7 +415,16 @@ impl Operation for AddServiceAccount { }) .await { - warn!(access_key = %new_cred.access_key, error = ?err, "site replication add service account hook failed"); + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, + event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE, + access_key = %new_cred.access_key, + action = "create", + result = "site_replication_hook_failed", + error = ?err, + "admin service account state" + ); } let resp = AddServiceAccountResp { @@ -871,7 +885,14 @@ impl Operation for ListServiceAccount { check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key) .await .map_err(|e| { - debug!("check key failed: {e:?}"); + debug!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, + event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE, + result = "check_key_failed", + error = ?e, + "admin service account state" + ); s3_error!(InternalError, "check key failed") })?; @@ -927,7 +948,15 @@ impl Operation for ListServiceAccount { }; let service_accounts = iam_store.list_service_accounts(&target_account).await.map_err(|e| { - debug!("list service account failed: {e:?}"); + debug!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, + event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE, + target_user = %target_account, + result = "list_service_accounts_failed", + error = ?e, + "admin service account state" + ); s3_error!(InternalError, "list service account failed") })?; @@ -1114,7 +1143,15 @@ impl Operation for ListAccessKeysBulk { if list_sts_keys { let sts_keys = iam_store.list_sts_accounts(&user).await.map_err(|e| { - debug!("list sts account failed: {e:?}"); + debug!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, + event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE, + target_user = %user, + result = "list_sts_accounts_failed", + error = ?e, + "admin service account state" + ); s3_error!(InternalError, "list sts account failed") })?; @@ -1135,7 +1172,15 @@ impl Operation for ListAccessKeysBulk { if list_service_accounts { let service_accounts = iam_store.list_service_accounts(&user).await.map_err(|e| { - debug!("list service account failed: {e:?}"); + debug!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, + event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE, + target_user = %user, + result = "list_service_accounts_failed", + error = ?e, + "admin service account state" + ); s3_error!(InternalError, "list service account failed") })?; @@ -1279,7 +1324,16 @@ impl Operation for DeleteServiceAccount { }) .await { - warn!(access_key = %query.access_key, error = ?err, "site replication delete service account hook failed"); + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT, + event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE, + access_key = %query.access_key, + action = "delete", + result = "site_replication_hook_failed", + error = ?err, + "admin service account state" + ); } let mut header = HeaderMap::new(); diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index ed0f00045..8f3d68e56 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -93,6 +93,10 @@ use tracing::warn; use url::{Url, form_urlencoded}; use uuid::Uuid; +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_SITE_REPLICATION: &str = "site_replication"; +const EVENT_ADMIN_SITE_REPLICATION_STATE: &str = "admin_site_replication_state"; + const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json"; const SITE_REPL_ADD_SUCCESS: &str = "Requested sites were configured for replication successfully."; const SITE_REPL_EDIT_SUCCESS: &str = "Requested site was updated successfully."; @@ -2000,7 +2004,15 @@ async fn build_status_info(state: &SiteReplicationState, local_peer: &PeerInfo, reachable_peers.insert(deployment_id.clone()); } Err(err) => { - warn!(peer = %peer.endpoint, error = ?err, "site replication peer metainfo fetch failed"); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + peer = %peer.endpoint, + result = "peer_metainfo_fetch_failed", + error = ?err, + "admin site replication state" + ); site_infos.insert(deployment_id.clone(), SRInfo::default()); } } @@ -2536,7 +2548,14 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local let buckets = match store.list_bucket(&BucketOptions::default()).await { Ok(b) => b, Err(err) => { - warn!(error = ?err, "site replication backfill: failed to list buckets"); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "backfill_list_buckets_failed", + error = ?err, + "admin site replication state" + ); return; } }; @@ -2546,14 +2565,38 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local let name = &bucket.name; if let Err(err) = ensure_site_replication_bucket_versioning(name).await { - warn!(bucket = %name, error = ?err, "site replication backfill: versioning setup failed"); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + result = "backfill_versioning_setup_failed", + error = ?err, + "admin site replication state" + ); continue; } if let Err(err) = ensure_site_replication_bucket_targets(name, state, local_peer, None).await { - warn!(bucket = %name, error = ?err, "site replication backfill: targets setup failed"); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + result = "backfill_targets_setup_failed", + error = ?err, + "admin site replication state" + ); } if let Err(err) = ensure_site_replication_bucket_replication_config(name, state, local_peer).await { - warn!(bucket = %name, error = ?err, "site replication backfill: replication config setup failed"); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + result = "backfill_replication_config_setup_failed", + error = ?err, + "admin site replication state" + ); } // Broadcast the bucket to peers so they create it too (idempotent on the peer side). // Read the real lock_enabled flag so peers recreate the bucket with the same object-lock @@ -2561,12 +2604,29 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local let lock_enabled = match metadata_sys::get(name).await { Ok(bm) => bm.lock_enabled, Err(err) => { - warn!(bucket = %name, error = ?err, "site replication backfill: failed to read bucket metadata, assuming lock_enabled=false"); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + result = "backfill_bucket_metadata_read_failed", + fallback = "lock_enabled=false", + error = ?err, + "admin site replication state" + ); false } }; if let Err(err) = site_replication_make_bucket_hook(name, lock_enabled).await { - warn!(bucket = %name, error = ?err, "site replication backfill: make-bucket broadcast failed"); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + result = "backfill_make_bucket_broadcast_failed", + error = ?err, + "admin site replication state" + ); } // Kick a resync toward every remote peer so existing objects travel across. for peer in state.peers.values() { @@ -2575,8 +2635,16 @@ async fn backfill_existing_buckets_after_add(state: &SiteReplicationState, local } let result = start_site_bucket_resync(name, peer, &resync_id).await; if result.status == "failed" { - warn!(bucket = %name, peer = %peer.endpoint, detail = %result.err_detail, - "site replication backfill: resync kick failed"); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + bucket = %name, + peer = %peer.endpoint, + result = "backfill_resync_kick_failed", + detail = %result.err_detail, + "admin site replication state" + ); } } } @@ -3173,7 +3241,15 @@ impl Operation for SiteReplicationRemoveHandler { .await { let err_detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint)); - warn!(peer = %peer.endpoint, error = %err_detail, "site replication peer remove notification failed"); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + peer = %peer.endpoint, + result = "peer_remove_notification_failed", + error = %err_detail, + "admin site replication state" + ); peer_errors.push(err_detail); } } @@ -3767,7 +3843,15 @@ impl Operation for SRRotateServiceAccountHandler { send_peer_admin_request(&peer.endpoint, SITE_REPLICATION_PEER_JOIN_PATH, &svc_ak, &new_sk, &join_req).await { let detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint)); - warn!(peer = %peer.endpoint, error = %detail, "site replication service account rotation failed for peer"); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + peer = %peer.endpoint, + result = "service_account_rotation_failed", + error = %detail, + "admin site replication state" + ); peer_errors.push(detail); } } diff --git a/rustfs/src/admin/handlers/target_descriptor.rs b/rustfs/src/admin/handlers/target_descriptor.rs index d7ea4b4ef..98210fd48 100644 --- a/rustfs/src/admin/handlers/target_descriptor.rs +++ b/rustfs/src/admin/handlers/target_descriptor.rs @@ -472,7 +472,7 @@ where if !allowed_keys.contains(key) { return Err(s3_error!( InvalidArgument, - "key '{}' not allowed for {} type '{}'", + "key '{}' is not allowed for {} type '{}'", key, target_label, target_type @@ -492,7 +492,7 @@ where pub(crate) async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> { if !queue_dir.is_empty() { if !Path::new(queue_dir).is_absolute() { - return Err(s3_error!(InvalidArgument, "queue_dir must be absolute path")); + return Err(s3_error!(InvalidArgument, "queue_dir must be an absolute path")); } retry_with_backoff( || async { tokio::fs::metadata(queue_dir).await.map(|_| ()) }, @@ -502,7 +502,7 @@ pub(crate) async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> { .await .map_err(|e| match e.kind() { ErrorKind::NotFound => s3_error!(InvalidArgument, "queue_dir does not exist"), - ErrorKind::PermissionDenied => s3_error!(InvalidArgument, "queue_dir exists but permission denied"), + ErrorKind::PermissionDenied => s3_error!(InvalidArgument, "queue_dir exists but permission is denied"), _ => s3_error!(InvalidArgument, "failed to access queue_dir: {}", e), })?; } @@ -533,7 +533,7 @@ where .ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?; timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, default_queue_dir)) .await - .map_err(|_| s3_error!(InvalidArgument, "target validation timed out"))??; + .map_err(|_| s3_error!(InvalidArgument, "target validation timed out after 10s"))??; let mut kvs = KVS::new(); for (key, value) in kv_map { @@ -652,13 +652,13 @@ async fn validate_webhook_request(kv_map: &HashMap) -> S3Result< .get("endpoint") .map(String::as_str) .ok_or_else(|| s3_error!(InvalidArgument, "endpoint is required"))?; - let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?; + let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint URL: {}", e))?; match parsed_endpoint.scheme() { "http" | "https" => {} other => { return Err(s3_error!( InvalidArgument, - "unsupported endpoint scheme: {} (only http and https are allowed)", + "unsupported endpoint scheme '{}'; only http and https are allowed", other )); } @@ -683,7 +683,7 @@ async fn validate_mqtt_request(kv_map: &HashMap) -> S3Result<()> let endpoint = kv_map .get(MQTT_BROKER) .map(String::as_str) - .ok_or_else(|| s3_error!(InvalidArgument, "broker endpoint is required"))?; + .ok_or_else(|| s3_error!(InvalidArgument, "broker URL is required"))?; let topic = kv_map .get(MQTT_TOPIC) .map(String::as_str) @@ -701,12 +701,12 @@ async fn validate_mqtt_request(kv_map: &HashMap) -> S3Result<()> .map_err(|e| s3_error!(InvalidArgument, "invalid MQTT TLS settings: {}", e))?; let parsed_broker = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid broker URL: {}", e))?; rustfs_targets::target::mqtt::validate_mqtt_broker_url(&parsed_broker, &tls) - .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + .map_err(|e| s3_error!(InvalidArgument, "invalid broker URL: {}", e))?; check_mqtt_broker_available_with_tls(parsed_broker.as_str(), topic, username, password, &tls) .await .map_err(|e| match e { - TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), - _ => s3_error!(InvalidArgument, "MQTT broker check failed: {}", e), + TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid MQTT configuration: {}", e), + _ => s3_error!(InvalidArgument, "mqtt connectivity check failed: {}", e), })?; if let Some(queue_dir) = kv_map.get("queue_dir") { @@ -755,10 +755,10 @@ async fn validate_nats_request_impl( validate_queue_dir(queue_dir.as_str()).await?; } let args = build_nats_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) - .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + .map_err(|e| s3_error!(InvalidArgument, "invalid NATS configuration: {}", e))?; check_nats_server_available(&args).await.map_err(|e| match e { - TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), - _ => s3_error!(InvalidArgument, "NATS server check failed: {}", e), + TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid NATS configuration: {}", e), + _ => s3_error!(InvalidArgument, "nats connectivity check failed: {}", e), }) } @@ -780,17 +780,17 @@ async fn validate_kafka_request_impl( } if !kv_map.contains_key(KAFKA_BROKERS) { - return Err(s3_error!(InvalidArgument, "Kafka brokers are required")); + return Err(s3_error!(InvalidArgument, "kafka brokers are required")); } if !kv_map.contains_key(KAFKA_TOPIC) { - return Err(s3_error!(InvalidArgument, "Kafka topic is required")); + return Err(s3_error!(InvalidArgument, "kafka topic is required")); } let args = build_kafka_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) - .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + .map_err(|e| s3_error!(InvalidArgument, "invalid Kafka configuration: {}", e))?; check_kafka_broker_available(&args).await.map_err(|e| match e { - TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), - _ => s3_error!(InvalidArgument, "Kafka broker check failed: {}", e), + TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid Kafka configuration: {}", e), + _ => s3_error!(InvalidArgument, "kafka connectivity check failed: {}", e), }) } @@ -811,10 +811,10 @@ async fn validate_amqp_request_impl( validate_queue_dir(queue_dir.as_str()).await?; } let args = build_amqp_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) - .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + .map_err(|e| s3_error!(InvalidArgument, "invalid AMQP configuration: {}", e))?; check_amqp_broker_available(&args).await.map_err(|e| match e { - TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), - _ => s3_error!(InvalidArgument, "AMQP broker check failed: {}", e), + TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid AMQP configuration: {}", e), + _ => s3_error!(InvalidArgument, "amqp connectivity check failed: {}", e), }) } @@ -835,10 +835,10 @@ async fn validate_pulsar_request_impl( validate_queue_dir(queue_dir.as_str()).await?; } let args = build_pulsar_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) - .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + .map_err(|e| s3_error!(InvalidArgument, "invalid Pulsar configuration: {}", e))?; check_pulsar_broker_available(&args).await.map_err(|e| match e { - TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), - _ => s3_error!(InvalidArgument, "Pulsar broker check failed: {}", e), + TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid Pulsar configuration: {}", e), + _ => s3_error!(InvalidArgument, "pulsar connectivity check failed: {}", e), }) } @@ -851,11 +851,11 @@ async fn validate_mysql_request( validate_queue_dir(queue_dir.as_str()).await?; } - let args = - build_mysql_args(&to_kvs(kv_map), default_queue_dir, target_type).map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + let args = build_mysql_args(&to_kvs(kv_map), default_queue_dir, target_type) + .map_err(|e| s3_error!(InvalidArgument, "invalid MySQL configuration: {}", e))?; check_mysql_server_available(&args).await.map_err(|e| match e { - TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), - _ => s3_error!(InvalidArgument, "MySQL server check failed: {}", e), + TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid MySQL configuration: {}", e), + _ => s3_error!(InvalidArgument, "mysql connectivity check failed: {}", e), }) } @@ -876,10 +876,10 @@ async fn validate_postgres_request_impl( validate_queue_dir(queue_dir.as_str()).await?; } let args = build_postgres_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) - .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + .map_err(|e| s3_error!(InvalidArgument, "invalid PostgreSQL configuration: {}", e))?; check_postgres_server_available(&args).await.map_err(|e| match e { - TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), - _ => s3_error!(InvalidArgument, "PostgreSQL server check failed: {}", e), + TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid PostgreSQL configuration: {}", e), + _ => s3_error!(InvalidArgument, "postgres connectivity check failed: {}", e), }) } @@ -894,13 +894,13 @@ async fn validate_redis_request( } validate_redis_config(&to_kvs(kv_map), default_queue_dir, default_channel) - .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + .map_err(|e| s3_error!(InvalidArgument, "invalid Redis configuration: {}", e))?; let args = build_redis_args(&to_kvs(kv_map), default_queue_dir, default_channel, domain.runtime_target_type()) - .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + .map_err(|e| s3_error!(InvalidArgument, "invalid Redis configuration: {}", e))?; check_redis_server_available(&args).await.map_err(|e| match e { - TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), - _ => s3_error!(InvalidArgument, "Redis server check failed: {}", e), + TargetError::Configuration(_) => s3_error!(InvalidArgument, "invalid Redis configuration: {}", e), + _ => s3_error!(InvalidArgument, "redis connectivity check failed: {}", e), }) } diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index a45ee805e..e1ccc9039 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -57,6 +57,10 @@ use time::OffsetDateTime; use tokio::spawn; use tracing::{debug, warn}; +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_TIER: &str = "tier"; +const EVENT_ADMIN_TIER_STATE: &str = "admin_tier_state"; + #[derive(Debug, Clone, serde::Deserialize, Default)] pub struct AddTierQuery { #[serde(rename = "accessKey")] @@ -91,9 +95,14 @@ fn spawn_transition_tier_config_propagation(action: &'static str) { for peer_result in notification_sys.load_transition_tier_config().await { if let Some(err) = peer_result.err { warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = action, host = if peer_result.host.is_empty() { "" } else { peer_result.host.as_str() }, + result = "propagation_failed", error = %err, - "tier {action} propagation failed after local save" + "admin tier state" ); } } @@ -174,7 +183,7 @@ impl Operation for AddTier { let query = { if let Some(query) = req.uri.query() { let input: AddTierQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { AddTierQuery::default() @@ -182,7 +191,7 @@ impl Operation for AddTier { }; let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -202,13 +211,21 @@ impl Operation for AddTier { let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await { Ok(b) => b, Err(e) => { - warn!("get body failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "add_tier", + result = "body_read_failed", + error = ?e, + "admin tier state" + ); return Err(s3_error!(InvalidRequest, "tier configuration body too large or failed to read")); } }; let mut args: TierConfig = serde_json::from_slice(&body) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?; + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?; match args.tier_type { TierType::S3 => { @@ -240,33 +257,67 @@ impl Operation for AddTier { } _ => (), } - debug!("add tier args {:?}", args); + debug!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "add_tier", + tier_name = %args.name, + tier_type = ?args.tier_type, + state = "decoded", + "admin tier state" + ); + let tier_name_for_log = args.name.clone(); let mut force: bool = false; let force_str = query.force.clone().unwrap_or_default(); if !force_str.is_empty() { force = force_str.parse().map_err(|e| { - warn!("parse force failed, e: {:?}", e); - s3_error!(InvalidRequest, "parse force failed") + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "add_tier", + result = "force_parse_failed", + error = ?e, + "admin tier state" + ); + s3_error!(InvalidRequest, "invalid force flag") })?; } match args.name.as_str() { storageclass::STANDARD | storageclass::RRS => { - warn!("tier reserved name, args.name: {}", args.name); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "add_tier", + tier_name = %args.name, + result = "reserved_name_rejected", + "admin tier state" + ); return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name")); } &_ => (), } let Some(store) = resolve_object_store_handle() else { - return Err(s3_error!(InvalidRequest, "object store not init")); + return Err(s3_error!(InternalError, "object store is not initialized")); }; { let tier_config_mgr_handle = resolve_tier_config_handle(); let mut tier_config_mgr = tier_config_mgr_handle.write().await; if let Err(err) = tier_config_mgr.reload(store).await { - warn!("tier_config_mgr reload failed, e: {:?}", err); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "add_tier", + result = "reload_failed", + error = ?err, + "admin tier state" + ); return Err(S3Error::with_message( S3ErrorCode::Custom("TierAddFailed".into()), format!("tier reload failed. {err}"), @@ -276,27 +327,36 @@ impl Operation for AddTier { return if err.code == ERR_TIER_ALREADY_EXISTS.code { Err(S3Error::with_message( S3ErrorCode::Custom("TierNameAlreadyExist".into()), - "tier name already exists!", + "tier name already exists", )) } else if err.code == ERR_TIER_NAME_NOT_UPPERCASE.code { Err(S3Error::with_message( S3ErrorCode::Custom("TierNameNotUppercase".into()), - "tier name not uppercase!", + "tier name must be uppercase", )) } else if err.code == ERR_TIER_BACKEND_IN_USE.code { Err(S3Error::with_message( S3ErrorCode::Custom("TierNameBackendInUse!".into()), - "tier name backend in use!", + "tier backend is already in use", )) } else if err.code == ERR_TIER_CONNECT_ERR.code { Err(S3Error::with_message( S3ErrorCode::Custom("TierConnectError".into()), - "tier connect error!", + "tier connectivity check failed", )) } else if err.code == ERR_TIER_INVALID_CREDENTIALS.code { Err(S3Error::with_message(S3ErrorCode::Custom(err.code.clone().into()), err.message)) } else { - warn!("tier_config_mgr add failed, e: {:?}", err); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "add_tier", + tier_name = %tier_name_for_log, + result = "add_failed", + error = ?err, + "admin tier state" + ); Err(S3Error::with_message( S3ErrorCode::Custom("TierAddFailed".into()), format!("tier add failed. {err}"), @@ -304,7 +364,15 @@ impl Operation for AddTier { }; } if let Err(e) = tier_config_mgr.save().await { - warn!("tier_config_mgr save failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "add_tier", + result = "save_failed", + error = ?e, + "admin tier state" + ); return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier save failed")); } } @@ -324,7 +392,7 @@ impl Operation for EditTier { let query = { if let Some(query) = req.uri.query() { let input: AddTierQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { AddTierQuery::default() @@ -332,7 +400,7 @@ impl Operation for EditTier { }; let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -352,27 +420,51 @@ impl Operation for EditTier { let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await { Ok(b) => b, Err(e) => { - warn!("get body failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "edit_tier", + result = "body_read_failed", + error = ?e, + "admin tier state" + ); return Err(s3_error!(InvalidRequest, "tier configuration body too large or failed to read")); } }; let creds: TierCreds = serde_json::from_slice(&body) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?; - - debug!("edit tier args {:?}", creds); + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?; let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default(); + debug!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "edit_tier", + tier_name = %tier_name, + state = "decoded", + "admin tier state" + ); + let Some(store) = resolve_object_store_handle() else { - return Err(s3_error!(InvalidRequest, "object store not init")); + return Err(s3_error!(InternalError, "object store is not initialized")); }; { let tier_config_mgr_handle = resolve_tier_config_handle(); let mut tier_config_mgr = tier_config_mgr_handle.write().await; if let Err(err) = tier_config_mgr.reload(store).await { - warn!("tier_config_mgr reload failed, e: {:?}", err); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "edit_tier", + result = "reload_failed", + error = ?err, + "admin tier state" + ); return Err(S3Error::with_message( S3ErrorCode::Custom("TierEditFailed".into()), format!("tier reload failed. {err}"), @@ -380,14 +472,23 @@ impl Operation for EditTier { } if let Err(err) = tier_config_mgr.edit(&tier_name, creds).await { return if err.code == ERR_TIER_NOT_FOUND.code { - Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!")) + Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found")) } else if err.code == ERR_TIER_MISSING_CREDENTIALS.code { Err(S3Error::with_message( S3ErrorCode::Custom("TierMissingCredentials".into()), - "tier missing credentials!", + "tier credentials are required", )) } else { - warn!("tier_config_mgr edit failed, e: {:?}", err); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "edit_tier", + tier_name = %tier_name, + result = "edit_failed", + error = ?err, + "admin tier state" + ); Err(S3Error::with_message( S3ErrorCode::Custom("TierEditFailed".into()), format!("tier edit failed. {err}"), @@ -395,7 +496,15 @@ impl Operation for EditTier { }; } if let Err(e) = tier_config_mgr.save().await { - warn!("tier_config_mgr save failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "edit_tier", + result = "save_failed", + error = ?e, + "admin tier state" + ); return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed")); } } @@ -466,7 +575,7 @@ impl Operation for RemoveTier { let query = { if let Some(query) = req.uri.query() { let input: AddTierQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { AddTierQuery::default() @@ -474,7 +583,7 @@ impl Operation for RemoveTier { }; let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -494,22 +603,38 @@ impl Operation for RemoveTier { let force_str = query.force.clone().unwrap_or_default(); if !force_str.is_empty() { force = force_str.parse().map_err(|e| { - warn!("parse force failed, e: {:?}", e); - s3_error!(InvalidRequest, "parse force failed") + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "remove_tier", + result = "force_parse_failed", + error = ?e, + "admin tier state" + ); + s3_error!(InvalidRequest, "invalid force flag") })?; } let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default(); let Some(store) = resolve_object_store_handle() else { - return Err(s3_error!(InvalidRequest, "object store not init")); + return Err(s3_error!(InternalError, "object store is not initialized")); }; { let tier_config_mgr_handle = resolve_tier_config_handle(); let mut tier_config_mgr = tier_config_mgr_handle.write().await; if let Err(err) = tier_config_mgr.reload(store).await { - warn!("tier_config_mgr reload failed, e: {:?}", err); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "remove_tier", + result = "reload_failed", + error = ?err, + "admin tier state" + ); return Err(S3Error::with_message( S3ErrorCode::Custom("TierRemoveFailed".into()), format!("tier reload failed. {err}"), @@ -517,11 +642,23 @@ impl Operation for RemoveTier { } if let Err(err) = tier_config_mgr.remove(&tier_name, force).await { return if err.code == ERR_TIER_NOT_FOUND.code { - Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found.")) + Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found")) } else if err.code == ERR_TIER_BACKEND_NOT_EMPTY.code { - Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse".into()), "tier is used.")) + Err(S3Error::with_message( + S3ErrorCode::Custom("TierNameBackendInUse".into()), + "tier backend is not empty", + )) } else { - warn!("tier_config_mgr remove failed, e: {:?}", err); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "remove_tier", + tier_name = %tier_name, + result = "remove_failed", + error = ?err, + "admin tier state" + ); Err(S3Error::with_message( S3ErrorCode::Custom("TierRemoveFailed".into()), format!("tier remove failed. {err}"), @@ -530,7 +667,15 @@ impl Operation for RemoveTier { } if let Err(e) = tier_config_mgr.save().await { - warn!("tier_config_mgr save failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "remove_tier", + result = "save_failed", + error = ?e, + "admin tier state" + ); return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier save failed")); } } @@ -549,7 +694,7 @@ pub struct VerifyTier {} impl Operation for VerifyTier { async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -601,7 +746,7 @@ impl Operation for GetTierInfo { let query = { if let Some(query) = req.uri.query() { let input: AddTierQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { AddTierQuery::default() @@ -683,7 +828,7 @@ impl Operation for ClearTier { let query = { if let Some(query) = req.uri.query() { let input: ClearTierQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get query failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { ClearTierQuery::default() @@ -691,7 +836,7 @@ impl Operation for ClearTier { }; let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -718,23 +863,46 @@ impl Operation for ClearTier { rand.insert_str(3, &t.day().to_string()); rand.insert_str(17, &t.month().to_string()); rand.insert_str(23, &t.year().to_string()); - warn!("tier_config_mgr rand: {}", rand); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "clear_tier", + state = "challenge_generated", + "admin tier state" + ); if query.rand != Some(rand) { - return Err(s3_error!(InvalidRequest, "get rand failed")); + return Err(s3_error!(InvalidRequest, "invalid clear-tier confirmation token")); }; let tier_config_mgr_handle = resolve_tier_config_handle(); let mut tier_config_mgr = tier_config_mgr_handle.write().await; //tier_config_mgr.reload(api); if let Err(err) = tier_config_mgr.clear_tier(force).await { - warn!("tier_config_mgr clear failed, e: {:?}", err); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "clear_tier", + result = "clear_failed", + error = ?err, + "admin tier state" + ); return Err(S3Error::with_message( S3ErrorCode::Custom("TierClearFailed".into()), format!("tier clear failed. {err}"), )); } if let Err(e) = tier_config_mgr.save().await { - warn!("tier_config_mgr save failed, e: {:?}", e); + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "clear_tier", + result = "save_failed", + error = ?e, + "admin tier state" + ); return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed")); } diff --git a/rustfs/src/admin/handlers/user.rs b/rustfs/src/admin/handlers/user.rs index 873e253c9..a7e1cabf2 100644 --- a/rustfs/src/admin/handlers/user.rs +++ b/rustfs/src/admin/handlers/user.rs @@ -51,6 +51,10 @@ use std::{collections::HashMap, io::Cursor, str::from_utf8}; use tracing::{debug, warn}; use zip::{ZipArchive, ZipWriter, result::ZipError, write::SimpleFileOptions}; +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_USER: &str = "user"; +const EVENT_ADMIN_USER_STATE: &str = "admin_user_state"; + #[derive(Debug, Deserialize, Default)] pub struct AddUserQuery { #[serde(rename = "accessKey", alias = "access-key")] @@ -184,7 +188,7 @@ impl Operation for AddUser { let query = { if let Some(query) = req.uri.query() { let input: AddUserQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { AddUserQuery::default() @@ -192,7 +196,7 @@ impl Operation for AddUser { }; let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -210,39 +214,41 @@ impl Operation for AddUser { // .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, format!("decrypt_data err {}", e)))?; let args: AddOrUpdateUserReq = serde_json::from_slice(&body) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {e}")))?; + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?; if args.secret_key.is_empty() { - return Err(s3_error!(InvalidArgument, "access key is empty")); + return Err(s3_error!(InvalidArgument, "secret key is required")); } if let Some(sys_cred) = get_global_action_cred() && constant_time_eq(&sys_cred.access_key, ak) { - return Err(s3_error!(InvalidArgument, "can't create user with system access key")); + return Err(s3_error!(InvalidArgument, "cannot create a user with the system access key")); } let Ok(iam_store) = rustfs_iam::get() else { - return Err(s3_error!(InvalidRequest, "iam not init")); + return Err(s3_error!(InternalError, "iam is not initialized")); }; if let Some(user) = iam_store.get_user(ak).await { if (user.credentials.is_temp() || user.credentials.is_service_account()) && cred.parent_user == ak { - return Err(s3_error!(InvalidArgument, "can't create user with service account access key")); + return Err(s3_error!(InvalidArgument, "cannot create a user with a service account access key")); } } else if has_space_be(ak) { - return Err(s3_error!(InvalidArgument, "access key has space")); + return Err(s3_error!(InvalidArgument, "access key contains spaces")); } if from_utf8(ak.as_bytes()).is_err() { - return Err(s3_error!(InvalidArgument, "access key is not utf8")); + return Err(s3_error!(InvalidArgument, "access key is not valid UTF-8")); } let check_deny_only = should_check_deny_only(ak, &cred); debug!( - target = "rustfs::admin::handlers::user", - operation = "AddUser", + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_USER, + event = EVENT_ADMIN_USER_STATE, + action = "add_user", query_access_key = %ak, signer_access_key = %cred.access_key, is_temp = cred.is_temp(), @@ -252,7 +258,8 @@ impl Operation for AddUser { jwt_parent_claim_present = cred.claims.as_ref().and_then(|c| c.get("parent")).is_some(), check_deny_only, is_owner = owner, - "authorization context before validate_admin_request (no secrets)" + state = "authorization_context", + "admin user state" ); // For eligible self operations, only explicit Deny should block the request. @@ -269,7 +276,7 @@ impl Operation for AddUser { let updated_at = iam_store .create_user(ak, &args) .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("create_user err {e}")))?; + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to create user: {e}")))?; if let Err(err) = site_replication_iam_change_hook(SRIAMItem { r#type: "iam-user".to_string(), @@ -285,7 +292,16 @@ impl Operation for AddUser { }) .await { - warn!(access_key = %ak, error = ?err, "site replication create user hook failed"); + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_USER, + event = EVENT_ADMIN_USER_STATE, + access_key = %ak, + action = "create_user", + result = "site_replication_hook_failed", + error = ?err, + "admin user state" + ); } let mut header = HeaderMap::new(); @@ -302,7 +318,7 @@ impl Operation for SetUserStatus { let query = { if let Some(query) = req.uri.query() { let input: AddUserQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { AddUserQuery::default() @@ -316,11 +332,11 @@ impl Operation for SetUserStatus { } let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; if constant_time_eq(&input_cred.access_key, ak) { - return Err(s3_error!(InvalidArgument, "can't change status of self")); + return Err(s3_error!(InvalidArgument, "cannot change the status of the current user")); } let (cred, owner) = @@ -340,13 +356,13 @@ impl Operation for SetUserStatus { .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, e))?; let Ok(iam_store) = rustfs_iam::get() else { - return Err(s3_error!(InvalidRequest, "iam not init")); + return Err(s3_error!(InternalError, "iam is not initialized")); }; iam_store .set_user_status(ak, status) .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("set_user_status err {e}")))?; + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to set user status: {e}")))?; let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); @@ -365,7 +381,7 @@ pub struct ListUsers {} impl Operation for ListUsers { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -384,7 +400,7 @@ impl Operation for ListUsers { let query = { if let Some(query) = req.uri.query() { let input: BucketQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { BucketQuery::default() @@ -392,7 +408,7 @@ impl Operation for ListUsers { }; let Ok(iam_store) = rustfs_iam::get() else { - return Err(s3_error!(InvalidRequest, "iam not init")); + return Err(s3_error!(InternalError, "iam is not initialized")); }; let users = { @@ -410,7 +426,7 @@ impl Operation for ListUsers { }; let data = serde_json::to_vec(&users) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal users err {e}")))?; + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize response: {e}")))?; let (data, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, data)?; let mut header = HeaderMap::new(); @@ -425,7 +441,7 @@ pub struct RemoveUser {} impl Operation for RemoveUser { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -444,7 +460,7 @@ impl Operation for RemoveUser { let query = { if let Some(query) = req.uri.query() { let input: AddUserQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { AddUserQuery::default() @@ -458,37 +474,35 @@ impl Operation for RemoveUser { } let sys_cred = get_global_action_cred() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "get_global_action_cred failed"))?; + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "failed to load global credentials"))?; if ak == sys_cred.access_key || ak == cred.access_key || cred.parent_user == ak { - return Err(s3_error!(InvalidArgument, "can't remove self")); + return Err(s3_error!(InvalidArgument, "cannot remove the current user")); } let Ok(iam_store) = rustfs_iam::get() else { - return Err(s3_error!(InvalidRequest, "iam not init")); + return Err(s3_error!(InternalError, "iam is not initialized")); }; - let (is_temp, _) = iam_store - .is_temp_user(ak) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("is_temp_user err {e}")))?; + let (is_temp, _) = iam_store.is_temp_user(ak).await.map_err(|e| { + S3Error::with_message(S3ErrorCode::InternalError, format!("failed to query temporary user state: {e}")) + })?; if is_temp { - return Err(s3_error!(InvalidArgument, "can't remove temp user")); + return Err(s3_error!(InvalidArgument, "cannot remove a temporary user")); } - let (is_service_account, _) = iam_store - .is_service_account(ak) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("is_service_account err {e}")))?; + let (is_service_account, _) = iam_store.is_service_account(ak).await.map_err(|e| { + S3Error::with_message(S3ErrorCode::InternalError, format!("failed to query service account state: {e}")) + })?; if is_service_account { - return Err(s3_error!(InvalidArgument, "can't remove service account")); + return Err(s3_error!(InvalidArgument, "cannot remove a service account")); } iam_store .delete_user(ak, true) .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("delete_user err {e}")))?; + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to delete user: {e}")))?; if let Err(err) = site_replication_iam_change_hook(SRIAMItem { r#type: "iam-user".to_string(), @@ -504,7 +518,16 @@ impl Operation for RemoveUser { }) .await { - warn!(access_key = %ak, error = ?err, "site replication delete user hook failed"); + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_USER, + event = EVENT_ADMIN_USER_STATE, + access_key = %ak, + action = "delete_user", + result = "site_replication_hook_failed", + error = ?err, + "admin user state" + ); } let mut header = HeaderMap::new(); @@ -521,7 +544,7 @@ impl Operation for GetUserInfo { let query = { if let Some(query) = req.uri.query() { let input: AddUserQuery = - from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "failed to decode query"))?; input } else { AddUserQuery::default() @@ -535,11 +558,11 @@ impl Operation for GetUserInfo { } let Ok(iam_store) = rustfs_iam::get() else { - return Err(s3_error!(InvalidRequest, "iam not init")); + return Err(s3_error!(InternalError, "iam is not initialized")); }; let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); + return Err(s3_error!(InvalidRequest, "authentication required")); }; let (cred, owner) = @@ -548,8 +571,10 @@ impl Operation for GetUserInfo { let check_deny_only = should_check_deny_only(ak, &cred); debug!( - target = "rustfs::admin::handlers::user", - operation = "GetUserInfo", + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_USER, + event = EVENT_ADMIN_USER_STATE, + action = "get_user_info", query_access_key = %ak, signer_access_key = %cred.access_key, is_temp = cred.is_temp(), @@ -559,7 +584,8 @@ impl Operation for GetUserInfo { jwt_parent_claim_present = cred.claims.as_ref().and_then(|c| c.get("parent")).is_some(), check_deny_only, is_owner = owner, - "authorization context before validate_admin_request (no secrets)" + state = "authorization_context", + "admin user state" ); // For eligible self operations, only explicit Deny should block the request. @@ -576,7 +602,7 @@ impl Operation for GetUserInfo { let info = iam_store.get_user_info(ak).await.map_err(iam_error_to_s3_error)?; let data = serde_json::to_vec(&info) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal user err {e}")))?; + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize response: {e}")))?; let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); @@ -845,7 +871,15 @@ impl Operation for ImportIam { let body = match input.store_all_limited(MAX_IAM_IMPORT_SIZE).await { Ok(b) => b, Err(e) => { - warn!("get body failed, e: {:?}", e); + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_USER, + event = EVENT_ADMIN_USER_STATE, + action = "import_iam", + result = "body_read_failed", + error = ?e, + "admin user state" + ); return Err(s3_error!(InvalidRequest, "get body failed")); } }; diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index a1fc32762..98e2c744b 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -104,6 +104,7 @@ use uuid::Uuid; const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_SUBSYSTEM_OBJECT_LAMBDA: &str = "object_lambda"; const LOG_SUBSYSTEM_LIVE_EVENTS: &str = "live_events"; +const EVENT_ADMIN_ROUTER_STATE: &str = "admin_router_state"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ReplicationExtRoute { @@ -634,11 +635,12 @@ async fn load_current_server_config() -> S3Result { Ok(config) => return Ok(config), Err(err) => { warn!( + event = EVENT_ADMIN_ROUTER_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA, - event = "object_lambda_config_reload_failed", + result = "config_reload_failed", error = %err, - "Failed to reload current server config for object lambda request" + "admin router state" ); } } @@ -663,11 +665,12 @@ fn build_object_lambda_http_client(config: &ObjectLambdaWebhookConfig) -> S3Resu if config.skip_tls_verify { warn!( + event = EVENT_ADMIN_ROUTER_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA, - event = "object_lambda_tls_verification_disabled", + result = "tls_verification_disabled", endpoint = %config.endpoint, - "Object Lambda target is configured to skip TLS certificate verification" + "admin router state" ); builder = builder.danger_accept_invalid_certs(true); } else if !config.client_ca.is_empty() { @@ -1220,23 +1223,25 @@ async fn fan_in_remote_live_events( Ok(Ok(batch)) => batch, Ok(Err(err)) => { warn!( + event = EVENT_ADMIN_ROUTER_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, - event = "peer_live_events_fetch_failed", peer = %peer.client.host, + result = "peer_fetch_failed", error = %err, - "Failed to fetch live events from peer" + "admin router state" ); break; } Err(_) => { warn!( + event = EVENT_ADMIN_ROUTER_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, - event = "peer_live_events_fetch_failed", peer = %peer.client.host, + result = "peer_fetch_timeout", error = "timeout", - "Failed to fetch live events from peer" + "admin router state" ); break; } @@ -1259,13 +1264,14 @@ async fn fan_in_remote_live_events( } Err(err) => { warn!( + event = EVENT_ADMIN_ROUTER_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, - event = "live_event_serialize_failed", source = "remote_peer", peer = %peer.client.host, + result = "event_serialize_failed", error = %err, - "Failed to serialize live event" + "admin router state" ); } } @@ -1273,12 +1279,13 @@ async fn fan_in_remote_live_events( } Err(err) => { warn!( + event = EVENT_ADMIN_ROUTER_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, - event = "peer_live_events_decode_failed", peer = %peer.client.host, + result = "peer_decode_failed", error = %err, - "Failed to decode live events from peer" + "admin router state" ); } } @@ -1333,23 +1340,25 @@ fn build_listen_notification_response(uri: &Uri, bucket: Option<&str>) -> S3Resu } Err(err) => { warn!( + event = EVENT_ADMIN_ROUTER_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, - event = "live_event_serialize_failed", source = "local_stream", + result = "event_serialize_failed", error = %err, - "Failed to serialize live event" + "admin router state" ); } } } Err(broadcast::error::RecvError::Lagged(skipped)) => { warn!( + event = EVENT_ADMIN_ROUTER_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_LIVE_EVENTS, - event = "live_event_stream_lagged", + result = "stream_lagged", skipped, - "Live event stream lagged" + "admin router state" ); } Err(broadcast::error::RecvError::Closed) => break, @@ -1509,7 +1518,14 @@ async fn authorize_replication_extension_request(req: &mut S3Request, ext_ license_check().map_err(|er| match er.kind() { std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"), _ => { - error!("license check failed due to unexpected error: {er}"); + error!( + event = EVENT_ADMIN_ROUTER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA, + result = "license_check_failed", + error = %er, + "admin router state" + ); s3_error!(InternalError, "License validation failed") } })?; @@ -2231,7 +2247,14 @@ async fn authorize_misc_extension_request(req: &mut S3Request, route: &Mis license_check().map_err(|er| match er.kind() { std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"), _ => { - error!("license check failed due to unexpected error: {er}"); + error!( + event = EVENT_ADMIN_ROUTER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA, + result = "license_check_failed", + error = %er, + "admin router state" + ); s3_error!(InternalError, "License validation failed") } })?; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index cec12926b..1562b214a 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -127,7 +127,7 @@ use tokio::io::{AsyncRead, ReadBuf}; use tokio::sync::RwLock; use tokio_tar::Archive; use tokio_util::io::{ReaderStream, StreamReader}; -use tracing::{debug, error, info, instrument, warn}; +use tracing::{debug, error, instrument, warn}; use uuid::Uuid; const ACCEPT_RANGES_BYTES: &str = "bytes"; @@ -1120,15 +1120,18 @@ fn is_post_object_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) async fn resolve_put_object_expiration(bucket: &str, obj_info: &ObjectInfo) -> Option { let Ok((lifecycle_config, _)) = metadata_sys::get_lifecycle_config(bucket).await else { - debug!("resolve_put_object_expiration: lifecycle config not found for bucket {bucket}"); + debug!(bucket, state = "config_missing", "PUT object expiration config missing"); return None; }; let obj_opts = lifecycle::ObjectOpts::from_object_info(obj_info); let event = lifecycle_config.predict_expiration(&obj_opts).await; debug!( - "resolve_put_object_expiration: bucket={bucket}, action={:?}, rule_id={}, due={:?}", - event.action, event.rule_id, event.due + bucket, + action = ?event.action, + rule_id = %event.rule_id, + due = ?event.due, + "PUT object expiration resolved" ); build_put_object_expiration_header(&event) } @@ -1184,7 +1187,7 @@ impl DefaultObjectUsecase { ), )), Err(e) => { - warn!("Quota check failed for bucket {bucket}: {e}, allowing operation"); + warn!(bucket, error = %e, state = "checker_failed", "Bucket quota check degraded to allow"); Ok(()) } _ => Ok(()), @@ -1427,7 +1430,7 @@ impl DefaultObjectUsecase { match ContentType::from_str(content_type) { Ok(res) => Some(res), Err(err) => { - error!("parse content-type err {} {:?}", content_type, err); + error!(content_type, error = ?err, "GET object content-type parse failed"); None } } @@ -1624,7 +1627,7 @@ impl DefaultObjectUsecase { { let (decrypted_checksums, _is_multipart) = info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| { - error!("decrypt_checksums error: {}", e); + error!(error = %e, "GetObject checksum decryption failed"); ApiError::from(e) })?; @@ -1667,25 +1670,22 @@ impl DefaultObjectUsecase { if should_buffer_encrypted_object { let mut buf = Vec::with_capacity(response_content_length as usize); if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { - error!("Failed to read decrypted object into memory: {}", e); + error!(error = %e, "GetObject decrypted object buffering failed"); return Err(ApiError::from(StorageError::other(format!("Failed to read decrypted object: {e}"))).into()); } if buf.len() != response_content_length as usize { warn!( - "Encrypted object size mismatch during read: expected={} actual={}", - response_content_length, - buf.len() + expected = response_content_length, + actual = buf.len(), + "Encrypted object size mismatch during read" ); } return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size)); } - info!( - "Encrypted object: Using unlimited stream for decryption with buffer size {}", - optimal_buffer_size - ); + debug!(buffer_size = optimal_buffer_size, "Encrypted object uses streaming decrypt path"); return Ok(Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size)); } @@ -1698,16 +1698,16 @@ impl DefaultObjectUsecase { Ok(_) => { if buf.len() != response_content_length as usize { warn!( - "Object size mismatch during seek support read: expected={} actual={}", - response_content_length, - buf.len() + expected = response_content_length, + actual = buf.len(), + "Object size mismatch during seek-support read" ); } return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size)); } Err(e) => { - error!("Failed to read object into memory for seek support: {}", e); + error!(error = %e, "GetObject seek-support buffering failed"); } } } @@ -2780,7 +2780,7 @@ impl DefaultObjectUsecase { && src_bucket == bucket && src_key == key { - error!("Rejected self-copy operation: bucket={}, key={}", bucket, key); + error!(bucket, key, "Rejected self-copy operation"); return Err(s3_error!( InvalidRequest, "Cannot copy an object to itself. Source and destination must be different." @@ -3679,7 +3679,7 @@ impl DefaultObjectUsecase { let has_children = match probe_prefix_has_children(store, &bucket, &key, false).await { Ok(has_children) => has_children, Err(e) => { - error!("Failed to probe children for prefix (bucket: {}, key: {}): {}", bucket, key, e); + error!(bucket, key, error = %e, "Failed to probe children for prefix"); false } }; @@ -3751,7 +3751,7 @@ impl DefaultObjectUsecase { match ContentType::from_str(content_type) { Ok(res) => Some(res), Err(err) => { - error!("parse content-type err {} {:?}", &content_type, err); + error!(content_type = %content_type, error = ?err, "Archive content-type parse failed"); // None } @@ -3765,7 +3765,7 @@ impl DefaultObjectUsecase { // TODO: range download let content_length = info.get_actual_size().map_err(|e| { - error!("get_actual_size error: {}", e); + error!(error = %e, "Failed to resolve actual object size"); ApiError::from(e) })?; @@ -3881,7 +3881,7 @@ impl DefaultObjectUsecase { if let Ok(header_value) = tag_count.to_string().parse::() { response.headers.insert(header_name, header_value); } else { - warn!("Failed to parse x-amz-tagging-count header value, skipping"); + warn!("Failed to parse x-amz-tagging-count header; skipping"); } } if let Some(retain_date) = metadata_map @@ -4127,7 +4127,7 @@ impl DefaultObjectUsecase { err.to_string() ); } else { - info!("successfully restored transitioned object: {}/{}", bucket_clone, object_clone); + debug!(bucket = %bucket_clone, object = %object_clone, "Transitioned object restored"); } }); @@ -4302,13 +4302,13 @@ impl DefaultObjectUsecase { let decoder = CompressionFormat::from_extension(&ext) .get_decoder(ExtractArchiveEtagReader::new(archive_reader, archive_etag.clone())) .map_err(|e| { - error!("get_decoder err {:?}", e); + error!(error = ?e, "Archive decoder creation failed"); s3_error!(InvalidArgument, "get_decoder err") })?; let mut ar = Archive::new(decoder); let mut entries = ar.entries().map_err(|e| { - error!("get entries err {:?}", e); + error!(error = ?e, "Archive entry listing failed"); s3_error!(InvalidArgument, "get entries err") })?; @@ -4337,10 +4337,10 @@ impl DefaultObjectUsecase { Ok(f) => f, Err(e) => { if extract_options.ignore_errors { - warn!("Skipping archive entry because read failed and ignore-errors is enabled: {e}"); + warn!(error = %e, "Archive entry read skipped due to ignore-errors"); continue; } - error!("Failed to read archive entry: {}", e); + error!(error = %e, "Archive entry read failed"); return Err(s3_error!(InvalidArgument, "Failed to read archive entry: {:?}", e)); } }; @@ -4349,7 +4349,7 @@ impl DefaultObjectUsecase { Ok(path) => path, Err(e) => { if extract_options.ignore_errors { - warn!("Skipping archive entry because path decode failed and ignore-errors is enabled: {e}"); + warn!(error = %e, "Archive path decode skipped due to ignore-errors"); continue; } return Err(s3_error!(InvalidArgument, "Failed to decode archive entry path")); @@ -4361,7 +4361,7 @@ impl DefaultObjectUsecase { Ok(fpath) => fpath, Err(err) => { if extract_options.ignore_errors { - warn!("Skipping archive entry because path is unsafe and ignore-errors is enabled: {err}"); + warn!(error = %err, "Unsafe archive path skipped due to ignore-errors"); continue; } return Err(err); @@ -4491,7 +4491,7 @@ impl DefaultObjectUsecase { Ok(info) => info, Err(e) => { if extract_options.ignore_errors { - warn!("Skipping archive entry because object write failed and ignore-errors is enabled: {e}"); + warn!(error = %e, "Archive object write skipped due to ignore-errors"); continue; } return Err(ApiError::from(e).into()); diff --git a/rustfs/src/auth.rs b/rustfs/src/auth.rs index dde215310..41e3a3a9b 100644 --- a/rustfs/src/auth.rs +++ b/rustfs/src/auth.rs @@ -151,7 +151,7 @@ impl S3Auth for IAMAuth { subsystem = LOG_SUBSYSTEM_KEYSTONE, principal = %MaskedAccessKey(&creds.parent_user), result = "token_auth", - "Keystone credentials found in task-local storage" + "Keystone task-local credentials detected" ); // Return empty secret key - Keystone uses token validation, not AWS signatures return Ok(SecretKey::from(String::new())); @@ -170,7 +170,7 @@ impl S3Auth for IAMAuth { subsystem = LOG_SUBSYSTEM_KEYSTONE, access_key = %MaskedAccessKey(access_key), result = "token_auth", - "Keystone access key detected for token-based auth" + "Keystone token-auth access key detected" ); // Return empty secret key - Keystone uses token validation, not AWS signatures // The actual credentials are stored in task-local storage by KeystoneAuthMiddleware @@ -202,7 +202,7 @@ impl S3Auth for IAMAuth { subsystem = LOG_SUBSYSTEM_CREDENTIALS, access_key = %MaskedAccessKey(access_key), reason = "no_such_user", - "Secret key lookup failed" + "Secret key lookup rejected" ); } Err(e) => { @@ -213,7 +213,7 @@ impl S3Auth for IAMAuth { access_key = %MaskedAccessKey(access_key), error = ?e, reason = "check_key_error", - "Secret key lookup failed" + "Secret key lookup errored" ); return Err(iam_lookup_error_to_s3_error(&e)); } @@ -225,7 +225,7 @@ impl S3Auth for IAMAuth { subsystem = LOG_SUBSYSTEM_CREDENTIALS, access_key = %MaskedAccessKey(access_key), reason = "iam_not_initialized", - "Secret key lookup failed" + "Secret key lookup skipped" ); } @@ -257,7 +257,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result< subsystem = LOG_SUBSYSTEM_CREDENTIALS, access_key = %MaskedAccessKey(access_key), has_session_token = !session_token.is_empty(), - "Starting access key validation" + "Access key validation started" ); if let Ok(Some(credentials)) = KEYSTONE_CREDENTIALS.try_with(|creds| creds.clone()) { debug!( @@ -265,7 +265,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result< component = LOG_COMPONENT_AUTH, subsystem = LOG_SUBSYSTEM_KEYSTONE, result = "task_local", - "Keystone credentials found in task-local storage" + "Keystone task-local credentials detected" ); if !auth_keystone::is_keystone_enabled() { @@ -283,7 +283,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result< .and_then(|c| c.get("keystone_project_name")) .and_then(|v| v.as_str()) .is_some(), - "Validated Keystone credentials from task-local storage" + "Keystone task-local credentials validated" ); // Determine if user is admin (owner-level access) @@ -307,7 +307,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result< subsystem = LOG_SUBSYSTEM_KEYSTONE, principal = %MaskedAccessKey(&credentials.parent_user), is_owner, - "Evaluated Keystone owner permissions" + "Keystone owner permissions evaluated" ); return Ok((credentials, is_owner)); @@ -320,7 +320,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result< component = LOG_COMPONENT_AUTH, subsystem = LOG_SUBSYSTEM_KEYSTONE, access_key = %MaskedAccessKey(access_key), - "Keystone access key detected without task-local credentials" + "Keystone context missing for access key" ); if !auth_keystone::is_keystone_enabled() { @@ -360,7 +360,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result< subsystem = LOG_SUBSYSTEM_CREDENTIALS, access_key = %MaskedAccessKey(access_key), reason = "user_not_found", - "Access key validation failed" + "Access key validation rejected" ); return Err(s3_error!(InvalidAccessKeyId, "check key failed")); }; @@ -372,7 +372,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result< subsystem = LOG_SUBSYSTEM_CREDENTIALS, access_key = %MaskedAccessKey(access_key), reason = "account_disabled", - "Access key validation failed" + "Access key validation rejected" ); return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled")); } @@ -383,7 +383,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result< subsystem = LOG_SUBSYSTEM_CREDENTIALS, access_key = %MaskedAccessKey(access_key), reason = "validation_failed", - "Access key validation failed" + "Access key validation rejected" ); return Err(s3_error!(InvalidRequest, "check key failed")); } diff --git a/rustfs/src/init.rs b/rustfs/src/init.rs index 06c797735..3efd34a4f 100644 --- a/rustfs/src/init.rs +++ b/rustfs/src/init.rs @@ -36,6 +36,8 @@ const LOG_SUBSYSTEM_KMS: &str = "kms"; const LOG_SUBSYSTEM_BUFFER: &str = "buffer_profile"; const LOG_SUBSYSTEM_AUTOTUNER: &str = "autotuner"; const LOG_SUBSYSTEM_PROTOCOL: &str = "protocol"; +const EVENT_PROTOCOL_RUNTIME_STATE: &str = "protocol_runtime_state"; +const EVENT_PROTOCOL_SERVER_STATE: &str = "protocol_server_state"; #[instrument] pub fn print_server_info() { @@ -666,7 +668,7 @@ where protocol = protocol_name, state = "runtime_failed", error = %e, - "Protocol server state changed" + "Protocol server failed" ); } info!( @@ -676,7 +678,7 @@ where subsystem = LOG_SUBSYSTEM_PROTOCOL, protocol = protocol_name, state = "stopped", - "Protocol server state changed" + "Protocol server stopped" ); }); @@ -786,7 +788,15 @@ pub async fn init_ftp_system() -> Result, Box Result, Box = FtpsServer::new(config, storage_client).await?; + let bind_addr = server.config().bind_addr; + let passive_ports = server.config().passive_ports.clone(); // Log server configuration - info!( - "FTP server configured on {} with passive ports {:?}", - server.config().bind_addr, - server.config().passive_ports + debug!( + target: "rustfs::init", + event = EVENT_PROTOCOL_RUNTIME_STATE, + component = LOG_COMPONENT_INIT, + subsystem = LOG_SUBSYSTEM_PROTOCOL, + protocol = "ftp", + state = "configured", + bind_addr = %bind_addr, + passive_ports = ?passive_ports, + tls_enabled = false, + "Protocol runtime configured" ); // Start FTP server in background task with proper shutdown support @@ -830,12 +849,39 @@ pub async fn init_ftp_system() -> Result, Box Result, Box Result, Box = FtpsServer::new(config, storage_client).await?; + let bind_addr = server.config().bind_addr; + let passive_ports = server.config().passive_ports.clone(); + let tls_enabled = server.config().tls_enabled; // Log server configuration - info!( - "FTPS server configured on {} with passive ports {:?}", - server.config().bind_addr, - server.config().passive_ports + debug!( + target: "rustfs::init", + event = EVENT_PROTOCOL_RUNTIME_STATE, + component = LOG_COMPONENT_INIT, + subsystem = LOG_SUBSYSTEM_PROTOCOL, + protocol = "ftps", + state = "configured", + bind_addr = %bind_addr, + passive_ports = ?passive_ports, + tls_enabled, + "Protocol runtime configured" ); // Start FTPS server in background task with proper shutdown support @@ -906,12 +970,39 @@ pub async fn init_ftps_system() -> Result, Box Result, Box Result, Box = WebDavServer::new(config, storage_client).await?; + let bind_addr = server.config().bind_addr; + let tls_enabled = server.config().tls_enabled; + let max_body_size = server.config().max_body_size; + let request_timeout_secs = server.config().request_timeout_secs; // Log server configuration - info!("WebDAV server configured on {}", server.config().bind_addr); + debug!( + target: "rustfs::init", + event = EVENT_PROTOCOL_RUNTIME_STATE, + component = LOG_COMPONENT_INIT, + subsystem = LOG_SUBSYSTEM_PROTOCOL, + protocol = "webdav", + state = "configured", + bind_addr = %bind_addr, + tls_enabled, + max_body_size, + request_timeout_secs, + "Protocol runtime configured" + ); // Start WebDAV server in background task with proper shutdown support let (shutdown_tx, shutdown_rx) = tokio::sync::broadcast::channel(1); let task_handle = tokio::spawn(async move { if let Err(e) = server.start(shutdown_rx).await { - error!("WebDAV server error: {}", e); + error!( + target: "rustfs::init", + event = EVENT_PROTOCOL_SERVER_STATE, + component = LOG_COMPONENT_INIT, + subsystem = LOG_SUBSYSTEM_PROTOCOL, + protocol = "webdav", + state = "runtime_failed", + error = %e, + "Protocol server failed" + ); } - info!("WebDAV server shutdown completed"); + info!( + target: "rustfs::init", + event = EVENT_PROTOCOL_SERVER_STATE, + component = LOG_COMPONENT_INIT, + subsystem = LOG_SUBSYSTEM_PROTOCOL, + protocol = "webdav", + state = "stopped", + "Protocol server stopped" + ); }); - info!("WebDAV system initialized successfully"); + info!( + target: "rustfs::init", + event = EVENT_PROTOCOL_RUNTIME_STATE, + component = LOG_COMPONENT_INIT, + subsystem = LOG_SUBSYSTEM_PROTOCOL, + protocol = "webdav", + state = "started", + bind_addr = %bind_addr, + tls_enabled, + "Protocol runtime started" + ); Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle))) } } @@ -1003,7 +1145,15 @@ pub async fn init_sftp_system() -> Result, Box Result, Box Result, Box Result<()> { // Set up AK and SK match init_global_action_credentials(Some(config.access_key.clone()), Some(config.secret_key.clone())) { Ok(_) => { - debug!(target: "rustfs::main::run", "Global action credentials initialized successfully."); + debug!( + target: "rustfs::main::run", + event = EVENT_ACTION_CREDENTIALS_INITIALIZED, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_AUTH, + result = "ok", + "Initialized global action credentials" + ); } Err(e) => { let msg = format!("init global action credentials failed: {e:?}"); @@ -419,9 +427,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> { .inspect_err(|err| { error!( target: "rustfs::main::run", + event = EVENT_STARTUP_STORAGE_STAGE, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STORAGE, stage = "endpoint_parsing", + state = "failed", error = ?err, - "startup storage stage failed" + "Endpoint parsing failed" ); }) .map_err(Error::other)?; @@ -445,9 +457,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> { .inspect_err(|err| { error!( target: "rustfs::main::run", + event = EVENT_STARTUP_STORAGE_STAGE, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STORAGE, stage = "local_disk_initialization", + state = "failed", error = ?err, - "startup storage stage failed" + "Local disk initialization failed" ); }) .map_err(Error::other)?; @@ -540,9 +556,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> { .inspect_err(|err| { error!( target: "rustfs::main::run", + event = EVENT_STARTUP_STORAGE_STAGE, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STORAGE, stage = "ecstore_initialization", + state = "failed", error = ?err, - "startup storage stage failed" + "ECStore initialization failed" ); })?; @@ -561,7 +581,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { state = "retrying", retry_count = retry_count + 1, error = ?e, - "Startup storage stage failed" + "Global config initialization retry failed" ); // TODO: check error type retry_count += 1; @@ -580,13 +600,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> { #[cfg(feature = "ftps")] let ftp_shutdown_tx = match init_ftp_system().await { Ok(Some(tx)) => { - info!( + debug!( event = EVENT_PROTOCOL_SYSTEM_STATE, component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "ftp", - state = "initialized", - "Protocol system state changed" + state = "started", + "Protocol runtime started" ); Some(tx) } @@ -597,7 +617,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "ftp", state = "disabled", - "Protocol system state changed" + "Protocol runtime disabled" ); None } @@ -609,7 +629,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { protocol = "ftp", state = "initialization_failed", error = %e, - "Protocol system state changed" + "Protocol runtime initialization failed" ); return Err(Error::other(e)); } @@ -622,13 +642,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> { #[cfg(feature = "ftps")] let ftps_shutdown_tx = match init_ftps_system().await { Ok(Some(tx)) => { - info!( + debug!( event = EVENT_PROTOCOL_SYSTEM_STATE, component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "ftps", - state = "initialized", - "Protocol system state changed" + state = "started", + "Protocol runtime started" ); Some(tx) } @@ -639,7 +659,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "ftps", state = "disabled", - "Protocol system state changed" + "Protocol runtime disabled" ); None } @@ -651,7 +671,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { protocol = "ftps", state = "initialization_failed", error = %e, - "Protocol system state changed" + "Protocol runtime initialization failed" ); return Err(Error::other(e)); } @@ -664,13 +684,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> { #[cfg(feature = "webdav")] let webdav_shutdown_tx = match init_webdav_system().await { Ok(Some(tx)) => { - info!( + debug!( event = EVENT_PROTOCOL_SYSTEM_STATE, component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "webdav", - state = "initialized", - "Protocol system state changed" + state = "started", + "Protocol runtime started" ); Some(tx) } @@ -681,7 +701,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "webdav", state = "disabled", - "Protocol system state changed" + "Protocol runtime disabled" ); None } @@ -693,7 +713,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { protocol = "webdav", state = "initialization_failed", error = %e, - "Protocol system state changed" + "Protocol runtime initialization failed" ); return Err(Error::other(e)); } @@ -706,13 +726,13 @@ async fn run(config: rustfs::config::Config) -> Result<()> { #[cfg(feature = "sftp")] let sftp_shutdown_tx = match init_sftp_system().await { Ok(Some(tx)) => { - info!( + debug!( event = EVENT_PROTOCOL_SYSTEM_STATE, component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "sftp", - state = "initialized", - "Protocol system state changed" + state = "started", + "Protocol runtime started" ); Some(tx) } @@ -723,7 +743,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "sftp", state = "disabled", - "Protocol system state changed" + "Protocol runtime disabled" ); None } @@ -735,7 +755,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { protocol = "sftp", state = "initialization_failed", error = %e, - "Protocol system state changed" + "Protocol runtime initialization failed" ); return Err(Error::other(e)); } @@ -758,7 +778,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, state = "started", - "Audit system state changed" + "Audit runtime started" ), Err(e) => error!( target: "rustfs::main::run", @@ -767,7 +787,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { subsystem = LOG_SUBSYSTEM_STARTUP, state = "start_failed", error = %e, - "Audit system state changed" + "Audit runtime failed to start" ), } @@ -781,7 +801,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, state = "started", - "Deadlock detector state changed" + "Deadlock detector started" ); } else { info!( @@ -790,7 +810,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, state = "disabled", - "Deadlock detector state changed" + "Deadlock detector disabled" ); } @@ -888,7 +908,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { subsystem = LOG_SUBSYSTEM_STARTUP, enable_scanner = enable_scanner, enable_heal = enable_heal, - "Configured background services" + "Background services configured" ); // Scanner depends on the heal channel/manager, so scanner implies heal. @@ -907,7 +927,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { enable_heal = false, ahm_state = "skipped", reason = "disabled", - "Background services configured" + "Background services disabled" ); } @@ -935,7 +955,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { server_address = %server_address, started_at = %jiff::Zoned::now(), iam_bootstrap = ?iam_bootstrap, - "RustFS server started successfully" + "RustFS server ready" ); if iam_bootstrap == IamBootstrapDisposition::ReadyInline { rustfs::server::publish_ready_when_runtime_ready(readiness.as_ref(), Some(state_manager.as_ref())).await?; @@ -971,7 +991,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { subsystem = LOG_SUBSYSTEM_STARTUP, state = ?state_manager.current_state(), result = "stopped", - "Server shutdown state changed" + "RustFS server stopped" ); Ok(()) } @@ -1051,7 +1071,7 @@ async fn handle_shutdown( component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, signal = shutdown_signal.log_label(), - "Shutdown signal received in main thread" + "Shutdown signal received" ); // update the status to stopping first state_manager.update(ServiceState::Stopping); @@ -1114,7 +1134,7 @@ async fn handle_shutdown( subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "ftp", state = "stopping", - "Protocol system state changed" + "Protocol runtime stopping" ); protocol_shutdowns.push(ftp_shutdown_tx.shutdown()); } @@ -1127,7 +1147,7 @@ async fn handle_shutdown( subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "ftps", state = "stopping", - "Protocol system state changed" + "Protocol runtime stopping" ); protocol_shutdowns.push(ftps_shutdown_tx.shutdown()); } @@ -1141,7 +1161,7 @@ async fn handle_shutdown( subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "webdav", state = "stopping", - "Protocol system state changed" + "Protocol runtime stopping" ); protocol_shutdowns.push(webdav_shutdown_tx.shutdown()); } @@ -1155,7 +1175,7 @@ async fn handle_shutdown( subsystem = LOG_SUBSYSTEM_STARTUP, protocol = "sftp", state = "stopping", - "Protocol system state changed" + "Protocol runtime stopping" ); protocol_shutdowns.push(sftp_shutdown_tx.shutdown()); } @@ -1178,7 +1198,7 @@ async fn handle_shutdown( component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, state = "stopping", - "Audit system state changed" + "Audit runtime stopping" ); match stop_audit_system().await { Ok(_) => info!( @@ -1187,7 +1207,7 @@ async fn handle_shutdown( component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, state = "stopped", - "Audit system state changed" + "Audit runtime stopped" ), Err(e) => error!( target: "rustfs::main::handle_shutdown", @@ -1196,7 +1216,7 @@ async fn handle_shutdown( subsystem = LOG_SUBSYSTEM_STARTUP, state = "stop_failed", error = %e, - "Audit system state changed" + "Audit runtime failed to stop" ), } @@ -1217,7 +1237,7 @@ async fn handle_shutdown( component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, state = "stopping", - "Server shutdown state changed" + "RustFS server stopping" ); if let Some(s3_shutdown_handle) = s3_shutdown_handle { s3_shutdown_handle.shutdown().await; @@ -1235,7 +1255,7 @@ async fn handle_shutdown( component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, state = "stopped", - "Server shutdown state changed" + "RustFS server stopped" ); } diff --git a/rustfs/src/profiling.rs b/rustfs/src/profiling.rs index cafc13982..7e41ad577 100644 --- a/rustfs/src/profiling.rs +++ b/rustfs/src/profiling.rs @@ -16,26 +16,37 @@ mod unsupported_impl { use std::path::PathBuf; use std::time::Duration; - use tracing::info; + use tracing::{debug, info}; + + const LOG_COMPONENT_PROFILING: &str = "profiling"; + const LOG_SUBSYSTEM_PLATFORM: &str = "platform"; pub async fn init_from_env() { let target_env = option_env!("CARGO_CFG_TARGET_ENV").unwrap_or("unknown"); info!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_PLATFORM, + event = "profiling_runtime_skipped", + reason = "unsupported_platform", target_os = std::env::consts::OS, target_env, target_arch = std::env::consts::ARCH, - "Profiling initialization skipped on unsupported platform" + "Profiling runtime skipped" ); } /// Stop all background profiling tasks pub fn shutdown_profiling() { let target_env = option_env!("CARGO_CFG_TARGET_ENV").unwrap_or("unknown"); - info!( + debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_PLATFORM, + event = "profiling_shutdown_skipped", + reason = "unsupported_platform", target_os = std::env::consts::OS, target_env, target_arch = std::env::consts::ARCH, - "profiling: shutdown called on unsupported platform (no-op)" + "Profiling shutdown skipped" ); } @@ -80,7 +91,10 @@ mod linux_impl { use tracing::{debug, info, warn}; const LOG_COMPONENT_PROFILING: &str = "profiling"; - const LOG_SUBSYSTEM_PROFILING: &str = "profiling"; + const LOG_SUBSYSTEM_CPU: &str = "cpu"; + const LOG_SUBSYSTEM_MEMORY: &str = "memory"; + const LOG_SUBSYSTEM_JEMALLOC: &str = "jemalloc"; + const LOG_SUBSYSTEM_RUNTIME: &str = "runtime"; static CPU_CONT_GUARD: OnceLock>>>> = OnceLock::new(); static PROFILING_CANCEL_TOKEN: OnceLock = OnceLock::new(); @@ -100,7 +114,7 @@ mod linux_impl { if let Err(e) = create_dir_all(&p) { warn!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_RUNTIME, event = "profiling_output_dir_fallback", path = %p.display(), error = %e, @@ -143,7 +157,7 @@ mod linux_impl { write_pprof_report_pb(&report, &out)?; info!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, event = "profiling_dump_exported", profile_type = "cpu", path = %out.display(), @@ -159,7 +173,7 @@ mod linux_impl { if let Some(ref guard) = *guard_slot { debug!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, event = "profiling_dump_source", profile_type = "cpu", source = "continuous_guard", @@ -195,7 +209,7 @@ mod linux_impl { f.write_all(&bytes).map_err(|e| format!("write file failed: {e}"))?; info!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_MEMORY, event = "profiling_dump_exported", profile_type = "memory", path = %out.display(), @@ -215,24 +229,69 @@ mod linux_impl { use tikv_jemalloc_ctl::{config, epoch, stats}; if let Err(e) = epoch::advance() { - warn!("jemalloc epoch advance failed: {e}"); + warn!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_JEMALLOC, + event = "jemalloc_epoch_advance_failed", + error = %e, + "Jemalloc profiling state changed" + ); } match config::malloc_conf::read() { - Ok(conf) => debug!("jemalloc malloc_conf: {}", conf), - Err(e) => debug!("jemalloc read malloc_conf failed: {e}"), + Ok(conf) => debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_JEMALLOC, + event = "jemalloc_malloc_conf", + result = "ok", + malloc_conf = %conf, + "Jemalloc profiling state checked" + ), + Err(e) => debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_JEMALLOC, + event = "jemalloc_malloc_conf", + result = "read_failed", + error = %e, + "Jemalloc profiling state checked" + ), } match std::env::var("MALLOC_CONF") { - Ok(v) => debug!("MALLOC_CONF={}", v), - Err(_) => debug!("MALLOC_CONF is not set"), + Ok(v) => debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_JEMALLOC, + event = "jemalloc_malloc_conf_env", + state = "set", + malloc_conf = %v, + "Jemalloc profiling state checked" + ), + Err(_) => debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_JEMALLOC, + event = "jemalloc_malloc_conf_env", + state = "unset", + "Jemalloc profiling state checked" + ), } if let Some(lock) = jemalloc_pprof::PROF_CTL.as_ref() { let ctl = lock.lock().await; - info!(activated = ctl.activated(), "jemalloc profiling status"); + info!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_JEMALLOC, + event = "jemalloc_profiling_status", + activated = ctl.activated(), + "Jemalloc profiling status checked" + ); } else { - info!("jemalloc profiling controller is NOT available"); + debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_JEMALLOC, + event = "jemalloc_profiling_status", + state = "unavailable", + "Jemalloc profiling status checked" + ); } let _ = epoch::advance(); @@ -253,7 +312,14 @@ mod linux_impl { #[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] pub async fn check_jemalloc_profiling() { - debug!("jemalloc profiling status check skipped on unsupported target"); + debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_JEMALLOC, + event = "jemalloc_profiling_status", + result = "skipped", + reason = "unsupported_target", + "Jemalloc profiling status checked" + ); } // Internal: start continuous CPU profiling @@ -263,11 +329,11 @@ mod linux_impl { if slot.is_some() { warn!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, event = "profiling_state", profile_type = "cpu_continuous", state = "already_running", - "Profiling state changed" + "CPU profiling already running" ); return; } @@ -280,22 +346,22 @@ mod linux_impl { *slot = Some(guard); info!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, event = "profiling_state", profile_type = "cpu_continuous", state = "started", freq_hz, - "Profiling state changed" + "CPU profiling started" ); } Err(e) => warn!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, event = "profiling_state", profile_type = "cpu_continuous", state = "start_failed", error = %e, - "Profiling state changed" + "CPU profiling failed to start" ), } } @@ -304,14 +370,14 @@ mod linux_impl { async fn start_cpu_periodic(freq_hz: i32, interval: Duration, duration: Duration, token: CancellationToken) { info!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, event = "profiling_state", profile_type = "cpu_periodic", state = "started", freq_hz, ?interval, ?duration, - "Profiling state changed" + "Periodic CPU profiling started" ); tokio::spawn(async move { loop { @@ -319,11 +385,11 @@ mod linux_impl { _ = token.cancelled() => { info!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, event = "profiling_state", profile_type = "cpu_periodic", state = "cancelled", - "Profiling state changed" + "Periodic CPU profiling cancelled" ); break; } @@ -337,7 +403,15 @@ mod linux_impl { let guard = match pprof::ProfilerGuard::new(freq_hz) { Ok(g) => g, Err(e) => { - warn!("periodic CPU profiler create failed: {e}"); + warn!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, + event = "profiling_capture_failed", + profile_type = "cpu_periodic", + stage = "create_guard", + error = %e, + "Profiling capture failed" + ); continue; } }; @@ -346,11 +420,11 @@ mod linux_impl { _ = token.cancelled() => { info!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, event = "profiling_state", profile_type = "cpu_periodic", state = "cancelled_during_capture", - "Profiling state changed" + "Periodic CPU profiling cancelled during capture" ); break; } @@ -361,11 +435,20 @@ mod linux_impl { Ok(report) => { let out = output_dir().join(format!("cpu_profile_{}.pb", ts())); if let Err(e) = write_pprof_report_pb(&report, &out) { - warn!("write periodic CPU pprof failed: {e}"); - } else { - info!( + warn!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, + event = "profiling_dump_failed", + profile_type = "cpu_periodic", + stage = "write_dump", + path = %out.display(), + error = %e, + "Periodic CPU dump write failed" + ); + } else { + debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, event = "profiling_dump_exported", profile_type = "cpu_periodic", path = %out.display(), @@ -373,7 +456,15 @@ mod linux_impl { ); } } - Err(e) => warn!("periodic CPU report build failed: {e}"), + Err(e) => warn!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, + event = "profiling_capture_failed", + profile_type = "cpu_periodic", + stage = "build_report", + error = %e, + "Profiling capture failed" + ), } } }); @@ -384,12 +475,12 @@ mod linux_impl { async fn start_memory_periodic(interval: Duration, token: CancellationToken) { info!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_MEMORY, event = "profiling_state", profile_type = "memory_periodic", state = "started", ?interval, - "Profiling state changed" + "Periodic memory profiling started" ); tokio::spawn(async move { loop { @@ -397,11 +488,11 @@ mod linux_impl { _ = token.cancelled() => { info!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_MEMORY, event = "profiling_state", profile_type = "memory_periodic", state = "cancelled", - "Profiling state changed" + "Periodic memory profiling cancelled" ); break; } @@ -409,29 +500,61 @@ mod linux_impl { } let Some(lock) = jemalloc_pprof::PROF_CTL.as_ref() else { - debug!("skip memory dump: PROF_CTL not available"); + debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_MEMORY, + event = "profiling_dump_skipped", + profile_type = "memory_periodic", + reason = "prof_ctl_unavailable", + "Profiling dump skipped" + ); continue; }; let mut ctl = lock.lock().await; if !ctl.activated() { - debug!("skip memory dump: jemalloc profiling not active"); + debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_MEMORY, + event = "profiling_dump_skipped", + profile_type = "memory_periodic", + reason = "jemalloc_inactive", + "Profiling dump skipped" + ); continue; } let out = output_dir().join(format!("mem_profile_periodic_{}.pb", ts())); match File::create(&out) { Err(e) => { - tracing::error!("periodic mem dump create file failed: {}", e); + tracing::error!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_MEMORY, + event = "profiling_dump_failed", + profile_type = "memory_periodic", + stage = "create_file", + path = %out.display(), + error = %e, + "Periodic memory dump file creation failed" + ); continue; } Ok(mut f) => match ctl.dump_pprof() { Ok(bytes) => { if let Err(e) = f.write_all(&bytes) { - tracing::error!("periodic mem dump write failed: {}", e); - } else { - info!( + tracing::error!( component = LOG_COMPONENT_PROFILING, - subsystem = LOG_SUBSYSTEM_PROFILING, + subsystem = LOG_SUBSYSTEM_MEMORY, + event = "profiling_dump_failed", + profile_type = "memory_periodic", + stage = "write_dump", + path = %out.display(), + error = %e, + "Periodic memory dump write failed" + ); + } else { + debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_MEMORY, event = "profiling_dump_exported", profile_type = "memory_periodic", path = %out.display(), @@ -439,7 +562,15 @@ mod linux_impl { ); } } - Err(e) => tracing::error!("periodic mem dump failed: {}", e), + Err(e) => tracing::error!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_MEMORY, + event = "profiling_dump_failed", + profile_type = "memory_periodic", + stage = "dump_pprof", + error = %e, + "Periodic memory dump export failed" + ), }, } } @@ -448,14 +579,27 @@ mod linux_impl { #[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] async fn start_memory_periodic(_interval: Duration, _token: CancellationToken) { - debug!("periodic memory profiling skipped on unsupported target"); + debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_MEMORY, + event = "profiling_runtime_skipped", + profile_type = "memory_periodic", + reason = "unsupported_target", + "Profiling runtime skipped" + ); } // Public: unified init entry, avoid duplication/conflict pub async fn init_from_env() { let enabled = get_env_bool(ENV_ENABLE_PROFILING, DEFAULT_ENABLE_PROFILING); if !enabled { - debug!("profiling: disabled by env"); + debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_RUNTIME, + event = "profiling_runtime_disabled", + reason = "env_flag", + "Profiling runtime disabled" + ); return; } @@ -472,7 +616,14 @@ mod linux_impl { let cpu_duration = Duration::from_secs(get_env_u64(ENV_CPU_DURATION_SECS, DEFAULT_CPU_DURATION_SECS)); match cpu_mode { - CpuMode::Off => debug!("profiling: CPU mode off"), + CpuMode::Off => debug!( + component = LOG_COMPONENT_PROFILING, + subsystem = LOG_SUBSYSTEM_CPU, + event = "profiling_mode_selected", + profile_type = "cpu", + state = "off", + "Profiling mode selected" + ), CpuMode::Continuous => start_cpu_continuous(cpu_freq).await, CpuMode::Periodic => start_cpu_periodic(cpu_freq, cpu_interval, cpu_duration, token.clone()).await, } diff --git a/rustfs/src/protocols/client.rs b/rustfs/src/protocols/client.rs index 3833288f2..340ba0af1 100644 --- a/rustfs/src/protocols/client.rs +++ b/rustfs/src/protocols/client.rs @@ -207,7 +207,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli bucket, object = %key, start_pos = ?start_pos, - "Protocol storage client request details" + "Protocol storage client request" ); let mut builder = GetObjectInput::builder().bucket(bucket.to_string()).key(key.to_string()); @@ -513,7 +513,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli object = %key, range_start = start_pos, range_length = length, - "Protocol storage client request details" + "Protocol storage client request" ); let range = s3s::dto::Range::Int { diff --git a/scripts/check_logging_guardrails.sh b/scripts/check_logging_guardrails.sh index b5f844c01..b4ff5afe7 100755 --- a/scripts/check_logging_guardrails.sh +++ b/scripts/check_logging_guardrails.sh @@ -6,15 +6,35 @@ cd "$repo_root" checked_files=( "rustfs/src/main.rs" + "rustfs/src/init.rs" + "rustfs/src/profiling.rs" "rustfs/src/startup_iam.rs" "rustfs/src/auth.rs" "rustfs/src/protocols/client.rs" + "rustfs/src/admin/router.rs" + "rustfs/src/admin/handlers/table_catalog.rs" + "rustfs/src/admin/handlers/service_account.rs" + "rustfs/src/admin/handlers/kms_dynamic.rs" + "rustfs/src/admin/handlers/site_replication.rs" + "rustfs/src/admin/handlers/group.rs" + "rustfs/src/admin/handlers/quota.rs" + "rustfs/src/admin/handlers/rebalance.rs" + "rustfs/src/admin/handlers/tier.rs" "crates/audit/src/pipeline.rs" "crates/audit/src/system.rs" "crates/audit/src/global.rs" "crates/notify/src/config_manager.rs" "crates/notify/src/runtime_facade.rs" + "crates/notify/src/integration.rs" "crates/notify/src/notifier.rs" + "crates/notify/src/bucket_config_manager.rs" + "crates/notify/src/rule_engine.rs" + "crates/notify/src/global.rs" + "crates/audit/src/registry.rs" + "crates/audit/src/observability.rs" + "crates/targets/src/store.rs" + "crates/targets/src/target/mqtt.rs" + "crates/targets/src/target/webhook.rs" "crates/ecstore/src/store/peer.rs" "crates/ecstore/src/store/init.rs" "crates/ecstore/src/tier/tier.rs" @@ -100,6 +120,25 @@ forbidden_patterns=( 'info!("Loaded notification config for bucket: {}"' 'info!("Updated notification rules for bucket: {}"' 'info!("Removed all notification rules for bucket: {}"' + 'info!(event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,' + 'info!("Notification system instance is being dropped"' + 'info!("Notification shutdown metric snapshot"' + 'info!("Notification system status snapshot"' + 'debug!("Audit replay stream skipped"' + 'warn!("Dropped queued audit payload"' + 'error!("Queued audit payload failed permanently"' + 'info!(target_id = %target_id.id, "Webhook target created")' + 'info!("Webhook target {} initialized"' + 'info!("Webhook sending queued payload to target: {}"' + 'debug!("Event saved to store for target: {}"' + 'debug!("Event sent from store and deleted for target: {}"' + 'info!("Webhook target closed: {}"' + 'info!(target_id = %target_id, "MQTT target created")' + 'info!(target_id = %self.id, "MQTT target initialized and connected.")' + 'info!(target_id = %target_id, "MQTT event loop task started.")' + 'info!(target_id = %self.id, "MQTT target close method finished.")' + 'debug!("Wrote event to store: {}"' + 'debug!("Deleted event from store: {}"' 'info!("Audit configuration reloaded"' 'info!("Audit system started"' 'info!("Audit metrics reset"' @@ -108,17 +147,117 @@ forbidden_patterns=( 'error!("Server encountered an error and is shutting down: {}"' 'error!("Failed to initialize Keystone authentication: {}"' 'error!("new_global_notification_sys failed {:?}"' + 'warn!(access_key = %new_cred.access_key, error = ?err, "site replication add service account hook failed")' + 'warn!(access_key = %query.access_key, error = ?err, "site replication delete service account hook failed")' + 'debug!("check key failed: {e:?}")' + 'debug!("list service account failed: {e:?}")' + 'debug!("list sts account failed: {e:?}")' + 'error!("license check failed due to unexpected error: {er}")' + 'info!("Created KMS key: {}"' + 'info!("Successfully deleted KMS key: {}"' + 'info!("Cancelled deletion for KMS key: {}"' + 'info!("Listed {} KMS keys"' + 'info!("Described KMS key: {}"' + 'error!("Failed to create KMS key: {}"' + 'error!("Failed to list KMS keys: {}"' + 'error!("Failed to describe KMS key {}: {}"' + 'warn!(access_key = %ak, error = ?err, "site replication create user hook failed")' + 'info!("OIDC authorize redirect for provider' + 'warn!("OIDC callback received error from IdP:' + 'error!("OIDC code exchange failed: {}"' + 'warn!("OIDC logout fallback triggered: {}"' + 'warn!("get body failed, e: {:?}")' + 'warn!("file path is invalid: {}")' + 'warn!("bucket metadata not found: {e}")' + 'warn!("get bucket metadata failed: {e}")' + 'warn!("create bucket failed: {e}")' + 'warn!("deserialize config failed: {e}")' + 'warn!(policy = %query.name, error = ?err, "site replication policy add hook failed")' + 'warn!(policy = %query.name, error = ?err, "site replication policy delete hook failed")' + 'warn!(target = %query.user_or_group, error = ?err, "site replication policy mapping hook failed")' + 'warn!(access_key = %ak, error = ?err, "site replication delete user hook failed")' + 'warn!(target = %target_name, error = ?err, "site replication policy association hook failed")' + 'error!("Failed to serialize response: {}"' + 'warn!(peer = %peer.endpoint, error = ?err, "site replication peer metainfo fetch failed")' + 'warn!(error = ?err, "site replication backfill: failed to list buckets")' + 'warn!(bucket = %name, error = ?err, "site replication backfill: versioning setup failed")' + 'warn!(bucket = %name, error = ?err, "site replication backfill: targets setup failed")' + 'warn!(bucket = %name, error = ?err, "site replication backfill: replication config setup failed")' + 'warn!(bucket = %name, error = ?err, "site replication backfill: failed to read bucket metadata, assuming lock_enabled=false")' + 'warn!(bucket = %name, error = ?err, "site replication backfill: make-bucket broadcast failed")' + 'warn!(bucket = %name, peer = %peer.endpoint, detail = %result.err_detail,' + 'warn!(peer = %peer.endpoint, error = %err_detail, "site replication peer remove notification failed")' + 'warn!(peer = %peer.endpoint, error = %detail, "site replication service account rotation failed for peer")' + 'warn!("handle ListGroups")' + 'warn!("handle GetGroup")' + 'warn!("handle DeleteGroup")' + 'warn!("handle SetGroupStatus")' + 'warn!("handle UpdateGroupMembers")' + 'warn!("list groups failed, e: {:?}")' + 'warn!("get group failed, e: {:?}")' + 'warn!("delete group failed, e: {:?}")' + 'warn!("site replication group delete hook failed, err: {err}")' + 'warn!("enable group failed, e: {:?}")' + 'warn!("UpdateGroupMembers args {:?}")' + 'warn!("remove group members")' + 'warn!("remove group members failed, e: {:?}")' + 'warn!("add group members")' + 'warn!("add group members failed, e: {:?}")' + 'warn!("site replication group membership hook failed, err: {err}")' + 'warn!("handle SetBucketQuota")' + 'warn!("handle GetBucketQuota")' + 'warn!("handle ClearBucketQuota")' + 'warn!("handle GetBucketQuotaStats")' + 'warn!("handle CheckBucketQuota")' + 'info!("Clearing quota for bucket: {}")' + 'info!("Successfully cleared quota for bucket: {}")' + 'debug!("Checking quota for bucket: {}, operation: {}, size: {}")' + 'warn!("handle RebalanceStart")' + 'warn!("Rebalance started with id: {}")' + 'warn!("RebalanceStart Loading rebalance meta start")' + 'warn!("rebalance start propagation failed after local state update: {err}")' + 'warn!("RebalanceStart Loading rebalance meta done")' + 'warn!("handle RebalanceStatus")' + 'warn!("handle RebalanceStop")' + 'warn!("handle RebalanceStop save_rebalance_stats done ")' + 'warn!("handle RebalanceStop notification_sys load_rebalance_meta")' + 'warn!("rebalance stop propagation failed after local state update: {err}")' + 'warn!("handle RebalanceStop notification_sys load_rebalance_meta done")' + 'warn!("get body failed, e: {:?}")' + 'debug!("add tier args {:?}")' + 'warn!("parse force failed, e: {:?}")' + 'warn!("tier reserved name, args.name: {}")' + 'warn!("tier_config_mgr reload failed, e: {:?}")' + 'warn!("tier_config_mgr add failed, e: {:?}")' + 'debug!("edit tier args {:?}")' + 'warn!("tier_config_mgr edit failed, e: {:?}")' + 'warn!("tier_config_mgr remove failed, e: {:?}")' + 'warn!("tier_config_mgr rand: {}")' + 'warn!("tier_config_mgr clear failed, e: {:?}")' + 'warn!("tier_config_mgr save failed, e: {:?}")' + 'warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA, + event = "object_lambda_tls_verification_disabled"' 'info!("FTP system initialized successfully"' + 'debug!("FTP system is disabled")' 'info!("FTP system disabled"' + 'info!("FTP server shutdown completed")' 'error!("Failed to initialize FTP system: {}"' 'info!("FTPS system initialized successfully"' + 'debug!("FTPS system is disabled")' 'info!("FTPS system disabled"' + 'info!("FTPS server shutdown completed")' 'error!("Failed to initialize FTPS system: {}"' 'info!("WebDAV system initialized successfully"' + 'debug!("WebDAV system is disabled")' 'info!("WebDAV system disabled"' + 'info!("WebDAV server shutdown completed")' 'error!("Failed to initialize WebDAV system: {}"' 'info!("SFTP system initialized successfully"' + 'debug!("SFTP system is disabled")' 'info!("SFTP system disabled"' + 'info!("SFTP server shutdown completed")' 'error!("Failed to initialize SFTP system: {}"' 'warn!("prewarm_local_disk_id_map: failed to load disk id for {}: {}"' 'info!("retrying get formats after {:?}"' @@ -174,6 +313,12 @@ forbidden_patterns=( 'warn!("KMS initialization skipped: {e}")' 'warn!("Audit system: {e}")' 'warn!("notification system: {e}")' + 'info!("jemalloc profiling controller is NOT available")' + 'warn!("periodic CPU profiler create failed: {e}")' + 'warn!("write periodic CPU pprof failed: {e}")' + 'warn!("periodic CPU report build failed: {e}")' + 'debug!("profiling: disabled by env")' + 'debug!("profiling: CPU mode off")' 'info!("Starting HealManager")' 'info!("Stopping HealManager")' 'info!("HealManager started successfully")' @@ -196,6 +341,27 @@ forbidden_patterns=( 'info!("Heal queue has {} pending requests, {} tasks active")' 'error!("Heal channel processor failed: {}")' 'info!("Heal manager with channel processor initialized successfully")' + '"Heal object workflow started"' + '"Heal object recovery started"' + '"Heal object completed"' + '"Heal object cleanup completed"' + '"Heal metadata workflow started"' + '"Heal metadata completed"' + '"Heal MRF workflow started"' + '"Heal MRF completed"' + '"Heal EC decode workflow started"' + '"Heal EC decode completed"' + '"Heal erasure set workflow started"' + '"Heal erasure set format repair completed"' + '"Heal erasure set completed"' + '"Heal storage repair finished"' + '"Heal storage admin operation finished"' + '"Heal storage request finished"' + '"Erasure set object heal completed"' + '"Erasure set heal resumed from checkpoint"' + '"Scanner disk bucket state updated"' + '"Scanner lifecycle action evaluation started"' + '"Scanner state persisted"' '"scanner deep heal downgraded to normal during new-object cooldown"' '"scanner detected folder with excessive direct subfolders"' '"scan_folder: failed to get size for item {}: {}"'