mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-29 17:48:58 +00:00
Compare commits
34 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1c8088d0b2 | |||
| ffcdab900a | |||
| b94bf874bf | |||
| 7d96ffd7fb | |||
| 027a749646 | |||
| 6c23b8506e | |||
| 8166561702 | |||
| 9866f68d86 | |||
| 0e8f1a187c | |||
| b32bd1f8a9 | |||
| c9848a6096 | |||
| 6e88ab2a25 | |||
| 05d4480f08 | |||
| abee09dad9 | |||
| a044d11443 | |||
| e1e6a8b020 | |||
| 5cfe4ccc7d | |||
| df2db15ce8 | |||
| 0321e9350d | |||
| cd9a2eecb1 | |||
| 8b09634e62 | |||
| 1ede77b1c1 | |||
| 666e251b78 | |||
| 65ba138c27 | |||
| 92ae19b340 | |||
| 4607c3be53 | |||
| e0bac66941 | |||
| 31dc78eab0 | |||
| daca7294c7 | |||
| 1655f3192e | |||
| 0adb3c5ea1 | |||
| 68e156a5c5 | |||
| 3f60cc743e | |||
| 35af4a611f |
Generated
+122
-112
@@ -698,6 +698,7 @@ version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d83a251fa1a4c9d0fe6e816b7acd60549e473e08d14f27a1d992c2675abff05f"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
@@ -708,7 +709,6 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
"rand 0.10.2",
|
||||
"regex",
|
||||
"ring",
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
@@ -769,7 +769,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.2",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -850,9 +850,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "aws-config"
|
||||
version = "1.9.0"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319"
|
||||
checksum = "701418aa459dac33e50a0f8e818e5662a16bc018a6ac7423659b70f3799d67a8"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
@@ -917,9 +917,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-runtime"
|
||||
version = "1.8.1"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7816e98ee912159f45d307e5ee6bfea4a335a55aee15f7f3e32f81a6f3000f1d"
|
||||
checksum = "a6b50a43f3ccdf331521c6d6c68b7cc9668b6e09d439ebda9569df5722324d76"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-sigv4",
|
||||
@@ -945,9 +945,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.138.1"
|
||||
version = "1.139.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fa59420755799fec8aac71f562aa37ba82263f5b1926bba8b5d3465d2f7a73f"
|
||||
checksum = "a159b9721a6a41468f967d1029bece78f410b0beb0594498435deb6ff72bfe48"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -982,9 +982,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sso"
|
||||
version = "1.103.0"
|
||||
version = "1.104.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0469f435f645ad2162cfb463b15bde37115966ee3acf2d87fb4871ee309b8401"
|
||||
checksum = "b53416d16c278234845392e38d93bd4481d2f09daa0f005a2277f0aa91f59c22"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1008,9 +1008,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-ssooidc"
|
||||
version = "1.105.0"
|
||||
version = "1.106.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "085faefb253f770655e162b9304321e62a1e71adf7f019ee1f4454228a377b3a"
|
||||
checksum = "cc9b706c3305ed0285d5b1b696c747aa34950f830fb03e3e6c76890f99b9f188"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1034,9 +1034,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sts"
|
||||
version = "1.108.0"
|
||||
version = "1.109.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c72b08911d8128dd360fe1b22a9fec0fa8b552dde8ec828dcf20ef5ec974e9f"
|
||||
checksum = "32d214cdfa5bbe17f117e76a7643fadf32a5234fb597322ef8b1fb4b2f17dbbd"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1197,11 +1197,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-query"
|
||||
version = "0.61.1"
|
||||
version = "0.62.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd22a6ba36e3f113cb8d5b3d1fe0ed31c76ee608ef63322d753bb8d2c9479e77"
|
||||
checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
"aws-smithy-types",
|
||||
"aws-smithy-xml",
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
@@ -1299,9 +1302,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-xml"
|
||||
version = "0.61.1"
|
||||
version = "0.62.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea3f68eec3607f02acd24067969ce2abc6ba16aa7d5ce59ca450ed2fb5f78957"
|
||||
checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
@@ -1311,9 +1314,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-types"
|
||||
version = "1.4.0"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e957a6c6dbce82b7a91f44231c09273159703769f447cbe85e854dfe9cf67f86"
|
||||
checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-async",
|
||||
@@ -1519,7 +1522,7 @@ version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1538,7 +1541,7 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1861,7 +1864,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -1879,9 +1882,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.3"
|
||||
version = "4.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776"
|
||||
checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
@@ -1901,14 +1904,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.6.3"
|
||||
version = "4.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32f2392eae7f16557a3d727ef3a12e57b2b2ca6f98566a5f4fb41ffe305df077"
|
||||
checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2319,7 +2322,7 @@ version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -2344,11 +2347,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
@@ -3545,7 +3548,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3645,7 +3648,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -3682,6 +3685,7 @@ dependencies = [
|
||||
"rustfs-protos",
|
||||
"rustfs-rio",
|
||||
"rustfs-signer",
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"s3s",
|
||||
"serde",
|
||||
@@ -3798,7 +3802,7 @@ dependencies = [
|
||||
"crypto-bigint 0.5.5",
|
||||
"digest 0.10.7",
|
||||
"ff 0.13.1",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"group 0.13.0",
|
||||
"hkdf 0.12.4",
|
||||
"pem-rfc7468 0.7.0",
|
||||
@@ -4244,9 +4248,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4259,7 +4263,7 @@ version = "1.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rustversion",
|
||||
"typenum",
|
||||
]
|
||||
@@ -4340,9 +4344,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.3"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
|
||||
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
|
||||
|
||||
[[package]]
|
||||
name = "gloo-timers"
|
||||
@@ -5259,7 +5263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5710,9 +5714,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.187"
|
||||
version = "0.2.189"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7743783ea728ef5c31194c6590797eed286449b4a4e87d626d8a51f0a94e732"
|
||||
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "libflate"
|
||||
@@ -6080,9 +6084,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique"
|
||||
version = "0.1.27"
|
||||
version = "0.1.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6737991308ea171744eac9bb26885d6caa0b747d3cca72f7a69f38e689a1886"
|
||||
checksum = "aa466af30a9fe0b1db1dae097e0ce7ddac4b666b1d3a9f5c682e889272511dd8"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"jiff",
|
||||
@@ -6110,9 +6114,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique-macro"
|
||||
version = "0.1.18"
|
||||
version = "0.1.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6512443e1f72c1faa670da0e7cc347dbdd6f17b8e97c3a10d0de56365887f18d"
|
||||
checksum = "1f5febfaf14fea234b60e0ad43c728db274033893a38d0fa1f87d9b7056d3d61"
|
||||
dependencies = [
|
||||
"Inflector",
|
||||
"darling 0.23.0",
|
||||
@@ -7947,9 +7951,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pyroscope"
|
||||
version = "2.1.0"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b9adfbc87341b5976c0bbab196ed2e2c188e2acf9f4d434d46eb9433f0f529c"
|
||||
checksum = "59b1989a61e856eba4ef903eadad8313f8376e88e7c9cc5d5db8217d832a64d9"
|
||||
dependencies = [
|
||||
"aligned-vec",
|
||||
"backtrace",
|
||||
@@ -8269,8 +8273,8 @@ version = "0.14.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"pem",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"time",
|
||||
"x509-parser",
|
||||
@@ -8394,7 +8398,7 @@ checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.2",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8821,7 +8825,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -8955,7 +8959,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -8977,7 +8981,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -8992,7 +8996,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"metrics",
|
||||
@@ -9007,7 +9011,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"insta",
|
||||
"rustfs-io-core",
|
||||
@@ -9019,7 +9023,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"serde",
|
||||
@@ -9028,7 +9032,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9041,7 +9045,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9061,7 +9065,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"path-clean",
|
||||
@@ -9072,7 +9076,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -9207,7 +9211,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -9216,7 +9220,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"byteorder",
|
||||
@@ -9242,7 +9246,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -9272,7 +9276,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9309,7 +9313,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"memmap2",
|
||||
@@ -9321,7 +9325,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"metrics",
|
||||
@@ -9384,7 +9388,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -9410,7 +9414,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -9441,7 +9445,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"proptest",
|
||||
@@ -9461,7 +9465,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"crossbeam-queue",
|
||||
@@ -9482,7 +9486,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-log-analyzer"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"flate2",
|
||||
@@ -9500,7 +9504,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"humantime",
|
||||
@@ -9514,7 +9518,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9537,8 +9541,10 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"starshard",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
@@ -9546,7 +9552,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -9564,7 +9570,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -9580,7 +9586,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -9632,7 +9638,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -9661,7 +9667,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -9720,7 +9726,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"prost 0.14.4",
|
||||
@@ -9742,7 +9748,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -9759,7 +9765,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -9797,7 +9803,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -9819,14 +9825,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"rustfs-s3-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -9834,7 +9840,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -9862,7 +9868,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
@@ -9878,7 +9884,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -9910,14 +9916,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9933,7 +9939,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"insta",
|
||||
@@ -9947,7 +9953,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -9955,6 +9961,7 @@ dependencies = [
|
||||
"chrono",
|
||||
"criterion",
|
||||
"deadpool-postgres",
|
||||
"futures-util",
|
||||
"hashbrown 0.17.1",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
@@ -9964,6 +9971,7 @@ dependencies = [
|
||||
"mysql_async",
|
||||
"parking_lot",
|
||||
"pulsar",
|
||||
"rayon",
|
||||
"rcgen",
|
||||
"redis",
|
||||
"regex",
|
||||
@@ -9988,6 +9996,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tokio-postgres",
|
||||
"tokio-postgres-rustls",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
@@ -9997,7 +10006,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"rustfs-data-usage",
|
||||
"rustfs-ecstore",
|
||||
@@ -10012,7 +10021,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"metrics",
|
||||
@@ -10032,7 +10041,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -10068,7 +10077,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
@@ -10089,6 +10098,7 @@ dependencies = [
|
||||
"netif",
|
||||
"proptest",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rustix",
|
||||
"serde",
|
||||
"sha1 0.11.0",
|
||||
@@ -10107,7 +10117,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -10184,7 +10194,6 @@ dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
"once_cell",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
"subtle",
|
||||
@@ -10455,7 +10464,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct 0.2.0",
|
||||
"der 0.7.10",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -10561,7 +10570,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.2",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10615,7 +10624,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.2",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11320,9 +11329,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.2"
|
||||
version = "3.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3"
|
||||
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -11541,7 +11550,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.2",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11742,9 +11751,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio-stream"
|
||||
version = "0.1.18"
|
||||
version = "0.1.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
|
||||
checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"pin-project-lite",
|
||||
@@ -11765,15 +11774,16 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.7.18"
|
||||
version = "0.7.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
|
||||
checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"futures-util",
|
||||
"libc",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
@@ -11784,6 +11794,7 @@ version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -11791,7 +11802,6 @@ dependencies = [
|
||||
"http 1.4.2",
|
||||
"httparse",
|
||||
"rand 0.8.7",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
@@ -12823,12 +12833,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202"
|
||||
dependencies = [
|
||||
"asn1-rs",
|
||||
"aws-lc-rs",
|
||||
"data-encoding",
|
||||
"der-parser",
|
||||
"lazy_static",
|
||||
"nom 7.1.3",
|
||||
"oid-registry",
|
||||
"ring",
|
||||
"rusticata-macros",
|
||||
"thiserror 2.0.19",
|
||||
"time",
|
||||
@@ -12876,9 +12886,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "xxhash-rust"
|
||||
version = "0.8.17"
|
||||
version = "0.8.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72"
|
||||
checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6"
|
||||
|
||||
[[package]]
|
||||
name = "yansi"
|
||||
|
||||
+58
-58
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.96.0"
|
||||
version = "1.0.0-beta.10"
|
||||
version = "1.0.0-beta.11"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,52 +86,52 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-beta.10" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.10" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.10" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.10" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-beta.10" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.10" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.10" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.10" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.10" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.10" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.10" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.10" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.10" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.10" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.10" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.10" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.10" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.10" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.10" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.10" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.10" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.10" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.10" }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.10" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.10" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.10" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.10" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.10" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.10" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.10" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.10" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.10" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.10" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.10" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.10" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.10" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.10" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.10" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.10" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.10" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.10" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.10" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.10" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.10" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.10" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.10" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-beta.11" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.11" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.11" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.11" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-beta.11" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.11" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.11" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.11" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.11" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.11" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.11" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.11" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.11" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.11" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.11" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.11" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.11" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.11" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.11" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.11" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.11" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.11" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.11" }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.11" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.11" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.11" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.11" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.11" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.11" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.11" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.11" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.11" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.11" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.11" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.11" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.11" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.11" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.11" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.11" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.11" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.11" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.11" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.11" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.11" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.11" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.11" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -140,7 +140,7 @@ mysql_async = { default-features = false, version = "0.37" }
|
||||
async-compression = { version = "0.4.42" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.91"
|
||||
async-nats = "0.50.0"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.33"
|
||||
futures-core = "0.3.33"
|
||||
@@ -161,9 +161,9 @@ rustfs-kafka-async = { version = "1.2.0" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.53.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.4" }
|
||||
tokio-stream = { version = "0.1.18" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.5"
|
||||
tokio-util = { version = "0.7.18" }
|
||||
tokio-util = { version = "0.7.19" }
|
||||
tonic = { version = "0.14.6" }
|
||||
tonic-prost = { version = "0.14.6" }
|
||||
tonic-prost-build = { version = "0.14.6" }
|
||||
@@ -225,16 +225,16 @@ arc-swap = "1.9.2"
|
||||
astral-tokio-tar = "0.6.4"
|
||||
atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.9.0" }
|
||||
aws-config = { version = "1.10.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.138.1" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.139.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
||||
aws-smithy-runtime-api = { version = "1.13.0" }
|
||||
aws-smithy-types = { version = "1.6.1" }
|
||||
base64 = "0.22.1"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.4"
|
||||
clap = { version = "4.6.3" }
|
||||
clap = { version = "4.6.4" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.11.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -247,7 +247,7 @@ derive_builder = "0.20.2"
|
||||
enumset = "1.1.14"
|
||||
faster-hex = "0.10.0"
|
||||
flate2 = "1.1.9"
|
||||
glob = "0.3.3"
|
||||
glob = "0.3.4"
|
||||
google-cloud-storage = "1.16.0"
|
||||
google-cloud-auth = "1.14.0"
|
||||
hashbrown = { version = "0.17.1" }
|
||||
@@ -256,7 +256,7 @@ hex-simd = "0.8.0"
|
||||
highway = { version = "1.3.0" }
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.0"
|
||||
libc = "0.2.187"
|
||||
libc = "0.2.189"
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
memmap2 = "0.9.11"
|
||||
@@ -313,7 +313,7 @@ vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
windows = { version = "0.62.2" }
|
||||
xxhash-rust = { version = "0.8.17" }
|
||||
xxhash-rust = { version = "0.8.18" }
|
||||
zip = "8.6.0"
|
||||
zstd = "0.13.3"
|
||||
|
||||
@@ -326,13 +326,13 @@ opentelemetry-otlp = { version = "0.32.0" }
|
||||
opentelemetry_sdk = { version = "0.32.1" }
|
||||
opentelemetry-semantic-conventions = { version = "0.32.1" }
|
||||
opentelemetry-stdout = { version = "0.32.0" }
|
||||
pyroscope = { version = "2.1.0" }
|
||||
pyroscope = { version = "2.1.1" }
|
||||
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = "0.14.8"
|
||||
rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.3" }
|
||||
russh-sftp = "2.3.0"
|
||||
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.10
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.11
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.10
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.11
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -292,8 +292,8 @@ impl AuditPipeline {
|
||||
}
|
||||
|
||||
pub async fn snapshot_target_health(&self) -> Vec<rustfs_targets::RuntimeTargetHealthSnapshot> {
|
||||
let registry = self.registry.lock().await;
|
||||
registry.runtime_manager().health_snapshots().await
|
||||
let targets = self.registry.lock().await.list_target_values();
|
||||
rustfs_targets::health_snapshots_for_targets(targets).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -570,7 +570,7 @@ mod tests {
|
||||
use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
|
||||
use rustfs_targets::{StoreError, Target, TargetError};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::{Mutex, Notify};
|
||||
|
||||
/// Mock target whose `save()` outcome is fixed at construction so tests can
|
||||
/// force full-success / full-failure / partial-failure fan-outs.
|
||||
@@ -578,6 +578,7 @@ mod tests {
|
||||
struct MockTarget {
|
||||
id: TargetID,
|
||||
fail: bool,
|
||||
health_gate: Option<(Arc<Notify>, Arc<Notify>)>,
|
||||
}
|
||||
|
||||
impl MockTarget {
|
||||
@@ -585,8 +586,14 @@ mod tests {
|
||||
Self {
|
||||
id: TargetID::new(id.to_string(), "webhook".to_string()),
|
||||
fail,
|
||||
health_gate: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_health_gate(mut self, started: Arc<Notify>, release: Arc<Notify>) -> Self {
|
||||
self.health_gate = Some((started, release));
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -599,6 +606,10 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn is_active(&self) -> Result<bool, TargetError> {
|
||||
if let Some((started, release)) = &self.health_gate {
|
||||
started.notify_one();
|
||||
release.notified().await;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -673,6 +684,24 @@ mod tests {
|
||||
pipeline.dispatch(entry()).await.expect("no targets should return Ok");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn health_probe_does_not_hold_the_registry_lock() {
|
||||
let started = Arc::new(Notify::new());
|
||||
let release = Arc::new(Notify::new());
|
||||
let pipeline = pipeline_with(vec![MockTarget::new("blocked", false).with_health_gate(started.clone(), release.clone())]);
|
||||
let registry = Arc::clone(&pipeline.registry);
|
||||
let snapshot_task = tokio::spawn(async move { pipeline.snapshot_target_health().await });
|
||||
started.notified().await;
|
||||
|
||||
let guard = tokio::time::timeout(std::time::Duration::from_secs(1), registry.lock())
|
||||
.await
|
||||
.expect("network health probe must not retain the audit registry lock");
|
||||
drop(guard);
|
||||
release.notify_one();
|
||||
|
||||
assert_eq!(snapshot_task.await.expect("snapshot task should finish").len(), 1);
|
||||
}
|
||||
|
||||
// backlog#962: dispatch_batch must mirror dispatch and propagate a
|
||||
// whole-batch loss instead of returning Ok.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -56,3 +56,33 @@ pub const DEFAULT_OBJECT_MMAP_READ_ENABLE: bool = true;
|
||||
///
|
||||
/// Prefer [`DEFAULT_OBJECT_MMAP_READ_ENABLE`].
|
||||
pub const DEFAULT_OBJECT_ZERO_COPY_ENABLE: bool = DEFAULT_OBJECT_MMAP_READ_ENABLE;
|
||||
|
||||
/// Environment variable capping the byte length a single mmap-copy read may
|
||||
/// materialize in memory.
|
||||
///
|
||||
/// The mmap-copy read path returns the whole requested range as one owned
|
||||
/// allocation before the first byte is served. GET/heal shard reads request
|
||||
/// the entire part span in one call, so for a large single-part object
|
||||
/// (e.g. a multi-gigabyte non-multipart upload) an uncapped mmap-copy read
|
||||
/// allocates the whole shard in memory — stalling first-byte latency past the
|
||||
/// disk-read timeout and OOM-killing memory-limited deployments
|
||||
/// (<https://github.com/rustfs/rustfs/issues/5123>). Reads longer than this
|
||||
/// cap fall back to the bounded streaming reader instead.
|
||||
///
|
||||
/// - Purpose: Bound per-shard-read memory for mmap-based reads
|
||||
/// - Acceptable values: byte count as an unsigned integer; `0` disables
|
||||
/// mmap-copy for all non-empty reads (every read streams)
|
||||
/// - Example: `export RUSTFS_OBJECT_MMAP_READ_MAX_LENGTH=8388608`
|
||||
pub const ENV_OBJECT_MMAP_READ_MAX_LENGTH: &str = "RUSTFS_OBJECT_MMAP_READ_MAX_LENGTH";
|
||||
|
||||
/// Default mmap-copy read length cap: 32 MiB per shard read.
|
||||
///
|
||||
/// Large enough that typical multipart part shards (parts up to a few hundred
|
||||
/// megabytes across the erasure set) keep the mmap fast path, small enough
|
||||
/// that whole-part reads of huge single-part objects stream instead of
|
||||
/// materializing gigabytes per shard.
|
||||
///
|
||||
/// The cap bounds memory per shard reader, so a single part read can still
|
||||
/// materialize up to `data_shards x cap` bytes; raising the cap raises that
|
||||
/// per-request bound proportionally.
|
||||
pub const DEFAULT_OBJECT_MMAP_READ_MAX_LENGTH: usize = 32 * 1024 * 1024;
|
||||
|
||||
@@ -33,6 +33,7 @@ rustfs-config = { workspace = true, features = ["constants"] }
|
||||
rustfs-ecstore.workspace = true
|
||||
rustfs-data-usage.workspace = true
|
||||
rustfs-rio.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["egress"] }
|
||||
flatbuffers.workspace = true
|
||||
futures.workspace = true
|
||||
rustfs-lock.workspace = true
|
||||
|
||||
@@ -39,6 +39,7 @@ use local_ip_address::local_ip;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
@@ -70,6 +71,11 @@ fn target_arn(target_name: &str) -> String {
|
||||
format!("arn:rustfs:sqs:{NOTIFY_REGION}:{target_name}:webhook")
|
||||
}
|
||||
|
||||
fn endpoint_origin(endpoint: &str) -> Result<String, BoxError> {
|
||||
let parsed = reqwest::Url::parse(endpoint)?;
|
||||
Ok(parsed.origin().ascii_serialization())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// In-test HTTP event receiver
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -153,10 +159,46 @@ async fn spawn_event_collector() -> Result<(String, mpsc::UnboundedReceiver<Valu
|
||||
let endpoint_ip = local_ip()?;
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
let handle = serve_event_collector(listener, tx);
|
||||
Ok((format!("http://{endpoint_ip}.nip.io:{port}/events"), rx, handle))
|
||||
Ok((format!("http://{}/events", std::net::SocketAddr::new(endpoint_ip, port)), rx, handle))
|
||||
}
|
||||
|
||||
fn spawn_https_event_collector(ca_path: &Path) -> Result<(String, Arc<AtomicBool>, thread::JoinHandle<()>), BoxError> {
|
||||
struct HttpsEventCollector {
|
||||
endpoint: String,
|
||||
running: Arc<AtomicBool>,
|
||||
handle: Option<thread::JoinHandle<()>>,
|
||||
events: mpsc::UnboundedReceiver<Value>,
|
||||
}
|
||||
|
||||
impl HttpsEventCollector {
|
||||
fn endpoint(&self) -> &str {
|
||||
&self.endpoint
|
||||
}
|
||||
|
||||
fn events_mut(&mut self) -> &mut mpsc::UnboundedReceiver<Value> {
|
||||
&mut self.events
|
||||
}
|
||||
|
||||
fn shutdown(&mut self) -> TestResult {
|
||||
self.running.store(false, Ordering::Relaxed);
|
||||
if let Ok(parsed) = self.endpoint.parse::<reqwest::Url>()
|
||||
&& let Some(port) = parsed.port()
|
||||
{
|
||||
let _ = std::net::TcpStream::connect(("127.0.0.1", port));
|
||||
}
|
||||
if let Some(handle) = self.handle.take() {
|
||||
handle.join().map_err(|_| "https event collector thread panicked")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for HttpsEventCollector {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_https_event_collector(ca_path: &Path) -> Result<HttpsEventCollector, BoxError> {
|
||||
use rustls::{
|
||||
ServerConfig,
|
||||
pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
|
||||
@@ -173,9 +215,9 @@ fn spawn_https_event_collector(ca_path: &Path) -> Result<(String, Arc<AtomicBool
|
||||
listener.set_nonblocking(true)?;
|
||||
let addr = listener.local_addr()?;
|
||||
let endpoint_ip = local_ip()?;
|
||||
let endpoint_host = format!("{endpoint_ip}.nip.io");
|
||||
let endpoint_host = endpoint_ip.to_string();
|
||||
|
||||
let rcgen::CertifiedKey { cert, signing_key } = rcgen::generate_simple_self_signed(vec![endpoint_host.clone()])?;
|
||||
let rcgen::CertifiedKey { cert, signing_key } = rcgen::generate_simple_self_signed(vec![endpoint_host])?;
|
||||
std::fs::write(ca_path, cert.pem())?;
|
||||
|
||||
let cert_chain = vec![cert.der().clone()];
|
||||
@@ -188,48 +230,103 @@ fn spawn_https_event_collector(ca_path: &Path) -> Result<(String, Arc<AtomicBool
|
||||
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let server_running = Arc::clone(&running);
|
||||
let (tx, events) = mpsc::unbounded_channel();
|
||||
let handle = thread::spawn(move || {
|
||||
let mut connections = Vec::new();
|
||||
while server_running.load(Ordering::Relaxed) {
|
||||
match listener.accept() {
|
||||
Ok((stream, _)) => {
|
||||
let config = Arc::clone(&server_config);
|
||||
handle_https_probe(stream, config);
|
||||
let tx = tx.clone();
|
||||
connections.push(thread::spawn(move || {
|
||||
let _ = handle_https_request(stream, config, tx);
|
||||
}));
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::WouldBlock => thread::sleep(Duration::from_millis(20)),
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
for connection in connections {
|
||||
let _ = connection.join();
|
||||
}
|
||||
});
|
||||
|
||||
Ok((format!("https://{endpoint_host}:{}/events", addr.port()), running, handle))
|
||||
Ok(HttpsEventCollector {
|
||||
endpoint: format!("https://{}/events", std::net::SocketAddr::new(endpoint_ip, addr.port())),
|
||||
running,
|
||||
handle: Some(handle),
|
||||
events,
|
||||
})
|
||||
}
|
||||
|
||||
fn handle_https_probe(stream: std::net::TcpStream, server_config: Arc<rustls::ServerConfig>) {
|
||||
use std::io::{Read, Write};
|
||||
|
||||
let _ = stream.set_read_timeout(Some(Duration::from_secs(5)));
|
||||
let _ = stream.set_write_timeout(Some(Duration::from_secs(5)));
|
||||
let Ok(connection) = rustls::ServerConnection::new(server_config) else {
|
||||
return;
|
||||
fn read_sync_http_message<R: std::io::Read>(stream: &mut R) -> Result<(String, Vec<u8>), BoxError> {
|
||||
let mut buffer = Vec::new();
|
||||
let mut chunk = [0_u8; 4096];
|
||||
let header_end = loop {
|
||||
let read = stream.read(&mut chunk)?;
|
||||
if read == 0 {
|
||||
return Err("connection closed before request headers were complete".into());
|
||||
}
|
||||
buffer.extend_from_slice(&chunk[..read]);
|
||||
if let Some(pos) = buffer.windows(4).position(|window| window == b"\r\n\r\n") {
|
||||
break pos;
|
||||
}
|
||||
};
|
||||
let mut tls_stream = rustls::StreamOwned::new(connection, stream);
|
||||
let mut buf = [0u8; 1024];
|
||||
if tls_stream.read(&mut buf).is_err() {
|
||||
return;
|
||||
|
||||
let header_text = std::str::from_utf8(&buffer[..header_end])?;
|
||||
let mut lines = header_text.split("\r\n");
|
||||
let method = lines
|
||||
.next()
|
||||
.and_then(|line| line.split_whitespace().next())
|
||||
.ok_or("missing request method")?
|
||||
.to_string();
|
||||
let mut content_length = 0usize;
|
||||
for line in lines {
|
||||
if let Some((name, value)) = line.split_once(':')
|
||||
&& name.trim().eq_ignore_ascii_case("content-length")
|
||||
{
|
||||
content_length = value.trim().parse()?;
|
||||
}
|
||||
}
|
||||
let response = "HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
|
||||
let _ = tls_stream.write_all(response.as_bytes());
|
||||
let _ = tls_stream.flush();
|
||||
|
||||
let body_offset = header_end + 4;
|
||||
while buffer.len().saturating_sub(body_offset) < content_length {
|
||||
let read = stream.read(&mut chunk)?;
|
||||
if read == 0 {
|
||||
return Err("connection closed before request body was complete".into());
|
||||
}
|
||||
buffer.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
Ok((method, buffer[body_offset..body_offset + content_length].to_vec()))
|
||||
}
|
||||
|
||||
fn stop_https_event_collector(endpoint: &str, running: Arc<AtomicBool>, handle: thread::JoinHandle<()>) -> TestResult {
|
||||
running.store(false, Ordering::Relaxed);
|
||||
if let Ok(parsed) = endpoint.parse::<reqwest::Url>()
|
||||
&& let Some(port) = parsed.port()
|
||||
{
|
||||
let _ = std::net::TcpStream::connect(("127.0.0.1", port));
|
||||
fn handle_https_request(
|
||||
stream: std::net::TcpStream,
|
||||
server_config: Arc<rustls::ServerConfig>,
|
||||
tx: mpsc::UnboundedSender<Value>,
|
||||
) -> Result<(), BoxError> {
|
||||
use std::io::Write;
|
||||
|
||||
stream.set_nonblocking(false)?;
|
||||
stream.set_read_timeout(Some(Duration::from_secs(5)))?;
|
||||
stream.set_write_timeout(Some(Duration::from_secs(5)))?;
|
||||
let connection = rustls::ServerConnection::new(server_config)?;
|
||||
let mut tls_stream = rustls::StreamOwned::new(connection, stream);
|
||||
let (method, body) = read_sync_http_message(&mut tls_stream)?;
|
||||
let response = "HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
|
||||
tls_stream.write_all(response.as_bytes())?;
|
||||
tls_stream.flush()?;
|
||||
tls_stream.conn.send_close_notify();
|
||||
while tls_stream.conn.wants_write() {
|
||||
tls_stream.conn.write_tls(&mut tls_stream.sock)?;
|
||||
}
|
||||
let _ = tls_stream.sock.shutdown(std::net::Shutdown::Write);
|
||||
if method == "POST"
|
||||
&& !body.is_empty()
|
||||
&& let Ok(event) = serde_json::from_slice(&body)
|
||||
{
|
||||
let _ = tx.send(event);
|
||||
}
|
||||
handle.join().map_err(|_| "https event collector thread panicked")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -415,17 +512,20 @@ async fn wait_for_target_registered(env: &RustFSTestEnvironment, target_name: &s
|
||||
Err(format!("target {target_name} was not registered in admin ARNs").into())
|
||||
}
|
||||
|
||||
async fn wait_for_target_listed(env: &RustFSTestEnvironment, target_name: &str) -> TestResult {
|
||||
async fn wait_for_target_online(env: &RustFSTestEnvironment, target_name: &str) -> TestResult {
|
||||
let url = format!("{}/rustfs/admin/v3/target/list", env.url);
|
||||
for _ in 0..40 {
|
||||
let response = signed_admin_request(env, http::Method::GET, &url, None).await?;
|
||||
if response.status() == StatusCode::OK {
|
||||
let body: Value = serde_json::from_slice(&response.bytes().await?)?;
|
||||
if body["notify_enabled"].as_bool() != Some(true) {
|
||||
return Err(format!("admin target list did not report notify_enabled=true: {body}").into());
|
||||
}
|
||||
let listed = body["notification_endpoints"].as_array().is_some_and(|endpoints| {
|
||||
endpoints.iter().any(|endpoint| {
|
||||
endpoint["account_id"].as_str() == Some(target_name)
|
||||
&& endpoint["service"].as_str() == Some("webhook")
|
||||
&& endpoint["status"].as_str().is_some()
|
||||
&& endpoint["status"].as_str() == Some("online")
|
||||
})
|
||||
});
|
||||
if listed {
|
||||
@@ -434,7 +534,7 @@ async fn wait_for_target_listed(env: &RustFSTestEnvironment, target_name: &str)
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
Err(format!("target {target_name} was not listed in admin targets").into())
|
||||
Err(format!("target {target_name} did not become online in admin targets").into())
|
||||
}
|
||||
|
||||
/// Binds a bucket to a webhook target for ObjectCreated:*/ObjectRemoved:* events,
|
||||
@@ -484,34 +584,58 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Regression for rustfs#5052: with the notify module enabled through
|
||||
/// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must be accepted
|
||||
/// and remain visible in the admin target list.
|
||||
/// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must become
|
||||
/// online and receive a real S3 event POST.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_https_webhook_target_lists_with_notify_env_enabled() -> TestResult {
|
||||
async fn test_https_webhook_target_delivers_event_with_notify_env_enabled() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_NOTIFY_ENABLE", "true")])
|
||||
.await?;
|
||||
|
||||
let ca_path = Path::new(&env.temp_dir).join("https-webhook-ca.pem");
|
||||
let (endpoint, running, handle) = spawn_https_event_collector(&ca_path)?;
|
||||
let mut collector = spawn_https_event_collector(&ca_path)?;
|
||||
let allowed_origin = endpoint_origin(collector.endpoint())?;
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_NOTIFY_ENABLE", "true"),
|
||||
(ENV_OUTBOUND_ALLOW_ORIGINS, allowed_origin.as_str()),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
let target = "peri1https";
|
||||
let bucket = "peri1-https-events";
|
||||
let key = "uploads/https.dat";
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
configure_webhook_target_with_key_values(
|
||||
&env,
|
||||
target,
|
||||
vec![
|
||||
("endpoint", endpoint.clone()),
|
||||
("endpoint", collector.endpoint().to_string()),
|
||||
("client_ca", ca_path.to_string_lossy().into_owned()),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
wait_for_target_listed(&env, target).await?;
|
||||
wait_for_target_online(&env, target).await?;
|
||||
wait_for_target_registered(&env, target).await?;
|
||||
put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"https webhook event body"))
|
||||
.send()
|
||||
.await?;
|
||||
let event = wait_for_event(collector.events_mut(), key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
|
||||
assert_eq!(event["EventName"].as_str(), Some("s3:ObjectCreated:Put"));
|
||||
assert_eq!(event["Records"][0]["s3"]["bucket"]["name"].as_str(), Some(bucket));
|
||||
assert_eq!(event_key(&event).as_deref(), Some(key));
|
||||
|
||||
env.stop_server();
|
||||
stop_https_event_collector(&endpoint, running, handle)?;
|
||||
collector.shutdown()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -523,9 +647,11 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (endpoint, mut rx, handle) = spawn_event_collector().await?;
|
||||
let allowed_origin = endpoint_origin(&endpoint)?;
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[(ENV_OUTBOUND_ALLOW_ORIGINS, allowed_origin.as_str())])
|
||||
.await?;
|
||||
enable_notify_module(&env).await?;
|
||||
|
||||
let bucket = "peri1-events";
|
||||
@@ -677,15 +803,6 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
enable_notify_module(&env).await?;
|
||||
|
||||
let bucket = "peri1-redeliver";
|
||||
let target = "peri1redeliver";
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
// Configure the target while its endpoint is reachable so activation and
|
||||
// ARN registration complete deterministically. Registering against a dead
|
||||
// endpoint stalls behind the reachability probe's timeout and flakes the
|
||||
@@ -694,10 +811,21 @@ async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult {
|
||||
let listener = TcpListener::bind("0.0.0.0:0").await?;
|
||||
let port = listener.local_addr()?.port();
|
||||
let endpoint_ip = local_ip()?;
|
||||
let endpoint = format!("http://{endpoint_ip}.nip.io:{port}/events");
|
||||
let endpoint = format!("http://{}/events", std::net::SocketAddr::new(endpoint_ip, port));
|
||||
let allowed_origin = endpoint_origin(&endpoint)?;
|
||||
let (setup_tx, _setup_rx) = mpsc::unbounded_channel();
|
||||
let setup_handle = serve_event_collector(listener, setup_tx);
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[(ENV_OUTBOUND_ALLOW_ORIGINS, allowed_origin.as_str())])
|
||||
.await?;
|
||||
enable_notify_module(&env).await?;
|
||||
|
||||
let bucket = "peri1-redeliver";
|
||||
let target = "peri1redeliver";
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
configure_webhook_target(&env, target, &endpoint).await?;
|
||||
wait_for_target_registered(&env, target).await?;
|
||||
put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
|
||||
|
||||
@@ -18,6 +18,7 @@ use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::{pre_sign_v4, sign_v4};
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use s3s::Body;
|
||||
use serial_test::serial;
|
||||
use std::collections::HashMap;
|
||||
@@ -49,7 +50,7 @@ fn find_header_terminator(buf: &[u8]) -> Option<usize> {
|
||||
|
||||
async fn read_http_request(
|
||||
stream: &mut tokio::net::TcpStream,
|
||||
) -> Result<(HashMap<String, String>, Vec<u8>), Box<dyn Error + Send + Sync>> {
|
||||
) -> Result<(String, HashMap<String, String>, Vec<u8>), Box<dyn Error + Send + Sync>> {
|
||||
let mut buffer = Vec::new();
|
||||
let mut chunk = [0_u8; 4096];
|
||||
|
||||
@@ -67,7 +68,7 @@ async fn read_http_request(
|
||||
let header_bytes = &buffer[..header_end];
|
||||
let header_text = std::str::from_utf8(header_bytes)?;
|
||||
let mut lines = header_text.split("\r\n");
|
||||
let _request_line = lines.next().ok_or("missing request line")?;
|
||||
let request_line = lines.next().ok_or("missing request line")?.to_string();
|
||||
let mut headers = HashMap::new();
|
||||
for line in lines {
|
||||
if line.is_empty() {
|
||||
@@ -79,8 +80,9 @@ async fn read_http_request(
|
||||
|
||||
let content_length = headers
|
||||
.get("content-length")
|
||||
.ok_or("missing content-length header")?
|
||||
.parse::<usize>()?;
|
||||
.map(|value| value.parse::<usize>())
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
let body_offset = header_end + 4;
|
||||
while buffer.len().saturating_sub(body_offset) < content_length {
|
||||
let read = stream.read(&mut chunk).await?;
|
||||
@@ -90,7 +92,7 @@ async fn read_http_request(
|
||||
buffer.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
|
||||
Ok((headers, buffer[body_offset..body_offset + content_length].to_vec()))
|
||||
Ok((request_line, headers, buffer[body_offset..body_offset + content_length].to_vec()))
|
||||
}
|
||||
|
||||
async fn spawn_object_lambda_webhook_server() -> Result<
|
||||
@@ -130,9 +132,17 @@ async fn spawn_object_lambda_webhook_server_with_response(
|
||||
let handle = tokio::spawn(async move {
|
||||
loop {
|
||||
let (mut stream, _) = listener.accept().await?;
|
||||
let Ok(Ok((headers, body))) = timeout(Duration::from_secs(2), read_http_request(&mut stream)).await else {
|
||||
let Ok(Ok((request_line, headers, body))) = timeout(Duration::from_secs(2), read_http_request(&mut stream)).await
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if request_line == "HEAD / HTTP/1.1" {
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
|
||||
.await?;
|
||||
stream.shutdown().await?;
|
||||
continue;
|
||||
}
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body)?;
|
||||
|
||||
let output_route = payload["getObjectContext"]["outputRoute"]
|
||||
@@ -412,9 +422,33 @@ async fn wait_for_target_absence(
|
||||
Err(format!("target {target_name} remained visible in admin APIs; targets={last_targets}, arns={last_arns:?}").into())
|
||||
}
|
||||
|
||||
async fn restart_rustfs_server(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
fn endpoint_origin(endpoint: &str) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
Ok(reqwest::Url::parse(endpoint)?.origin().ascii_serialization())
|
||||
}
|
||||
|
||||
async fn start_rustfs_server_for_endpoint(
|
||||
env: &mut RustFSTestEnvironment,
|
||||
endpoint: &str,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let origin = endpoint_origin(endpoint)?;
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
(ENV_OUTBOUND_ALLOW_ORIGINS, origin.as_str()),
|
||||
("RUSTFS_NOTIFY_ENABLE", "true"),
|
||||
],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn restart_rustfs_server(env: &mut RustFSTestEnvironment, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let origin = endpoint_origin(endpoint)?;
|
||||
env.stop_server();
|
||||
env.start_rustfs_server_without_cleanup(vec![]).await
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[
|
||||
(ENV_OUTBOUND_ALLOW_ORIGINS, origin.as_str()),
|
||||
("RUSTFS_NOTIFY_ENABLE", "true"),
|
||||
])
|
||||
.await
|
||||
}
|
||||
|
||||
async fn spawn_http_origin_probe_server() -> Result<
|
||||
@@ -521,7 +555,7 @@ async fn test_notification_target_persists_across_restart_and_delete() -> Result
|
||||
let (webhook_url, _request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
|
||||
|
||||
let target_name = "restart-target";
|
||||
configure_webhook_target(&env, target_name, &webhook_url, "secret-token").await?;
|
||||
@@ -535,7 +569,7 @@ async fn test_notification_target_persists_across_restart_and_delete() -> Result
|
||||
"target ARN missing after initial configure: {visible_arns:?}"
|
||||
);
|
||||
|
||||
restart_rustfs_server(&mut env).await?;
|
||||
restart_rustfs_server(&mut env, &webhook_url).await?;
|
||||
|
||||
let (targets_after_restart, arns_after_restart) = wait_for_target_visibility(&env, target_name).await?;
|
||||
assert!(notification_target_is_listed(&targets_after_restart, target_name));
|
||||
@@ -556,7 +590,7 @@ async fn test_notification_target_persists_across_restart_and_delete() -> Result
|
||||
"target ARN still visible after delete: {arns_after_delete:?}"
|
||||
);
|
||||
|
||||
restart_rustfs_server(&mut env).await?;
|
||||
restart_rustfs_server(&mut env, &webhook_url).await?;
|
||||
|
||||
let (targets_after_delete_restart, arns_after_delete_restart) = wait_for_target_absence(&env, target_name).await?;
|
||||
assert!(!notification_target_is_listed(&targets_after_delete_restart, target_name));
|
||||
@@ -581,8 +615,7 @@ async fn test_notification_target_with_path_is_online_via_transport_probe() -> R
|
||||
let (webhook_url, mut probe_rx, probe_handle) = spawn_http_origin_probe_server().await?;
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_NOTIFY_ENABLE", "true")])
|
||||
.await?;
|
||||
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
|
||||
|
||||
let target_name = "path-probe";
|
||||
configure_webhook_target(&env, target_name, &webhook_url, "secret-token").await?;
|
||||
@@ -615,7 +648,7 @@ async fn test_get_object_lambda_accepts_presigned_requests() -> Result<(), Box<d
|
||||
let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
|
||||
|
||||
let bucket = "object-lambda-e2e-presigned";
|
||||
let key = "input.txt";
|
||||
@@ -656,7 +689,7 @@ async fn test_get_object_lambda_accepts_named_webhook_target_arn() -> Result<(),
|
||||
let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
|
||||
|
||||
let bucket = "object-lambda-e2e-named-target";
|
||||
let key = "input.txt";
|
||||
@@ -696,7 +729,7 @@ async fn test_get_object_lambda_invokes_runtime_webhook_target() -> Result<(), B
|
||||
let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
|
||||
|
||||
let bucket = "object-lambda-e2e";
|
||||
let key = "input.txt";
|
||||
@@ -777,7 +810,7 @@ async fn test_get_object_lambda_passthroughs_non_success_webhook_response() -> R
|
||||
.await?;
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
|
||||
|
||||
let bucket = "object-lambda-e2e-failure";
|
||||
let key = "input.txt";
|
||||
@@ -832,7 +865,7 @@ async fn test_get_object_lambda_rejects_success_response_without_auth_headers()
|
||||
.await?;
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
|
||||
|
||||
let bucket = "object-lambda-e2e-missing-auth";
|
||||
let key = "input.txt";
|
||||
@@ -879,7 +912,7 @@ async fn test_get_object_lambda_rejects_success_response_with_mismatched_auth_he
|
||||
.await?;
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
|
||||
|
||||
let bucket = "object-lambda-e2e-mismatched-auth";
|
||||
let key = "input.txt";
|
||||
|
||||
@@ -4048,17 +4048,23 @@ async fn test_site_replication_allows_private_ca_https_with_ca_cert_pem_real_dua
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
async fn test_site_replication_resync_lifecycle_survives_real_server_restart() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let resync_process_env = [
|
||||
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true"),
|
||||
// Verbose server logging can block startup when this focused test is run
|
||||
// through a captured test process rather than nextest.
|
||||
("RUST_LOG", "error"),
|
||||
];
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
source_env
|
||||
.start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
|
||||
.await?;
|
||||
source_env.capture_log_path = Some(format!("{}/server.log", source_env.temp_dir));
|
||||
source_env.start_rustfs_server_with_env(vec![], &resync_process_env).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.capture_log_path = Some(format!("{}/server.log", target_env.temp_dir));
|
||||
target_env
|
||||
.start_rustfs_server_without_cleanup_with_env(LOOPBACK_REPLICATION_TARGET_ENV)
|
||||
.start_rustfs_server_without_cleanup_with_env(&resync_process_env)
|
||||
.await?;
|
||||
|
||||
let source_bucket = "site-repl-resync-src";
|
||||
@@ -4106,12 +4112,12 @@ async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> R
|
||||
wait_for_bucket_on_target(&source_client, source_bucket).await?;
|
||||
let target_arn = wait_for_remote_target_arn(&source_env, source_bucket).await?;
|
||||
|
||||
for idx in 0..32 {
|
||||
for idx in 0..96 {
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(format!("resync-object-{idx:02}"))
|
||||
.body(ByteStream::from(vec![b'x'; 256 * 1024]))
|
||||
.body(ByteStream::from(vec![b'x'; 512 * 1024]))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
@@ -4119,18 +4125,31 @@ async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> R
|
||||
let started = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
|
||||
assert_eq!(started.status, "success", "unexpected start result: {:?}", started);
|
||||
assert!(
|
||||
started
|
||||
.buckets
|
||||
.iter()
|
||||
.any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")),
|
||||
started.buckets.iter().any(|bucket| {
|
||||
bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "running" | "completed" | "success")
|
||||
}),
|
||||
"source bucket start status missing: {:?}",
|
||||
started
|
||||
);
|
||||
assert!(!started.resync_id.is_empty(), "start response omitted the resync id: {:?}", started);
|
||||
let started_reset_id = started.resync_id.clone();
|
||||
|
||||
assert!(
|
||||
matches!(started.state.as_str(), "pending" | "running"),
|
||||
"the fixture must keep the first generation active long enough to test duplicate start: {:?}",
|
||||
started
|
||||
);
|
||||
let duplicate_err = site_replication_resync_op(&source_env, "start", &remote_peer)
|
||||
.await
|
||||
.expect_err("duplicate start must be rejected while a generation is active");
|
||||
assert!(
|
||||
duplicate_err.to_string().contains("already active"),
|
||||
"unexpected duplicate start error: {duplicate_err}"
|
||||
);
|
||||
|
||||
let canceled = site_replication_resync_op(&source_env, "cancel", &remote_peer).await?;
|
||||
assert_eq!(canceled.status, "success", "unexpected cancel result: {:?}", canceled);
|
||||
assert_eq!(canceled.state, "canceled");
|
||||
assert!(
|
||||
canceled
|
||||
.buckets
|
||||
@@ -4139,34 +4158,56 @@ async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> R
|
||||
"source bucket cancel status missing: {:?}",
|
||||
canceled
|
||||
);
|
||||
let canceled_again = site_replication_resync_op(&source_env, "cancel", &remote_peer).await?;
|
||||
assert_eq!(canceled_again.resync_id, canceled.resync_id, "repeated cancel must be idempotent");
|
||||
assert_eq!(canceled_again.state, "canceled");
|
||||
|
||||
let canceled_target =
|
||||
wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| target.status == "Canceled").await?;
|
||||
assert_eq!(canceled_target.status, "Canceled");
|
||||
assert_eq!(canceled_target.reset_id, started_reset_id);
|
||||
|
||||
let restarted = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
|
||||
assert_eq!(restarted.status, "success", "unexpected restart result: {:?}", restarted);
|
||||
assert_ne!(restarted.resync_id, started_reset_id);
|
||||
assert!(
|
||||
matches!(restarted.state.as_str(), "pending" | "running"),
|
||||
"the second generation must be active before the process restart: {:?}",
|
||||
restarted
|
||||
.buckets
|
||||
.iter()
|
||||
.any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")),
|
||||
"source bucket restart status missing: {:?}",
|
||||
restarted
|
||||
);
|
||||
let restarted_reset_id = restarted.resync_id.clone();
|
||||
|
||||
source_env.restart_server_preserving_data(vec![], &resync_process_env).await?;
|
||||
wait_for_site_replication_enabled(&source_env, 2).await?;
|
||||
|
||||
let after_restart = site_replication_resync_op(&source_env, "status", &remote_peer).await?;
|
||||
assert_eq!(
|
||||
after_restart.resync_id, restarted_reset_id,
|
||||
"server restart changed the durable resync id"
|
||||
);
|
||||
assert_eq!(after_restart.generation, restarted.generation);
|
||||
assert_eq!(after_restart.created_at, restarted.created_at);
|
||||
assert!(
|
||||
matches!(after_restart.state.as_str(), "pending" | "running" | "completed" | "failed"),
|
||||
"unexpected recovered lifecycle state: {:?}",
|
||||
after_restart
|
||||
);
|
||||
assert!(
|
||||
after_restart.buckets.iter().any(|bucket| bucket.bucket == source_bucket),
|
||||
"durable status lost the source bucket after restart: {:?}",
|
||||
after_restart
|
||||
);
|
||||
let restart_snapshot = get_replication_reset_status(&source_env, source_bucket, &target_arn).await?;
|
||||
let restarted_target = wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| {
|
||||
!target.reset_id.is_empty() && target.reset_id != started_reset_id
|
||||
target.reset_id == restarted_reset_id
|
||||
})
|
||||
.await
|
||||
.map_err(|err| {
|
||||
format!(
|
||||
"restart ids: start={} restart={} snapshot={:?}; {err}",
|
||||
started_reset_id, restarted.resync_id, restart_snapshot.targets
|
||||
started_reset_id, restarted_reset_id, restart_snapshot.targets
|
||||
)
|
||||
})?;
|
||||
assert_ne!(restarted_target.reset_id, started_reset_id);
|
||||
assert_eq!(restarted_target.reset_id, restarted_reset_id);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -241,8 +241,10 @@ pub mod config {
|
||||
pub use crate::config::com::{
|
||||
COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
|
||||
ServerConfigCorruptError, delete_config, is_server_config_corrupt_error, lookup_configs, read_config,
|
||||
read_config_no_lock, read_config_with_metadata, read_config_without_migrate, save_config, save_config_with_opts,
|
||||
save_server_config, try_migrate_server_config,
|
||||
read_config_no_lock, read_config_with_metadata, read_config_without_migrate, read_config_without_migrate_no_lock,
|
||||
read_existing_server_config_no_lock, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
|
||||
save_server_config_no_lock, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock,
|
||||
with_server_config_read_lock, with_server_config_write_lock,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -383,9 +385,9 @@ pub mod rio {
|
||||
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
|
||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor,
|
||||
gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature,
|
||||
|
||||
@@ -2809,6 +2809,8 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
Err(err) => return Err(std::io::Error::other(err)),
|
||||
};
|
||||
|
||||
tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?;
|
||||
|
||||
let ret = new_getobjectreader(rs, oi, opts, h);
|
||||
if let Err(err) = ret {
|
||||
return Err(error_resp_to_object_err(err, vec![bucket, object]));
|
||||
@@ -4046,6 +4048,56 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_rejects_nonempty_remote_version_before_backend_io() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
let body = Bytes::from_static(b"transitioned object body");
|
||||
let remote_version = backend
|
||||
.put(
|
||||
&remote_object,
|
||||
ReaderImpl::Body(body.clone()),
|
||||
i64::try_from(body.len()).expect("body length should fit"),
|
||||
)
|
||||
.await
|
||||
.expect("mock remote object should be stored");
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: i64::try_from(body.len()).expect("body length should fit"),
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object,
|
||||
version_id: remote_version,
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = match get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("a provider that rejects versioned GET must fail before remote IO"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert!(err.to_string().contains("requires an unversioned remote object"));
|
||||
assert_eq!(backend.get_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_remote_delete_requires_persisted_destination_identity() {
|
||||
|
||||
@@ -152,13 +152,13 @@ pub(crate) fn tier_delete_journal_object_name(je: &Jentry) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
fn decode_tier_delete_journal_entry(data: &[u8]) -> Result<Jentry> {
|
||||
pub(crate) fn decode_tier_delete_journal_entry(data: &[u8]) -> Result<Jentry> {
|
||||
let persisted: PersistedTierDeleteJournalEntry =
|
||||
serde_json::from_slice(data).map_err(|err| Error::other(format!("decode tier delete journal failed: {err}")))?;
|
||||
persisted.into_jentry()
|
||||
}
|
||||
|
||||
fn encode_tier_delete_journal_entry(je: &Jentry) -> Result<Vec<u8>> {
|
||||
pub(crate) fn encode_tier_delete_journal_entry(je: &Jentry) -> Result<Vec<u8>> {
|
||||
serde_json::to_vec(&PersistedTierDeleteJournalEntry::from_jentry(je))
|
||||
.map_err(|err| Error::other(format!("encode tier delete journal failed: {err}")))
|
||||
}
|
||||
|
||||
@@ -339,6 +339,8 @@ async fn delete_object_from_remote_tier_raw_with_lease(
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
) -> Result<(), std::io::Error> {
|
||||
lease.validate_remote_version_id(rv_id)?;
|
||||
|
||||
if remote_delete_breaker_is_open(Instant::now()).await {
|
||||
metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1);
|
||||
return Err(std::io::Error::other(ERR_REMOTE_DELETE_BREAKER_OPEN));
|
||||
@@ -622,6 +624,46 @@ mod test {
|
||||
assert_eq!(backend.remove_count().await, 1);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn journal_delete_rejects_nonempty_remote_version_before_backend_io() {
|
||||
let manager = crate::services::tier::tier::TierConfigMgr::new();
|
||||
let backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await;
|
||||
let lease = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM")
|
||||
.await
|
||||
.expect("test tier lease should be available");
|
||||
let identity = lease.backend_identity();
|
||||
drop(lease);
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
|
||||
let err = delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
"remote/object",
|
||||
"remote-version",
|
||||
"WARM",
|
||||
identity,
|
||||
&manager,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect_err("a provider that rejects a versioned delete must fail before remote IO");
|
||||
|
||||
assert!(err.to_string().contains("requires an unversioned remote object"));
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
"remote/object",
|
||||
"",
|
||||
"WARM",
|
||||
identity,
|
||||
&manager,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("unversioned remote delete should continue without a version ID");
|
||||
|
||||
assert_eq!(backend.remove_versions().await, vec![("remote/object".to_string(), String::new())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breaker_opens_at_threshold_and_recovers_after_window() {
|
||||
let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30));
|
||||
|
||||
@@ -26,6 +26,7 @@ use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idemp
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::services::tier::{tier::TierConfigMgr, warm_backend::TransitionCandidateProbe};
|
||||
use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectOperations as _};
|
||||
use crate::store::ECStore;
|
||||
|
||||
@@ -612,6 +613,11 @@ pub enum TransitionTransactionRecoveryOutcome {
|
||||
Retained,
|
||||
}
|
||||
|
||||
pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
|
||||
let transaction_id = transition_transaction_id_from_record_object_name(object)?;
|
||||
TransitionTransaction::decode(transaction_id, data)
|
||||
}
|
||||
|
||||
fn transition_transaction_id_from_record_object_name(object: &str) -> Result<Uuid> {
|
||||
let prefix = format!("{TRANSITION_TRANSACTION_RECORD_PREFIX}/");
|
||||
let suffix = object
|
||||
@@ -658,20 +664,84 @@ pub async fn process_transition_transaction_record(
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
},
|
||||
TransitionTransactionState::LocalCommitStarted if local_commit_matches_transaction(api.clone(), transaction).await? => {
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
TransitionTransactionState::LocalCommitStarted => {
|
||||
match local_commit_matches_transaction(api.clone(), transaction).await {
|
||||
Ok(true) => {
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
TransitionTransactionState::UploadStarted
|
||||
| TransitionTransactionState::UploadOutcomeUnknown
|
||||
| TransitionTransactionState::LocalCommitStarted => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await,
|
||||
TransitionTransactionState::UploadStarted => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
}
|
||||
}
|
||||
|
||||
async fn recover_unknown_upload_outcome(
|
||||
api: Arc<ECStore>,
|
||||
transaction: &TransitionTransaction,
|
||||
) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
|
||||
&api.tier_config_mgr(),
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
|
||||
match lease
|
||||
.probe_transition_candidate(&transaction.remote_object)
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
TransitionCandidateProbe::Missing => {
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
TransitionCandidateProbe::UnversionedPresent => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
|
||||
}
|
||||
TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_recovered_unknown_upload_candidate(
|
||||
api: Arc<ECStore>,
|
||||
transaction: &TransitionTransaction,
|
||||
remote_version: TransitionRemoteVersion,
|
||||
) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
|
||||
let mut cleanup = transaction.clone();
|
||||
cleanup
|
||||
.mark_cleanup_pending(
|
||||
transaction.fence(),
|
||||
TransitionCleanupProof {
|
||||
transaction_id: transaction.transaction_id,
|
||||
write_id: transaction.write_id,
|
||||
remote_object: transaction.remote_object.clone(),
|
||||
remote_version,
|
||||
backend_fingerprint: transaction.backend_fingerprint,
|
||||
decision: TransitionCleanupDecision::RemoteVersionRecoveredAfterCancellation,
|
||||
},
|
||||
)
|
||||
.map_err(transition_transaction_store_error)?;
|
||||
save_transition_transaction_record(api.clone(), &cleanup).await?;
|
||||
delete_transition_remote_candidate(api.clone(), &cleanup).await?;
|
||||
delete_transition_transaction_record(api, cleanup.transaction_id).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
|
||||
fn transition_source_is_missing(err: &Error) -> bool {
|
||||
matches!(
|
||||
err,
|
||||
|
||||
@@ -124,6 +124,24 @@ fn classify_listing_quorum_failure(errors: &[DiskError]) -> DiskError {
|
||||
DiskError::ErasureReadQuorum
|
||||
}
|
||||
|
||||
/// Returns true when a metacache listing missed quorum purely because the
|
||||
/// volume or path is absent on a quorum of drives, i.e. every recorded failure
|
||||
/// is [`DiskError::VolumeNotFound`] or [`DiskError::FileNotFound`].
|
||||
///
|
||||
/// This is a benign, expected outcome: the caller is handed
|
||||
/// `VolumeNotFound`/`FileNotFound` and decides how to react. The most common
|
||||
/// trigger is a startup race where the system bucket has not yet been created
|
||||
/// on every drive when an early reader (e.g. the IAM config loader) lists it.
|
||||
/// It must be distinguished from a listing that failed for a real reason (I/O,
|
||||
/// timeout, corruption) so the former is not surfaced at `error`. See
|
||||
/// rustfs/rustfs#5076.
|
||||
fn is_benign_not_found_listing_failure(errors: &[DiskError]) -> bool {
|
||||
!errors.is_empty()
|
||||
&& errors
|
||||
.iter()
|
||||
.all(|err| matches!(err, DiskError::VolumeNotFound | DiskError::FileNotFound))
|
||||
}
|
||||
|
||||
struct PublishedBytesWriter<W> {
|
||||
inner: W,
|
||||
published: bool,
|
||||
@@ -840,17 +858,38 @@ async fn list_path_raw_inner(
|
||||
_ => {}
|
||||
});
|
||||
|
||||
error!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts.bucket,
|
||||
path = %opts.path,
|
||||
state = "quorum_failed",
|
||||
error = %combined_err.join(", "),
|
||||
"Metacache listing quorum failed"
|
||||
);
|
||||
let failures = errs.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
// A listing that misses quorum purely because the volume/path is
|
||||
// absent on a quorum of drives is benign and expected — the caller
|
||||
// receives VolumeNotFound/FileNotFound and decides how to react.
|
||||
// The common trigger is a startup race where the system bucket is
|
||||
// not yet created on every drive when an early reader (e.g. the IAM
|
||||
// config loader) lists it, so surfacing it at `error` is misleading
|
||||
// noise (rustfs/rustfs#5076). Keep `error` for listings that failed
|
||||
// for a real reason (I/O, timeout, corruption).
|
||||
if is_benign_not_found_listing_failure(&failures) {
|
||||
debug!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts.bucket,
|
||||
path = %opts.path,
|
||||
state = "quorum_not_found",
|
||||
error = %combined_err.join(", "),
|
||||
"Metacache listing quorum not reached (volume/path absent)"
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts.bucket,
|
||||
path = %opts.path,
|
||||
state = "quorum_failed",
|
||||
error = %combined_err.join(", "),
|
||||
"Metacache listing quorum failed"
|
||||
);
|
||||
}
|
||||
return Err(classify_listing_quorum_failure(&failures));
|
||||
}
|
||||
|
||||
@@ -1037,6 +1076,21 @@ mod tests {
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn benign_not_found_listing_failure_detection() {
|
||||
// Pure not-found quorum misses are benign (the volume/path simply does
|
||||
// not exist on a quorum of drives) and must not be logged at ERROR.
|
||||
assert!(is_benign_not_found_listing_failure(&[DiskError::VolumeNotFound]));
|
||||
assert!(is_benign_not_found_listing_failure(
|
||||
&[DiskError::VolumeNotFound, DiskError::FileNotFound,]
|
||||
));
|
||||
// No recorded failure is not a not-found case.
|
||||
assert!(!is_benign_not_found_listing_failure(&[]));
|
||||
// Any real error must keep the failure at ERROR severity.
|
||||
assert!(!is_benign_not_found_listing_failure(&[DiskError::VolumeNotFound, DiskError::Timeout,]));
|
||||
assert!(!is_benign_not_found_listing_failure(&[DiskError::DiskNotFound]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_empty_disks_returns_read_quorum() {
|
||||
let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::client::{
|
||||
ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart,
|
||||
},
|
||||
credentials,
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body},
|
||||
};
|
||||
use crate::storage_api_contracts::bucket::BucketInfo;
|
||||
use http::{HeaderMap, StatusCode};
|
||||
@@ -88,7 +88,7 @@ impl TransitionClient {
|
||||
url_values.insert("max-keys".to_string(), max_keys.to_string());
|
||||
}
|
||||
|
||||
let mut resp = self
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::GET,
|
||||
&mut RequestMetadata {
|
||||
@@ -164,7 +164,7 @@ impl TransitionClient {
|
||||
Ok(list_bucket_result)
|
||||
}
|
||||
|
||||
pub fn list_object_versions_query(
|
||||
pub async fn list_object_versions_query(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
opts: &ListObjectsOptions,
|
||||
@@ -172,93 +172,88 @@ impl TransitionClient {
|
||||
version_id_marker: &str,
|
||||
delimiter: &str,
|
||||
) -> Result<ListVersionsResult, std::io::Error> {
|
||||
/*if err := s3utils.CheckValidBucketName(bucketName); err != nil {
|
||||
return ListVersionsResult{}, err
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("versions".to_string(), "".to_string());
|
||||
url_values.insert("prefix".to_string(), opts.prefix.clone());
|
||||
url_values.insert("delimiter".to_string(), delimiter.to_string());
|
||||
url_values.insert("encoding-type".to_string(), "url".to_string());
|
||||
|
||||
if !key_marker.is_empty() {
|
||||
url_values.insert("key-marker".to_string(), key_marker.to_string());
|
||||
}
|
||||
if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
|
||||
return ListVersionsResult{}, err
|
||||
}
|
||||
urlValues := make(url.Values)
|
||||
|
||||
urlValues.Set("versions", "")
|
||||
|
||||
urlValues.Set("prefix", opts.Prefix)
|
||||
|
||||
urlValues.Set("delimiter", delimiter)
|
||||
|
||||
if keyMarker != "" {
|
||||
urlValues.Set("key-marker", keyMarker)
|
||||
}
|
||||
|
||||
if opts.max_keys > 0 {
|
||||
urlValues.Set("max-keys", fmt.Sprintf("%d", opts.max_keys))
|
||||
url_values.insert("max-keys".to_string(), opts.max_keys.to_string());
|
||||
}
|
||||
if !version_id_marker.is_empty() {
|
||||
url_values.insert("version-id-marker".to_string(), version_id_marker.to_string());
|
||||
}
|
||||
if opts.with_metadata {
|
||||
url_values.insert("metadata".to_string(), "true".to_string());
|
||||
}
|
||||
|
||||
if versionIDMarker != "" {
|
||||
urlValues.Set("version-id-marker", versionIDMarker)
|
||||
let mut resp = self
|
||||
.execute_method(
|
||||
http::Method::GET,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: "".to_string(),
|
||||
query_values: url_values,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
custom_header: opts.headers.clone(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp_status = resp.status();
|
||||
let headers = resp.headers().clone();
|
||||
let body = collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE).await?;
|
||||
if resp_status != StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&headers,
|
||||
body,
|
||||
bucket_name,
|
||||
"",
|
||||
)));
|
||||
}
|
||||
|
||||
if opts.WithMetadata {
|
||||
urlValues.Set("metadata", "true")
|
||||
let mut versions = quick_xml::de::from_reader::<_, ListVersionsResult>(body.as_slice())
|
||||
.map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?;
|
||||
for version in &mut versions.versions {
|
||||
version.key = decode_s3_name(&version.key, &versions.encoding_type)?;
|
||||
}
|
||||
for marker in &mut versions.delete_markers {
|
||||
marker.key = decode_s3_name(&marker.key, &versions.encoding_type)?;
|
||||
}
|
||||
for prefix in &mut versions.common_prefixes {
|
||||
prefix.prefix = decode_s3_name(&prefix.prefix, &versions.encoding_type)?;
|
||||
}
|
||||
if !versions.next_key_marker.is_empty() {
|
||||
versions.next_key_marker = decode_s3_name(&versions.next_key_marker, &versions.encoding_type)?;
|
||||
}
|
||||
|
||||
urlValues.Set("encoding-type", "url")
|
||||
|
||||
let resp = self.executeMethod(http::Method::GET, &mut RequestMetadata{
|
||||
bucketName: bucketName,
|
||||
queryValues: urlValues,
|
||||
contentSHA256Hex: emptySHA256Hex,
|
||||
customHeader: opts.headers,
|
||||
}).await?;
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return ListVersionsResult{}, err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "")
|
||||
}
|
||||
}
|
||||
|
||||
listObjectVersionsOutput := ListVersionsResult{}
|
||||
err = xml_decoder(resp.Body, &listObjectVersionsOutput)
|
||||
if err != nil {
|
||||
return ListVersionsResult{}, err
|
||||
}
|
||||
|
||||
for i, obj := range listObjectVersionsOutput.Versions {
|
||||
listObjectVersionsOutput.Versions[i].Key, err = decode_s3_name(obj.Key, listObjectVersionsOutput.EncodingType)
|
||||
if err != nil {
|
||||
return listObjectVersionsOutput, err
|
||||
}
|
||||
}
|
||||
|
||||
for i, obj := range listObjectVersionsOutput.CommonPrefixes {
|
||||
listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decode_s3_name(obj.Prefix, listObjectVersionsOutput.EncodingType)
|
||||
if err != nil {
|
||||
return listObjectVersionsOutput, err
|
||||
}
|
||||
}
|
||||
|
||||
if listObjectVersionsOutput.NextKeyMarker != "" {
|
||||
listObjectVersionsOutput.NextKeyMarker, err = decode_s3_name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType)
|
||||
if err != nil {
|
||||
return listObjectVersionsOutput, err
|
||||
}
|
||||
}
|
||||
|
||||
Ok(listObjectVersionsOutput)*/
|
||||
Err(std::io::Error::new(
|
||||
ErrorKind::Unsupported,
|
||||
credentials::ErrorResponse {
|
||||
if versions.is_truncated && versions.next_key_marker.is_empty() {
|
||||
return Err(std::io::Error::other(credentials::ErrorResponse {
|
||||
sts_error: credentials::STSError {
|
||||
r#type: "".to_string(),
|
||||
code: "NotImplemented".to_string(),
|
||||
message: format!("list_object_versions_query is not implemented for bucket {bucket_name}"),
|
||||
message: "Truncated ListObjectVersions response should have next key marker set".to_string(),
|
||||
},
|
||||
request_id: "".to_string(),
|
||||
},
|
||||
))
|
||||
}));
|
||||
}
|
||||
|
||||
Ok(versions)
|
||||
}
|
||||
|
||||
pub fn list_objects_query(
|
||||
@@ -364,6 +359,7 @@ impl TransitionClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct ListObjectsOptions {
|
||||
reverse_versions: bool,
|
||||
@@ -431,3 +427,51 @@ fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Er
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn list_versions_xml_preserves_versions_and_delete_markers() {
|
||||
let xml = br#"
|
||||
<ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>tier-bucket</Name>
|
||||
<Prefix>archive/object</Prefix>
|
||||
<KeyMarker></KeyMarker>
|
||||
<VersionIdMarker></VersionIdMarker>
|
||||
<MaxKeys>2</MaxKeys>
|
||||
<IsTruncated>true</IsTruncated>
|
||||
<NextKeyMarker>archive/object</NextKeyMarker>
|
||||
<NextVersionIdMarker>version-a</NextVersionIdMarker>
|
||||
<Version>
|
||||
<Key>archive/object</Key>
|
||||
<VersionId>version-a</VersionId>
|
||||
<IsLatest>true</IsLatest>
|
||||
<LastModified>2026-07-22T00:00:00Z</LastModified>
|
||||
<ETag>"etag-a"</ETag>
|
||||
<Size>5</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Version>
|
||||
<DeleteMarker>
|
||||
<Key>archive/object</Key>
|
||||
<VersionId>marker-a</VersionId>
|
||||
<IsLatest>false</IsLatest>
|
||||
<LastModified>2026-07-22T00:00:01Z</LastModified>
|
||||
</DeleteMarker>
|
||||
</ListVersionsResult>
|
||||
"#;
|
||||
|
||||
let parsed =
|
||||
quick_xml::de::from_reader::<_, ListVersionsResult>(xml.as_slice()).expect("ListObjectVersions XML should parse");
|
||||
|
||||
assert!(parsed.is_truncated);
|
||||
assert_eq!(parsed.next_key_marker, "archive/object");
|
||||
assert_eq!(parsed.next_version_id_marker, "version-a");
|
||||
assert_eq!(parsed.versions.len(), 1);
|
||||
assert_eq!(parsed.versions[0].key, "archive/object");
|
||||
assert_eq!(parsed.versions[0].version_id, "version-a");
|
||||
assert_eq!(parsed.delete_markers.len(), 1);
|
||||
assert_eq!(parsed.delete_markers[0].version_id, "marker-a");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,34 +55,38 @@ pub struct ListBucketV2Result {
|
||||
pub start_after: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "PascalCase")]
|
||||
pub struct Version {
|
||||
etag: String,
|
||||
is_latest: bool,
|
||||
key: String,
|
||||
last_modified: OffsetDateTime,
|
||||
owner: Owner,
|
||||
size: i64,
|
||||
storage_class: String,
|
||||
version_id: String,
|
||||
user_metadata: HashMap<String, String>,
|
||||
user_tags: HashMap<String, String>,
|
||||
is_delete_marker: bool,
|
||||
#[serde(rename = "ETag")]
|
||||
pub etag: String,
|
||||
pub is_latest: bool,
|
||||
pub key: String,
|
||||
pub size: i64,
|
||||
pub storage_class: String,
|
||||
pub version_id: String,
|
||||
pub user_metadata: HashMap<String, String>,
|
||||
pub user_tags: HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "PascalCase")]
|
||||
pub struct ListVersionsResult {
|
||||
versions: Vec<Version>,
|
||||
common_prefixes: Vec<CommonPrefix>,
|
||||
name: String,
|
||||
prefix: String,
|
||||
delimiter: String,
|
||||
max_keys: i64,
|
||||
encoding_type: String,
|
||||
is_truncated: bool,
|
||||
key_marker: String,
|
||||
version_id_marker: String,
|
||||
next_key_marker: String,
|
||||
next_version_id_marker: String,
|
||||
#[serde(rename = "Version")]
|
||||
pub versions: Vec<Version>,
|
||||
#[serde(rename = "DeleteMarker")]
|
||||
pub delete_markers: Vec<Version>,
|
||||
pub common_prefixes: Vec<CommonPrefix>,
|
||||
pub name: String,
|
||||
pub prefix: String,
|
||||
pub delimiter: String,
|
||||
pub max_keys: i64,
|
||||
pub encoding_type: String,
|
||||
pub is_truncated: bool,
|
||||
pub key_marker: String,
|
||||
pub version_id_marker: String,
|
||||
pub next_key_marker: String,
|
||||
pub next_version_id_marker: String,
|
||||
}
|
||||
|
||||
pub struct ListBucketResult {
|
||||
|
||||
@@ -37,8 +37,8 @@ pub use http_auth::{
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
pub use peer_rest_client::{
|
||||
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerPeerActivity,
|
||||
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
|
||||
};
|
||||
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
|
||||
pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys};
|
||||
|
||||
@@ -219,31 +219,62 @@ impl PeerRestClient {
|
||||
recovery_running: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
pub async fn new_clients(eps: EndpointServerPools) -> (Vec<Option<Self>>, Vec<Option<Self>>) {
|
||||
if !runtime_sources::setup_is_dist_erasure().await {
|
||||
return (Vec::new(), Vec::new());
|
||||
}
|
||||
|
||||
let eps = eps.clone();
|
||||
let hosts = eps.hosts_sorted();
|
||||
let mut remote = Vec::with_capacity(hosts.len());
|
||||
let mut all = vec![None; hosts.len()];
|
||||
for (i, hs_host) in hosts.iter().enumerate() {
|
||||
if let Some(host) = hs_host
|
||||
&& let Some(grid_host) = eps.find_grid_hosts_from_peer(host)
|
||||
{
|
||||
let client = PeerRestClient::new(host.clone(), grid_host);
|
||||
fn build_clients_from_slots(
|
||||
slots: Vec<(String, Option<String>, bool)>,
|
||||
) -> (Vec<Option<Self>>, Vec<Option<Self>>, Vec<String>) {
|
||||
let mut remote = Vec::with_capacity(slots.len().saturating_sub(1));
|
||||
let mut all = vec![None; slots.len()];
|
||||
let mut remote_topology_hosts = Vec::with_capacity(slots.len().saturating_sub(1));
|
||||
|
||||
all[i] = Some(client.clone());
|
||||
remote.push(Some(client));
|
||||
for (idx, (peer_host_port, grid_host, is_local)) in slots.into_iter().enumerate() {
|
||||
if is_local {
|
||||
continue;
|
||||
}
|
||||
|
||||
let client = match grid_host {
|
||||
Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
|
||||
Ok(host) => Some(PeerRestClient::new(host, grid_host)),
|
||||
Err(err) => {
|
||||
warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
|
||||
None
|
||||
}
|
||||
},
|
||||
None => {
|
||||
warn!(peer = %peer_host_port, "grid host is missing while constructing peer client");
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
all[idx] = client.clone();
|
||||
remote.push(client);
|
||||
remote_topology_hosts.push(peer_host_port);
|
||||
}
|
||||
|
||||
(remote, all, remote_topology_hosts)
|
||||
}
|
||||
|
||||
pub async fn new_clients(eps: EndpointServerPools) -> (Vec<Option<Self>>, Vec<Option<Self>>) {
|
||||
let (remote, all, _) = Self::new_clients_with_topology(eps).await;
|
||||
(remote, all)
|
||||
}
|
||||
|
||||
pub async fn new_clients_with_topology(eps: EndpointServerPools) -> (Vec<Option<Self>>, Vec<Option<Self>>, Vec<String>) {
|
||||
if !runtime_sources::setup_is_dist_erasure().await {
|
||||
return (Vec::new(), Vec::new(), Vec::new());
|
||||
}
|
||||
|
||||
let (remote, all, remote_topology_hosts) = Self::build_clients_from_slots(eps.peer_grid_host_slots_sorted());
|
||||
|
||||
if all.len() != remote.len() + 1 {
|
||||
warn!("Expected number of all hosts ({}) to be remote +1 ({})", all.len(), remote.len());
|
||||
warn!(
|
||||
all_hosts = all.len(),
|
||||
remote_slots = remote.len(),
|
||||
"Expected number of all hosts to be remote slots + local node"
|
||||
);
|
||||
}
|
||||
|
||||
(remote, all)
|
||||
(remote, all, remote_topology_hosts)
|
||||
}
|
||||
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
@@ -1536,6 +1567,36 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_clients_from_slots_preserves_missing_remote_topology_slots() {
|
||||
let slots = vec![
|
||||
("127.0.0.1:9000".to_string(), None, true),
|
||||
("127.0.0.1:9001".to_string(), Some("http://127.0.0.1:9001".to_string()), false),
|
||||
("127.0.0.1:notaport".to_string(), Some("http://127.0.0.1:notaport".to_string()), false),
|
||||
("127.0.0.1:9003".to_string(), None, false),
|
||||
];
|
||||
|
||||
let (remote, all, remote_topology_hosts) = PeerRestClient::build_clients_from_slots(slots);
|
||||
|
||||
assert_eq!(remote.len(), 3, "local node is excluded but remote slots are not compacted away");
|
||||
assert_eq!(all.len(), 4, "all slots preserve the sorted cluster topology shape");
|
||||
assert_eq!(
|
||||
remote_topology_hosts,
|
||||
vec![
|
||||
"127.0.0.1:9001".to_string(),
|
||||
"127.0.0.1:notaport".to_string(),
|
||||
"127.0.0.1:9003".to_string()
|
||||
]
|
||||
);
|
||||
assert!(remote[0].is_some(), "valid remote peer should get a client");
|
||||
assert!(remote[1].is_none(), "unparseable remote peer should remain observable as a missing slot");
|
||||
assert!(remote[2].is_none(), "missing grid host should remain observable as a missing slot");
|
||||
assert!(all[0].is_none(), "local node is represented by the local server_info row");
|
||||
assert!(all[1].is_some());
|
||||
assert!(all[2].is_none());
|
||||
assert!(all[3].is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_requires_restart_safe_peer_identity() {
|
||||
let missing_instance = ScannerActivityResponse {
|
||||
|
||||
@@ -17,12 +17,15 @@ use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::{
|
||||
admin::StorageAdminApi,
|
||||
heal::HealOperations,
|
||||
namespace::NamespaceLocking,
|
||||
object::{DeletedObject, EcstoreObjectIO, ObjectIO, ObjectOperations, ObjectToDelete},
|
||||
range::HTTPRangeSpec,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use http::HeaderMap;
|
||||
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_config::audit::{
|
||||
@@ -45,10 +48,94 @@ use serde_json::{Map, Value};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tokio::sync::RwLock as AsyncRwLock;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
pub const CONFIG_PREFIX: &str = "config";
|
||||
const CONFIG_FILE: &str = "config.json";
|
||||
const SERVER_CONFIG_OBJECT: &str = "config/config.json";
|
||||
|
||||
// Server-config lock order: SERVER_CONFIG_LOCK -> distributed namespace lock
|
||||
// for SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order.
|
||||
static SERVER_CONFIG_LOCK: LazyLock<AsyncRwLock<()>> = LazyLock::new(|| AsyncRwLock::new(()));
|
||||
|
||||
fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError) -> Error {
|
||||
let outcome = if error.is_cancelled() { "cancelled" } else { "panicked" };
|
||||
Error::other(format!("{operation} task {outcome}"))
|
||||
}
|
||||
|
||||
/// Runs one complete server-config transaction while holding both the local
|
||||
/// process guard and the distributed namespace write lock for `config.json`.
|
||||
/// The operation must use the corresponding no-lock read/save functions.
|
||||
pub async fn with_server_config_write_lock<F, Fut, T>(store: Arc<ECStore>, operation: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
tokio::spawn(async move {
|
||||
// Lock order: SERVER_CONFIG_LOCK -> namespace write lock.
|
||||
let _local_guard = SERVER_CONFIG_LOCK.write().await;
|
||||
let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
|
||||
let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
|
||||
Ok(operation().await)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| config_task_join_error("server config write", error))?
|
||||
}
|
||||
|
||||
/// Reads and synchronously publishes one server-config snapshot while holding
|
||||
/// a shared namespace lock. Concurrent peer reloads may proceed together, but
|
||||
/// no writer can interleave between the snapshot read and generation accept.
|
||||
pub async fn with_server_config_read_lock<F, Fut, T>(store: Arc<ECStore>, operation: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
tokio::spawn(async move {
|
||||
// Lock order: SERVER_CONFIG_LOCK -> namespace read lock.
|
||||
let _local_guard = SERVER_CONFIG_LOCK.read().await;
|
||||
let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
|
||||
let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?;
|
||||
Ok(operation().await)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| config_task_join_error("server config read", error))?
|
||||
}
|
||||
|
||||
/// Runs a cancellation-safe transaction under the distributed write lock for
|
||||
/// one metadata config object. The operation must use no-lock object I/O.
|
||||
pub async fn with_config_object_write_lock<F, Fut, T>(store: Arc<ECStore>, object: String, operation: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
tokio::spawn(async move {
|
||||
let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
|
||||
Ok(operation().await)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| config_task_join_error("config object write", error))?
|
||||
}
|
||||
|
||||
/// Runs one read-and-publish transaction under the distributed read lock for
|
||||
/// a metadata config object. The operation must use no-lock object I/O.
|
||||
pub async fn with_config_object_read_lock<F, Fut, T>(store: Arc<ECStore>, object: String, operation: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
tokio::spawn(async move {
|
||||
let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?;
|
||||
Ok(operation().await)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| config_task_join_error("config object read", error))?
|
||||
}
|
||||
|
||||
/// Environment variable gating the startup fallback to the default server
|
||||
/// config when the persisted `config.json` object is corrupt beyond repair
|
||||
@@ -393,6 +480,31 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn save_config_no_lock<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
save_config_with_opts(
|
||||
api,
|
||||
file,
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(skip(api))]
|
||||
pub async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
|
||||
where
|
||||
@@ -462,8 +574,17 @@ where
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
async fn new_and_save_server_config_no_lock<S>(api: Arc<S>) -> Result<Config>
|
||||
where
|
||||
S: EcstoreObjectIO + StorageAdminApi,
|
||||
{
|
||||
let cfg = new_server_config();
|
||||
save_server_config_no_lock(api, &cfg).await?;
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
fn get_config_file() -> String {
|
||||
format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}")
|
||||
SERVER_CONFIG_OBJECT.to_string()
|
||||
}
|
||||
|
||||
fn storage_class_kvs_mut(cfg: &mut Config) -> &mut KVS {
|
||||
@@ -1267,13 +1388,17 @@ where
|
||||
}
|
||||
|
||||
/// Handle the situation where the configuration file does not exist, create and save a new configuration
|
||||
async fn handle_missing_config<S>(api: Arc<S>, context: &str) -> Result<Config>
|
||||
async fn handle_missing_config<S>(api: Arc<S>, context: &str, namespace_lock_held: bool) -> Result<Config>
|
||||
where
|
||||
S: EcstoreObjectIO + StorageAdminApi,
|
||||
{
|
||||
warn!("Configuration not found ({}): Start initializing new configuration", context);
|
||||
let cfg = if runtime_sources::first_cluster_node_is_local().await {
|
||||
new_and_save_server_config(api.clone()).await?
|
||||
if namespace_lock_held {
|
||||
new_and_save_server_config_no_lock(api.clone()).await?
|
||||
} else {
|
||||
new_and_save_server_config(api.clone()).await?
|
||||
}
|
||||
} else {
|
||||
new_server_config()
|
||||
};
|
||||
@@ -1288,6 +1413,46 @@ fn handle_config_read_error(err: Error, file_path: &str) -> Result<Config> {
|
||||
}
|
||||
|
||||
pub async fn read_config_without_migrate<S>(api: Arc<S>) -> Result<Config>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + StorageAdminApi,
|
||||
{
|
||||
read_config_without_migrate_inner(api, false).await
|
||||
}
|
||||
|
||||
/// Reads the server config while an upper layer holds the namespace write lock
|
||||
/// for [`SERVER_CONFIG_OBJECT`]. Missing-config initialization uses the matching
|
||||
/// no-lock save path and therefore cannot recursively acquire the same lock.
|
||||
pub async fn read_config_without_migrate_no_lock<S>(api: Arc<S>) -> Result<Config>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + StorageAdminApi,
|
||||
{
|
||||
read_config_without_migrate_inner(api, true).await
|
||||
}
|
||||
|
||||
/// Reads an already-initialized server config while a caller owns a namespace
|
||||
/// read lock. This never initializes or migrates a missing object.
|
||||
pub async fn read_existing_server_config_no_lock(api: Arc<ECStore>) -> Result<Config> {
|
||||
let data = read_config_no_lock(api, SERVER_CONFIG_OBJECT).await?;
|
||||
Ok(decode_persisted_server_config(&data)?.merge())
|
||||
}
|
||||
|
||||
async fn read_config_without_migrate_inner<S>(api: Arc<S>, namespace_lock_held: bool) -> Result<Config>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
@@ -1303,13 +1468,13 @@ where
|
||||
|
||||
// Try to read the configuration file
|
||||
match read_config_no_lock(api.clone(), &config_file).await {
|
||||
Ok(data) => read_server_config(api, &data).await,
|
||||
Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await,
|
||||
Ok(data) => read_server_config(api, &data, namespace_lock_held).await,
|
||||
Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", namespace_lock_held).await,
|
||||
Err(err) => handle_config_read_error(err, &config_file),
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_server_config<S>(api: Arc<S>, data: &[u8]) -> Result<Config>
|
||||
async fn read_server_config<S>(api: Arc<S>, data: &[u8], namespace_lock_held: bool) -> Result<Config>
|
||||
where
|
||||
S: EcstoreObjectIO + StorageAdminApi,
|
||||
{
|
||||
@@ -1324,7 +1489,9 @@ where
|
||||
let cfg = decode_persisted_server_config(&cfg_data)?;
|
||||
return Ok(cfg.merge());
|
||||
}
|
||||
Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await,
|
||||
Err(Error::ConfigNotFound) => {
|
||||
return handle_missing_config(api, "Read alternate configuration", namespace_lock_held).await;
|
||||
}
|
||||
Err(err) => return handle_config_read_error(err, &config_file),
|
||||
}
|
||||
}
|
||||
@@ -1534,6 +1701,38 @@ fn fallback_server_config_after_corruption(err: Error, config_file: &str, recove
|
||||
}
|
||||
|
||||
pub async fn save_server_config<S>(api: Arc<S>, cfg: &Config) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
save_server_config_inner(api, cfg, false).await
|
||||
}
|
||||
|
||||
/// Saves the server config while an upper layer holds the namespace write
|
||||
/// lock for [`SERVER_CONFIG_OBJECT`].
|
||||
pub async fn save_server_config_no_lock<S>(api: Arc<S>, cfg: &Config) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
save_server_config_inner(api, cfg, true).await
|
||||
}
|
||||
|
||||
async fn save_server_config_inner<S>(api: Arc<S>, cfg: &Config, no_lock: bool) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
@@ -1546,7 +1745,11 @@ where
|
||||
>,
|
||||
{
|
||||
let config_file = get_config_file();
|
||||
let existing = match read_config(api.clone(), &config_file).await {
|
||||
let existing = match if no_lock {
|
||||
read_config_no_lock(api.clone(), &config_file).await
|
||||
} else {
|
||||
read_config(api.clone(), &config_file).await
|
||||
} {
|
||||
Ok(v) => Some(v),
|
||||
Err(Error::ConfigNotFound) => None,
|
||||
Err(err) => {
|
||||
@@ -1570,7 +1773,21 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
save_config(api, &config_file, data).await
|
||||
if no_lock {
|
||||
save_config_with_opts(
|
||||
api,
|
||||
&config_file,
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
save_config(api, &config_file, data).await
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn lookup_configs<S>(cfg: &mut Config, api: Arc<S>) -> Result<()>
|
||||
@@ -1630,9 +1847,9 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
apply_dynamic_config_for_sub_sys_with, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
|
||||
is_standard_object_server_config, lookup_configs, read_config, read_config_preserve_empty, read_config_with_metadata,
|
||||
storage_class_kvs_mut,
|
||||
apply_dynamic_config_for_sub_sys_with, config_task_join_error, configs_semantically_equal, decode_server_config_blob,
|
||||
encode_server_config_blob, is_standard_object_server_config, lookup_configs, read_config, read_config_preserve_empty,
|
||||
read_config_with_metadata, storage_class_kvs_mut,
|
||||
};
|
||||
use crate::config::{audit, notify, oidc};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
@@ -1652,6 +1869,17 @@ mod tests {
|
||||
use rustfs_config::{
|
||||
DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn config_task_join_error_does_not_expose_panic_payload() {
|
||||
let join_error = tokio::spawn(async { panic!("do-not-expose-payload") })
|
||||
.await
|
||||
.expect_err("test task should panic");
|
||||
|
||||
let rendered = config_task_join_error("server config write", join_error).to_string();
|
||||
assert!(rendered.contains("panicked"));
|
||||
assert!(!rendered.contains("do-not-expose-payload"));
|
||||
}
|
||||
use rustfs_lock::client::LockClient;
|
||||
use rustfs_lock::client::local::LocalClient;
|
||||
use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
|
||||
|
||||
@@ -15,7 +15,11 @@
|
||||
use crate::cluster::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend_cached};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::{disk::endpoint::Endpoint, runtime::sources as runtime_sources};
|
||||
use crate::{
|
||||
disk::endpoint::{Endpoint, EndpointType},
|
||||
layout::endpoints::EndpointServerPools,
|
||||
runtime::sources as runtime_sources,
|
||||
};
|
||||
|
||||
use crate::data_usage::load_data_usage_cache;
|
||||
use crate::storage_api_contracts::admin::StorageAdminApi;
|
||||
@@ -291,6 +295,20 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
let after4 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("backend_info end {:?}", after4 - after3);
|
||||
if let Some(endpoints) = runtime_sources::endpoint_pools() {
|
||||
let (added, report) = reconcile_servers_with_endpoint_topology(&mut servers, &endpoints);
|
||||
if added > 0 || !report.is_complete() {
|
||||
warn!(
|
||||
event = "admin_v3_info_topology_incomplete",
|
||||
synthesized_servers = added,
|
||||
expected_drives = report.expected_drives,
|
||||
observed_drives = report.observed_drives,
|
||||
missing_drives = report.missing_drive_ids.len(),
|
||||
duplicate_drives = report.duplicate_drive_ids.len(),
|
||||
"admin v3 server_info reconciled endpoint topology before computing backend counters"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut all_disks: Vec<Disk> = Vec::new();
|
||||
for server in servers.iter() {
|
||||
@@ -397,6 +415,228 @@ fn get_online_offline_disks_stats(disks_info: &[Disk]) -> (BackendDisks, Backend
|
||||
(BackendDisks(online_disks), BackendDisks(offline_disks), BackendDisks(unknown_disks))
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
struct TopologyCompletenessReport {
|
||||
expected_drives: usize,
|
||||
observed_drives: usize,
|
||||
missing_drive_ids: Vec<TopologyDriveKey>,
|
||||
duplicate_drive_ids: Vec<TopologyDriveKey>,
|
||||
}
|
||||
|
||||
impl TopologyCompletenessReport {
|
||||
fn is_complete(&self) -> bool {
|
||||
self.expected_drives == self.observed_drives && self.missing_drive_ids.is_empty() && self.duplicate_drive_ids.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TopologyMember {
|
||||
display_endpoint: String,
|
||||
disks: Vec<Disk>,
|
||||
drive_keys: Vec<TopologyDriveKey>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
struct TopologyDriveKey {
|
||||
pool_index: i32,
|
||||
set_index: i32,
|
||||
disk_index: i32,
|
||||
member_index: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct EndpointTopology {
|
||||
members: Vec<TopologyMember>,
|
||||
aliases: HashMap<String, usize>,
|
||||
expected_drive_ids: HashSet<TopologyDriveKey>,
|
||||
}
|
||||
|
||||
impl EndpointTopology {
|
||||
fn from_endpoint_pools(endpoints: &EndpointServerPools) -> Self {
|
||||
let mut topology = Self::default();
|
||||
let mut by_host_port = HashMap::new();
|
||||
let mut display_aliases: HashMap<String, Option<usize>> = HashMap::new();
|
||||
|
||||
for pool in endpoints.as_ref() {
|
||||
for ep in pool.endpoints.as_ref() {
|
||||
if ep.get_type() != EndpointType::Url {
|
||||
continue;
|
||||
}
|
||||
let host_port = ep.host_port();
|
||||
if host_port.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let member_index = match by_host_port.get(&host_port).copied() {
|
||||
Some(index) => index,
|
||||
None => {
|
||||
let display_endpoint = ep.url.host_str().map(str::to_owned).unwrap_or_else(|| host_port.clone());
|
||||
let index = topology.members.len();
|
||||
topology.members.push(TopologyMember {
|
||||
display_endpoint,
|
||||
disks: Vec::new(),
|
||||
drive_keys: Vec::new(),
|
||||
});
|
||||
by_host_port.insert(host_port.clone(), index);
|
||||
index
|
||||
}
|
||||
};
|
||||
|
||||
topology.aliases.entry(host_port.clone()).or_insert(member_index);
|
||||
topology.aliases.entry(ep.to_string()).or_insert(member_index);
|
||||
if let Some(display_endpoint) = ep.url.host_str() {
|
||||
match display_aliases.entry(display_endpoint.to_owned()) {
|
||||
std::collections::hash_map::Entry::Vacant(entry) => {
|
||||
entry.insert(Some(member_index));
|
||||
}
|
||||
std::collections::hash_map::Entry::Occupied(mut entry) => {
|
||||
if entry.get().is_some_and(|index| index != member_index) {
|
||||
entry.insert(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let drive_key = TopologyDriveKey {
|
||||
pool_index: ep.pool_idx,
|
||||
set_index: ep.set_idx,
|
||||
disk_index: ep.disk_idx,
|
||||
member_index,
|
||||
};
|
||||
topology.expected_drive_ids.insert(drive_key.clone());
|
||||
topology.members[member_index].drive_keys.push(drive_key);
|
||||
topology.members[member_index].disks.push(Disk {
|
||||
endpoint: ep.to_string(),
|
||||
state: ITEM_UNKNOWN.to_string(),
|
||||
pool_index: ep.pool_idx,
|
||||
set_index: ep.set_idx,
|
||||
disk_index: ep.disk_idx,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (display_endpoint, member_index) in display_aliases {
|
||||
if let Some(member_index) = member_index {
|
||||
topology.aliases.entry(display_endpoint).or_insert(member_index);
|
||||
}
|
||||
}
|
||||
|
||||
topology
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
self.members.is_empty()
|
||||
}
|
||||
|
||||
fn observe_servers(&self, servers: &[ServerProperties]) -> (Vec<bool>, HashMap<TopologyDriveKey, usize>) {
|
||||
let mut observed_members = vec![false; self.members.len()];
|
||||
let mut observed_drive_counts = HashMap::with_capacity(self.expected_drive_ids.len());
|
||||
|
||||
for server in servers {
|
||||
if let Some(member_index) = self.member_index_from_endpoint(&server.endpoint) {
|
||||
observed_members[member_index] = true;
|
||||
}
|
||||
|
||||
for disk in &server.disks {
|
||||
if let Some(member_index) = self.member_index_from_endpoint(&disk.endpoint) {
|
||||
observed_members[member_index] = true;
|
||||
let drive_key = TopologyDriveKey {
|
||||
pool_index: disk.pool_index,
|
||||
set_index: disk.set_index,
|
||||
disk_index: disk.disk_index,
|
||||
member_index,
|
||||
};
|
||||
if self.expected_drive_ids.contains(&drive_key) {
|
||||
*observed_drive_counts.entry(drive_key).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(observed_members, observed_drive_counts)
|
||||
}
|
||||
|
||||
fn member_index_from_endpoint(&self, endpoint: &str) -> Option<usize> {
|
||||
if let Some(index) = self.aliases.get(endpoint) {
|
||||
return Some(*index);
|
||||
}
|
||||
|
||||
Endpoint::try_from(endpoint)
|
||||
.ok()
|
||||
.and_then(|ep| self.aliases.get(&ep.host_port()).copied())
|
||||
}
|
||||
|
||||
fn report_from_counts(&self, observed_drive_counts: HashMap<TopologyDriveKey, usize>) -> TopologyCompletenessReport {
|
||||
let mut missing_drive_ids = Vec::new();
|
||||
let mut duplicate_drive_ids = Vec::new();
|
||||
for id in &self.expected_drive_ids {
|
||||
match observed_drive_counts.get(id).copied().unwrap_or(0) {
|
||||
0 => missing_drive_ids.push(id.clone()),
|
||||
1 => {}
|
||||
_ => duplicate_drive_ids.push(id.clone()),
|
||||
}
|
||||
}
|
||||
missing_drive_ids.sort();
|
||||
duplicate_drive_ids.sort();
|
||||
|
||||
TopologyCompletenessReport {
|
||||
expected_drives: self.expected_drive_ids.len(),
|
||||
observed_drives: observed_drive_counts.values().sum(),
|
||||
missing_drive_ids,
|
||||
duplicate_drive_ids,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn reconcile_servers_with_endpoint_topology(
|
||||
servers: &mut Vec<ServerProperties>,
|
||||
endpoints: &EndpointServerPools,
|
||||
) -> (usize, TopologyCompletenessReport) {
|
||||
let topology = EndpointTopology::from_endpoint_pools(endpoints);
|
||||
if topology.is_empty() {
|
||||
return (0, TopologyCompletenessReport::default());
|
||||
}
|
||||
|
||||
let (observed_members, mut observed_drive_counts) = topology.observe_servers(servers);
|
||||
let mut missing: Vec<_> = topology
|
||||
.members
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(index, _)| !observed_members[*index])
|
||||
.map(|(_, member)| {
|
||||
for drive_key in &member.drive_keys {
|
||||
*observed_drive_counts.entry(drive_key.clone()).or_insert(0) += 1;
|
||||
}
|
||||
ServerProperties {
|
||||
endpoint: member.display_endpoint.clone(),
|
||||
state: ITEM_UNKNOWN.to_string(),
|
||||
disks: member.disks.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
missing.sort_by(|a, b| a.endpoint.cmp(&b.endpoint));
|
||||
|
||||
let added = missing.len();
|
||||
servers.extend(missing);
|
||||
let report = topology.report_from_counts(observed_drive_counts);
|
||||
|
||||
(added, report)
|
||||
}
|
||||
|
||||
fn server_topology_completeness_report(
|
||||
servers: &[ServerProperties],
|
||||
endpoints: &EndpointServerPools,
|
||||
) -> TopologyCompletenessReport {
|
||||
let topology = EndpointTopology::from_endpoint_pools(endpoints);
|
||||
if topology.is_empty() {
|
||||
return TopologyCompletenessReport::default();
|
||||
}
|
||||
let (_, observed_drive_counts) = topology.observe_servers(servers);
|
||||
topology.report_from_counts(observed_drive_counts)
|
||||
}
|
||||
|
||||
async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32, ErasureSetInfo>>> {
|
||||
let Some(store) = runtime_sources::object_store_handle() else {
|
||||
return Err(Error::other("ServerNotInitialized"));
|
||||
@@ -449,12 +689,16 @@ pub fn get_commit_id() -> String {
|
||||
mod tests {
|
||||
use serial_test::serial;
|
||||
|
||||
use crate::layout::{
|
||||
endpoint::Endpoint,
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use rustfs_madmin::{Disk, ITEM_OFFLINE, ITEM_UNKNOWN};
|
||||
use rustfs_madmin::{Disk, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, ServerProperties};
|
||||
|
||||
use super::{
|
||||
DATA_USAGE_UNAVAILABLE_ERROR, apply_data_usage_result, get_local_server_property, get_online_offline_disks_stats,
|
||||
get_server_info,
|
||||
get_server_info, reconcile_servers_with_endpoint_topology, server_topology_completeness_report,
|
||||
};
|
||||
|
||||
fn disk_with_state(endpoint: &str, state: &str) -> Disk {
|
||||
@@ -465,6 +709,50 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn topology_endpoint(host: &str, pool_index: usize, set_index: usize, disk_index: usize) -> Endpoint {
|
||||
topology_endpoint_url(format!("http://{host}:9000/data{disk_index}").as_str(), pool_index, set_index, disk_index)
|
||||
}
|
||||
|
||||
fn topology_endpoint_url(url: &str, pool_index: usize, set_index: usize, disk_index: usize) -> Endpoint {
|
||||
let mut endpoint = Endpoint::try_from(url).expect("URL endpoint should parse");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(set_index);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoint
|
||||
}
|
||||
|
||||
fn topology_with_hosts(hosts: &[&str]) -> EndpointServerPools {
|
||||
let endpoints: Vec<Endpoint> = hosts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(disk_index, host)| topology_endpoint(host, 0, 0, disk_index))
|
||||
.collect();
|
||||
EndpointServerPools::from(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: hosts.len(),
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: String::new(),
|
||||
platform: String::new(),
|
||||
}])
|
||||
}
|
||||
|
||||
fn server_with_disk(host: &str, disk_index: i32, state: &str) -> ServerProperties {
|
||||
ServerProperties {
|
||||
endpoint: host.to_string(),
|
||||
state: ITEM_ONLINE.to_string(),
|
||||
disks: vec![Disk {
|
||||
endpoint: format!("http://{host}:9000/data{disk_index}"),
|
||||
state: state.to_string(),
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
disk_index,
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_stats_split_unknown_into_its_own_bucket() {
|
||||
// A member whose properties RPC could not be answered contributes
|
||||
@@ -493,6 +781,93 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_reconcile_synthesizes_missing_member_as_unknown() {
|
||||
let endpoints = topology_with_hosts(&["rustfs-1", "rustfs-2", "rustfs-3", "rustfs-4"]);
|
||||
let mut servers = vec![
|
||||
server_with_disk("rustfs-1", 0, "ok"),
|
||||
server_with_disk("rustfs-2", 1, "ok"),
|
||||
server_with_disk("rustfs-3", 2, "ok"),
|
||||
];
|
||||
|
||||
let (added, report) = reconcile_servers_with_endpoint_topology(&mut servers, &endpoints);
|
||||
let (_, _, unknown) =
|
||||
get_online_offline_disks_stats(&servers.iter().flat_map(|server| server.disks.clone()).collect::<Vec<_>>());
|
||||
|
||||
assert_eq!(added, 1, "the missing fourth topology member must be synthesized");
|
||||
assert!(report.is_complete(), "synthesized drives should complete the topology report");
|
||||
assert_eq!(servers.len(), 4, "v3 server list must preserve topology membership length");
|
||||
let synthesized = servers
|
||||
.iter()
|
||||
.find(|server| server.endpoint == "rustfs-4")
|
||||
.expect("missing member should be present");
|
||||
assert_eq!(synthesized.state, ITEM_UNKNOWN);
|
||||
assert_eq!(synthesized.disks.len(), 1);
|
||||
assert_eq!(synthesized.disks[0].endpoint, "http://rustfs-4:9000/data3");
|
||||
assert_eq!(synthesized.disks[0].disk_index, 3);
|
||||
assert_eq!(unknown.sum(), 1, "the synthesized drive must land in unknownDisks");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_reconcile_does_not_duplicate_existing_synthesized_rows() {
|
||||
let endpoints = topology_with_hosts(&["rustfs-1", "rustfs-2"]);
|
||||
let mut servers = vec![
|
||||
server_with_disk("rustfs-1", 0, "ok"),
|
||||
server_with_disk("rustfs-2", 1, ITEM_UNKNOWN),
|
||||
];
|
||||
servers[1].state = ITEM_UNKNOWN.to_string();
|
||||
|
||||
let (added, report) = reconcile_servers_with_endpoint_topology(&mut servers, &endpoints);
|
||||
|
||||
assert_eq!(
|
||||
added, 0,
|
||||
"an existing unknown/degraded/offline row with topology drives already represents the member"
|
||||
);
|
||||
assert!(report.is_complete(), "existing synthesized rows should already complete the topology");
|
||||
assert_eq!(servers.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_reconcile_does_not_over_match_ambiguous_host_only_aliases() {
|
||||
let endpoints = EndpointServerPools::from(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 2,
|
||||
endpoints: Endpoints::from(vec![
|
||||
topology_endpoint_url("http://rustfs-1:9000/data0", 0, 0, 0),
|
||||
topology_endpoint_url("http://rustfs-1:9001/data1", 0, 0, 1),
|
||||
]),
|
||||
cmd_line: String::new(),
|
||||
platform: String::new(),
|
||||
}]);
|
||||
let mut servers = vec![server_with_disk("rustfs-1", 0, "ok")];
|
||||
|
||||
let (added, report) = reconcile_servers_with_endpoint_topology(&mut servers, &endpoints);
|
||||
|
||||
assert_eq!(added, 1, "host-only aliases are ambiguous when one host has multiple topology ports");
|
||||
assert!(report.is_complete());
|
||||
assert!(
|
||||
servers
|
||||
.iter()
|
||||
.any(|server| server.disks.iter().any(|disk| disk.endpoint == "http://rustfs-1:9001/data1")),
|
||||
"the second host:port member should be synthesized from exact topology"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_report_detects_duplicate_drive_identity_with_balanced_total() {
|
||||
let endpoints = topology_with_hosts(&["rustfs-1", "rustfs-2"]);
|
||||
let servers = vec![server_with_disk("rustfs-1", 0, "ok"), server_with_disk("rustfs-1", 0, "ok")];
|
||||
|
||||
let report = server_topology_completeness_report(&servers, &endpoints);
|
||||
|
||||
assert_eq!(report.expected_drives, 2);
|
||||
assert_eq!(report.observed_drives, 2, "a plain total-count check would look balanced");
|
||||
assert_eq!(report.missing_drive_ids.len(), 1, "rustfs-2's drive identity is absent");
|
||||
assert_eq!(report.duplicate_drive_ids.len(), 1, "rustfs-1's drive identity is duplicated");
|
||||
assert!(!report.is_complete());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_usage_errors_are_sanitized_in_server_info() {
|
||||
let mut buckets = rustfs_madmin::Buckets::default();
|
||||
|
||||
@@ -22,7 +22,10 @@ use crate::diagnostics::get::{
|
||||
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
|
||||
use crate::erasure::coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
|
||||
use bytes::Bytes;
|
||||
use rustfs_config::{DEFAULT_OBJECT_MMAP_READ_ENABLE, ENV_OBJECT_MMAP_READ_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
|
||||
use rustfs_config::{
|
||||
DEFAULT_OBJECT_MMAP_READ_ENABLE, DEFAULT_OBJECT_MMAP_READ_MAX_LENGTH, ENV_OBJECT_MMAP_READ_ENABLE,
|
||||
ENV_OBJECT_MMAP_READ_MAX_LENGTH, ENV_OBJECT_ZERO_COPY_ENABLE,
|
||||
};
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use std::future::Future;
|
||||
use std::io::{self, Cursor};
|
||||
@@ -81,6 +84,21 @@ pub(crate) fn object_mmap_read_enabled() -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
/// Mmap-copy read length cap. Cached: this is consulted on every shard open
|
||||
/// and `std::env::var` takes a process-global lock. In test builds the env
|
||||
/// var is read directly so `temp_env` overrides take effect.
|
||||
pub(crate) fn object_mmap_read_max_length() -> usize {
|
||||
#[cfg(test)]
|
||||
{
|
||||
rustfs_utils::get_env_usize(ENV_OBJECT_MMAP_READ_MAX_LENGTH, DEFAULT_OBJECT_MMAP_READ_MAX_LENGTH)
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
static CACHED: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
|
||||
*CACHED.get_or_init(|| rustfs_utils::get_env_usize(ENV_OBJECT_MMAP_READ_MAX_LENGTH, DEFAULT_OBJECT_MMAP_READ_MAX_LENGTH))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct BitrotReaderSource {
|
||||
inline_data: Option<Bytes>,
|
||||
@@ -305,7 +323,28 @@ async fn open_disk_reader(
|
||||
let metrics_path = metrics_path.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
|
||||
let stage_metrics_enabled = metrics_path.is_some();
|
||||
|
||||
if use_mmap_read && disk.is_local() {
|
||||
// Mmap-copy materializes the whole `offset..offset+length` range as one
|
||||
// owned allocation before any byte is served, and GET/heal shard reads
|
||||
// request the entire part span in one call. Over-cap reads (e.g. a huge
|
||||
// single-part object, rustfs#5123) take the bounded streaming path below.
|
||||
let use_mmap_copy = if use_mmap_read && disk.is_local() {
|
||||
let mmap_read_cap = object_mmap_read_max_length();
|
||||
let within_cap = length <= mmap_read_cap;
|
||||
if !within_cap {
|
||||
rustfs_io_metrics::record_zero_copy_fallback("read_length_exceeds_cap");
|
||||
debug!(
|
||||
length,
|
||||
mmap_read_cap,
|
||||
path = %path,
|
||||
"zero_copy_read_skipped_over_cap"
|
||||
);
|
||||
}
|
||||
within_cap
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
if use_mmap_copy {
|
||||
let start = stage_metrics_enabled.then(Instant::now);
|
||||
let zero_copy_start = Instant::now();
|
||||
let mmap_metrics = metrics_path.map(|metrics_path| MmapCopyStageMetrics {
|
||||
@@ -685,6 +724,96 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_mmap_read_max_length_defaults_and_env_override() {
|
||||
temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, None::<&str>, || {
|
||||
assert_eq!(object_mmap_read_max_length(), DEFAULT_OBJECT_MMAP_READ_MAX_LENGTH);
|
||||
});
|
||||
temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, Some("1024"), || {
|
||||
assert_eq!(object_mmap_read_max_length(), 1024);
|
||||
});
|
||||
temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, Some("0"), || {
|
||||
assert_eq!(object_mmap_read_max_length(), 0);
|
||||
});
|
||||
}
|
||||
|
||||
// rustfs#5123: whole-part shard reads of large single-part objects must not
|
||||
// be materialized in memory by the mmap-copy path; over-cap reads stream.
|
||||
#[tokio::test]
|
||||
async fn open_disk_reader_streams_when_length_exceeds_mmap_cap() {
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::{DiskOption, new_disk};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("local disk should be created");
|
||||
|
||||
let payload = vec![7u8; 4096];
|
||||
disk.make_volume("test-bucket").await.expect("volume should be created");
|
||||
disk.write_all("test-bucket", "obj/part.1", Bytes::from(payload.clone()))
|
||||
.await
|
||||
.expect("shard file should be written");
|
||||
|
||||
temp_env::async_with_vars([(ENV_OBJECT_MMAP_READ_MAX_LENGTH, Some("1024"))], async {
|
||||
let over_cap = open_disk_reader(&disk, "test-bucket", "obj/part.1", 0, payload.len(), true, None)
|
||||
.await
|
||||
.expect("over-cap read should open");
|
||||
assert!(
|
||||
matches!(over_cap, ShardReader::Stream(_)),
|
||||
"read longer than the mmap cap must take the streaming path"
|
||||
);
|
||||
let mut over_cap = over_cap;
|
||||
let mut streamed = Vec::new();
|
||||
over_cap
|
||||
.read_to_end(&mut streamed)
|
||||
.await
|
||||
.expect("streaming fallback should read the range");
|
||||
assert_eq!(streamed, payload, "streaming fallback must return the same bytes");
|
||||
|
||||
let under_cap = open_disk_reader(&disk, "test-bucket", "obj/part.1", 0, 512, true, None)
|
||||
.await
|
||||
.expect("under-cap read should open");
|
||||
assert!(
|
||||
matches!(under_cap, ShardReader::InMemory(_)),
|
||||
"read within the mmap cap keeps the mmap-copy fast path"
|
||||
);
|
||||
|
||||
let at_cap = open_disk_reader(&disk, "test-bucket", "obj/part.1", 0, 1024, true, None)
|
||||
.await
|
||||
.expect("at-cap read should open");
|
||||
assert!(
|
||||
matches!(at_cap, ShardReader::InMemory(_)),
|
||||
"read of exactly the mmap cap keeps the mmap-copy fast path"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
|
||||
// Cap of 0 disables mmap-copy for every non-empty read.
|
||||
temp_env::async_with_vars([(ENV_OBJECT_MMAP_READ_MAX_LENGTH, Some("0"))], async {
|
||||
let disabled = open_disk_reader(&disk, "test-bucket", "obj/part.1", 0, 512, true, None)
|
||||
.await
|
||||
.expect("read with cap 0 should open");
|
||||
assert!(
|
||||
matches!(disabled, ShardReader::Stream(_)),
|
||||
"cap 0 must route every non-empty read to the streaming path"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_reader_with_inline_data() {
|
||||
let test_data = b"hello world test data";
|
||||
|
||||
@@ -1020,18 +1020,14 @@ impl EndpointServerPools {
|
||||
|
||||
#[instrument]
|
||||
pub fn hosts_sorted(&self) -> Vec<Option<XHost>> {
|
||||
let (mut peers, local) = self.peers();
|
||||
let peers = self.peer_grid_hosts_sorted();
|
||||
|
||||
let mut ret = vec![None; peers.len()];
|
||||
|
||||
peers.sort();
|
||||
|
||||
for (i, peer) in peers.iter().enumerate() {
|
||||
if &local == peer {
|
||||
for (i, peer) in peers.into_iter().enumerate() {
|
||||
let Some((peer_host_port, _)) = peer else {
|
||||
continue;
|
||||
}
|
||||
|
||||
let host = match XHost::try_from(peer.clone()) {
|
||||
};
|
||||
let host = match XHost::try_from(peer_host_port) {
|
||||
Ok(res) => res,
|
||||
Err(err) => {
|
||||
warn!("Xhost parse failed {:?}", err);
|
||||
@@ -1044,6 +1040,61 @@ impl EndpointServerPools {
|
||||
|
||||
ret
|
||||
}
|
||||
|
||||
pub fn peer_host_ports_sorted(&self) -> Vec<Option<String>> {
|
||||
let (mut peers, local) = self.peers();
|
||||
let mut ret = vec![None; peers.len()];
|
||||
|
||||
peers.sort();
|
||||
|
||||
for (i, peer) in peers.into_iter().enumerate() {
|
||||
if local == peer {
|
||||
continue;
|
||||
}
|
||||
ret[i] = Some(peer);
|
||||
}
|
||||
|
||||
ret
|
||||
}
|
||||
|
||||
pub fn peer_grid_hosts_sorted(&self) -> Vec<Option<(String, String)>> {
|
||||
self.peer_grid_host_slots_sorted()
|
||||
.into_iter()
|
||||
.map(|(peer, grid_host, is_local)| {
|
||||
if is_local {
|
||||
None
|
||||
} else {
|
||||
grid_host.map(|grid_host| (peer, grid_host))
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn peer_grid_host_slots_sorted(&self) -> Vec<(String, Option<String>, bool)> {
|
||||
let (mut peers, local) = self.peers();
|
||||
let mut grid_hosts = HashMap::with_capacity(peers.len());
|
||||
|
||||
for ep in self.0.iter() {
|
||||
for endpoint in ep.endpoints.0.iter() {
|
||||
if endpoint.get_type() != EndpointType::Url || endpoint.is_local {
|
||||
continue;
|
||||
}
|
||||
grid_hosts.entry(endpoint.host_port()).or_insert_with(|| endpoint.grid_host());
|
||||
}
|
||||
}
|
||||
|
||||
peers.sort();
|
||||
|
||||
peers
|
||||
.into_iter()
|
||||
.map(|peer| {
|
||||
let is_local = local == peer;
|
||||
let grid_host = if is_local { None } else { grid_hosts.get(&peer).cloned() };
|
||||
(peer, grid_host, is_local)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn peers(&self) -> (Vec<String>, String) {
|
||||
let mut local = None;
|
||||
let mut set = HashSet::new();
|
||||
@@ -1087,6 +1138,25 @@ impl EndpointServerPools {
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub fn find_grid_host_from_peer_host_port(&self, peer_host_port: &str) -> Option<String> {
|
||||
for ep in self.0.iter() {
|
||||
for endpoint in ep.endpoints.0.iter() {
|
||||
if endpoint.is_local {
|
||||
continue;
|
||||
}
|
||||
if endpoint.host_port() == peer_host_port {
|
||||
return Some(endpoint.grid_host());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(host) = XHost::try_from(peer_host_port.to_owned()) {
|
||||
return self.find_grid_hosts_from_peer(&host);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()> {
|
||||
@@ -1407,6 +1477,43 @@ mod test {
|
||||
assert!(!eps[2].is_local, "a genuinely remote host must stay remote");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_host_ports_sorted_preserves_raw_remote_hosts() {
|
||||
let mut local =
|
||||
Endpoint::try_from("http://rustfs-1.storage.swarm.private:9000/data").expect("local endpoint should parse");
|
||||
local.is_local = true;
|
||||
let mut remote =
|
||||
Endpoint::try_from("http://rustfs-4.storage.swarm.private:9000/data").expect("remote endpoint should parse");
|
||||
remote.is_local = false;
|
||||
let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 2,
|
||||
endpoints: Endpoints::from(vec![local, remote]),
|
||||
cmd_line: String::new(),
|
||||
platform: String::new(),
|
||||
}]);
|
||||
|
||||
let peers = endpoint_pools.peer_host_ports_sorted();
|
||||
|
||||
assert_eq!(
|
||||
peers,
|
||||
vec![None, Some("rustfs-4.storage.swarm.private:9000".to_string())],
|
||||
"membership enumeration must keep raw topology host:port instead of requiring DNS resolution"
|
||||
);
|
||||
assert_eq!(
|
||||
endpoint_pools
|
||||
.find_grid_host_from_peer_host_port("rustfs-4.storage.swarm.private:9000")
|
||||
.as_deref(),
|
||||
Some("http://rustfs-4.storage.swarm.private:9000"),
|
||||
"raw host:port should map directly to the configured grid host"
|
||||
);
|
||||
let peer_grid_hosts = endpoint_pools.peer_grid_hosts_sorted();
|
||||
let remote = peer_grid_hosts[1].as_ref().expect("remote peer should be resolved");
|
||||
assert_eq!(remote.0, "rustfs-4.storage.swarm.private:9000");
|
||||
assert_eq!(remote.1, "http://rustfs-4.storage.swarm.private:9000");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orchestrated_policy_defers_dns_ip_same_path_check() {
|
||||
// Two remote endpoints on the same address, same path, different ports
|
||||
|
||||
@@ -95,16 +95,18 @@ pub fn get_global_notification_sys() -> Option<Arc<NotificationSys>> {
|
||||
pub struct NotificationSys {
|
||||
pub peer_clients: Vec<Option<PeerRestClient>>,
|
||||
pub all_peer_clients: Vec<Option<PeerRestClient>>,
|
||||
peer_topology_hosts: Vec<String>,
|
||||
peer_admin_caches: Vec<Mutex<PeerAdminCache>>,
|
||||
}
|
||||
|
||||
impl NotificationSys {
|
||||
pub async fn new(eps: EndpointServerPools) -> Self {
|
||||
let (peer_clients, all_peer_clients) = PeerRestClient::new_clients(eps).await;
|
||||
let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
|
||||
let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
|
||||
Self {
|
||||
peer_clients,
|
||||
all_peer_clients,
|
||||
peer_topology_hosts,
|
||||
peer_admin_caches,
|
||||
}
|
||||
}
|
||||
@@ -253,7 +255,7 @@ impl NotificationSys {
|
||||
join_all(futures).await
|
||||
}
|
||||
|
||||
pub async fn reload_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
|
||||
async fn signal_dynamic_config(&self, sub_sys: &str, dry_run: bool) -> Vec<NotificationPeerErr> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter() {
|
||||
let sub_sys = sub_sys.to_string();
|
||||
@@ -263,7 +265,7 @@ impl NotificationSys {
|
||||
.signal_service(
|
||||
crate::cluster::rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
&sub_sys,
|
||||
false,
|
||||
dry_run,
|
||||
SystemTime::UNIX_EPOCH,
|
||||
)
|
||||
.await
|
||||
@@ -288,6 +290,14 @@ impl NotificationSys {
|
||||
join_all(futures).await
|
||||
}
|
||||
|
||||
pub async fn preflight_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
|
||||
self.signal_dynamic_config(sub_sys, true).await
|
||||
}
|
||||
|
||||
pub async fn reload_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
|
||||
self.signal_dynamic_config(sub_sys, false).await
|
||||
}
|
||||
|
||||
pub async fn refresh_config_snapshot(&self) -> Vec<NotificationPeerErr> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter() {
|
||||
@@ -371,22 +381,19 @@ impl NotificationSys {
|
||||
let peer_timeout = Duration::from_secs(5);
|
||||
|
||||
for (idx, client) in self.peer_clients.iter().enumerate() {
|
||||
let endpoints = endpoints.clone();
|
||||
let cache = self.peer_admin_caches.get(idx);
|
||||
let host = self
|
||||
.peer_topology_hosts
|
||||
.get(idx)
|
||||
.cloned()
|
||||
.or_else(|| client.as_ref().map(|client| client.host.to_string()))
|
||||
.unwrap_or_default();
|
||||
futures.push(async move {
|
||||
// `peer_clients` comes from `new_clients`, which only ever pushes
|
||||
// `Some(client)` (local hosts are excluded, not slotted as
|
||||
// `None`), so this branch is unreachable in practice. Kept as a
|
||||
// defensive fallback: report an explicit `unknown` state rather
|
||||
// than a blank `default()` entry, so it can never contribute a
|
||||
// hollow row to `servers[]` (rustfs/backlog#1049 P3).
|
||||
let Some(client) = client else {
|
||||
return ServerProperties {
|
||||
state: ItemState::Unknown.to_string().to_owned(),
|
||||
..Default::default()
|
||||
return PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::NoClient),
|
||||
};
|
||||
};
|
||||
let host = client.host.to_string();
|
||||
|
||||
// First attempt. A single evicted or half-open internode channel
|
||||
// is enough to fail one probe and, before retrying, would drop
|
||||
@@ -395,8 +402,7 @@ impl NotificationSys {
|
||||
// before falling back (rustfs/backlog#1049, P1-B).
|
||||
match timeout(peer_timeout, client.server_info()).await {
|
||||
Ok(Ok(info)) => {
|
||||
update_server_info_cache(cache, &host, &info);
|
||||
return info;
|
||||
return PeerServerInfoProbe { host, result: Ok(info) };
|
||||
}
|
||||
Ok(Err(err)) => debug!("peer {host} server_info failed (attempt 1/2): {err}"),
|
||||
Err(_) => debug!("peer {host} server_info timed out (attempt 1/2) after {peer_timeout:?}"),
|
||||
@@ -412,26 +418,29 @@ impl NotificationSys {
|
||||
|
||||
// Second and final attempt on the fresh channel.
|
||||
match timeout(peer_timeout, client.server_info()).await {
|
||||
Ok(Ok(info)) => {
|
||||
update_server_info_cache(cache, &host, &info);
|
||||
info
|
||||
}
|
||||
Ok(Ok(info)) => PeerServerInfoProbe { host, result: Ok(info) },
|
||||
Ok(Err(err)) => {
|
||||
warn!("peer {host} server_info failed after retry: {err}");
|
||||
let health = peer_disk_health(&host).await;
|
||||
handle_server_info_failure(cache, &host, &endpoints, health.as_ref())
|
||||
PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("peer {host} server_info timed out after retry ({peer_timeout:?})");
|
||||
client.evict_connection().await;
|
||||
let health = peer_disk_health(&host).await;
|
||||
handle_server_info_failure(cache, &host, &endpoints, health.as_ref())
|
||||
PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
join_all(futures).await
|
||||
publish_server_info_probe_round(&self.peer_admin_caches, &endpoints, join_all(futures).await)
|
||||
}
|
||||
|
||||
pub async fn load_user(&self, access_key: &str, temp: bool) -> Vec<NotificationPeerErr> {
|
||||
@@ -1255,6 +1264,16 @@ struct PeerDiskHealth {
|
||||
disks: Vec<rustfs_madmin::Disk>,
|
||||
}
|
||||
|
||||
struct PeerServerInfoProbe {
|
||||
host: String,
|
||||
result: std::result::Result<ServerProperties, PeerServerInfoProbeFailure>,
|
||||
}
|
||||
|
||||
enum PeerServerInfoProbeFailure {
|
||||
Rpc { health: Option<PeerDiskHealth> },
|
||||
NoClient,
|
||||
}
|
||||
|
||||
/// Consult the local disk-health state for `host` without issuing any RPC.
|
||||
///
|
||||
/// On the aggregating node a peer's drives are remote-disk handles whose
|
||||
@@ -1416,6 +1435,30 @@ fn handle_server_info_failure(
|
||||
unknown_server_properties(host, endpoints)
|
||||
}
|
||||
|
||||
fn publish_server_info_probe_round(
|
||||
caches: &[Mutex<PeerAdminCache>],
|
||||
endpoints: &EndpointServerPools,
|
||||
probes: Vec<PeerServerInfoProbe>,
|
||||
) -> Vec<ServerProperties> {
|
||||
probes
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(idx, probe)| {
|
||||
let cache = caches.get(idx);
|
||||
match probe.result {
|
||||
Ok(info) => {
|
||||
update_server_info_cache(cache, &probe.host, &info);
|
||||
info
|
||||
}
|
||||
Err(PeerServerInfoProbeFailure::Rpc { health }) => {
|
||||
handle_server_info_failure(cache, &probe.host, endpoints, health.as_ref())
|
||||
}
|
||||
Err(PeerServerInfoProbeFailure::NoClient) => unknown_server_properties(&probe.host, endpoints),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn update_server_info_cache(cache: Option<&Mutex<PeerAdminCache>>, host: &str, info: &ServerProperties) {
|
||||
let Some(cache) = cache else {
|
||||
return;
|
||||
@@ -1626,6 +1669,7 @@ mod tests {
|
||||
"127.0.0.1:9000".to_string().try_into().expect("peer host should parse"),
|
||||
"http://127.0.0.1:9000".to_string(),
|
||||
))],
|
||||
peer_topology_hosts: Vec::new(),
|
||||
peer_admin_caches: Vec::new(),
|
||||
};
|
||||
|
||||
@@ -1669,6 +1713,7 @@ mod tests {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: Vec::new(),
|
||||
peer_topology_hosts: vec!["node-a:9000".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
|
||||
@@ -1688,6 +1733,7 @@ mod tests {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: vec![None, None],
|
||||
peer_topology_hosts: vec!["node-a:9000".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
|
||||
@@ -1704,6 +1750,7 @@ mod tests {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: Vec::new(),
|
||||
peer_topology_hosts: Vec::new(),
|
||||
peer_admin_caches: Vec::new(),
|
||||
};
|
||||
|
||||
@@ -1724,6 +1771,7 @@ mod tests {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![Some(client)],
|
||||
all_peer_clients: vec![None],
|
||||
peer_topology_hosts: vec!["127.0.0.1:9000".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
|
||||
@@ -1735,6 +1783,51 @@ mod tests {
|
||||
assert!(err.to_string().contains("peer topology is incomplete"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn server_info_no_client_slot_uses_topology_host_without_counting_rpc_failure() {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: vec![None, None],
|
||||
peer_topology_hosts: vec!["node-a:9000".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
|
||||
let servers = sys.server_info().await;
|
||||
|
||||
assert_eq!(servers.len(), 1);
|
||||
assert_eq!(servers[0].endpoint, "node-a:9000");
|
||||
assert_eq!(servers[0].state, ItemState::Unknown.to_string());
|
||||
let cache = sys.peer_admin_caches[0].lock().expect("cache mutex should not be poisoned");
|
||||
assert_eq!(cache.server_failures, 0, "construction-only missing slots are not failed RPC attempts");
|
||||
assert!(cache.last_server_info.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_info_failure_cache_stays_aligned_with_topology_slot() {
|
||||
let cache_a = Mutex::new(PeerAdminCache {
|
||||
last_server_info: Some(build_props("cached-a")),
|
||||
last_server_success: Some(SystemTime::now()),
|
||||
server_failures: 1,
|
||||
storage_failures: 0,
|
||||
last_storage_info: None,
|
||||
});
|
||||
let cache_b = Mutex::new(PeerAdminCache {
|
||||
last_server_info: Some(build_props("cached-b")),
|
||||
last_server_success: Some(SystemTime::now()),
|
||||
server_failures: 1,
|
||||
storage_failures: 0,
|
||||
last_storage_info: None,
|
||||
});
|
||||
let caches = [cache_a, cache_b];
|
||||
let endpoints = EndpointServerPools::from(Vec::new());
|
||||
|
||||
let rendered = handle_server_info_failure(Some(&caches[1]), "node-b:9000", &endpoints, None);
|
||||
|
||||
assert_eq!(rendered.endpoint, "cached-b");
|
||||
assert_eq!(caches[0].lock().expect("cache mutex should not be poisoned").server_failures, 1);
|
||||
assert_eq!(caches[1].lock().expect("cache mutex should not be poisoned").server_failures, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_activity_probe_times_out() {
|
||||
let err = scanner_activity_with_timeout(
|
||||
@@ -1754,6 +1847,7 @@ mod tests {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: Vec::new(),
|
||||
peer_topology_hosts: vec!["node-a:9000".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
|
||||
@@ -1773,6 +1867,7 @@ mod tests {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: Vec::new(),
|
||||
peer_topology_hosts: vec!["node-a:9000".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
|
||||
@@ -1788,6 +1883,7 @@ mod tests {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: Vec::new(),
|
||||
peer_topology_hosts: vec!["node-a:9000".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
let mutation_id = Uuid::from_u128(1);
|
||||
@@ -1988,6 +2084,60 @@ mod tests {
|
||||
assert!(!cached_snapshot_is_fresh(Some(stale)), "an old success is stale");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_info_probe_round_commits_failures_only_when_published() {
|
||||
let caches = vec![Mutex::new(PeerAdminCache::new())];
|
||||
let endpoints = EndpointServerPools::default();
|
||||
let probes = vec![PeerServerInfoProbe {
|
||||
host: "peer-1".to_string(),
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health: None }),
|
||||
}];
|
||||
|
||||
assert_eq!(
|
||||
caches[0]
|
||||
.lock()
|
||||
.expect("peer cache should lock before publish")
|
||||
.server_failures,
|
||||
0
|
||||
);
|
||||
|
||||
let replies = publish_server_info_probe_round(&caches, &endpoints, probes);
|
||||
|
||||
assert_eq!(replies.len(), 1);
|
||||
assert_eq!(replies[0].endpoint, "peer-1");
|
||||
assert_eq!(replies[0].state, ItemState::Unknown.to_string());
|
||||
assert_eq!(
|
||||
caches[0]
|
||||
.lock()
|
||||
.expect("peer cache should lock after publish")
|
||||
.server_failures,
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_info_probe_round_does_not_count_no_client_slots_as_rpc_failures() {
|
||||
let caches = vec![Mutex::new(PeerAdminCache::new())];
|
||||
let endpoints = EndpointServerPools::default();
|
||||
let probes = vec![PeerServerInfoProbe {
|
||||
host: "node-a:9000".to_string(),
|
||||
result: Err(PeerServerInfoProbeFailure::NoClient),
|
||||
}];
|
||||
|
||||
let replies = publish_server_info_probe_round(&caches, &endpoints, probes);
|
||||
|
||||
assert_eq!(replies.len(), 1);
|
||||
assert_eq!(replies[0].endpoint, "node-a:9000");
|
||||
assert_eq!(replies[0].state, ItemState::Unknown.to_string());
|
||||
assert_eq!(
|
||||
caches[0]
|
||||
.lock()
|
||||
.expect("peer cache should lock after no-client publish")
|
||||
.server_failures,
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_host_matches_direct_and_canonicalized() {
|
||||
// Direct match (IP deployment): peer host already equals host_port.
|
||||
|
||||
@@ -74,7 +74,9 @@ use crate::disk::format::FormatV3;
|
||||
use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType};
|
||||
use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options};
|
||||
use crate::services::tier::warm_backend::{
|
||||
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||
};
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
|
||||
@@ -142,6 +144,7 @@ pub enum MockWarmOp {
|
||||
Put { object: String },
|
||||
Get { object: String },
|
||||
Remove { object: String },
|
||||
Probe { object: String },
|
||||
ExternalRemove { object: String },
|
||||
InUse,
|
||||
}
|
||||
@@ -160,7 +163,10 @@ struct MockWarmBackendInner {
|
||||
faults: Mutex<FaultConfig>,
|
||||
put_read_limit: Mutex<Option<usize>>,
|
||||
put_remote_version: Mutex<Option<String>>,
|
||||
response_loss_after_put: AtomicBool,
|
||||
transition_candidate_probe_override: Mutex<Option<TransitionCandidateProbe>>,
|
||||
reject_non_empty_remote_versions: AtomicBool,
|
||||
reject_non_empty_remote_version_validations: AtomicUsize,
|
||||
fail_remove: AtomicBool,
|
||||
exact_remove_count: AtomicUsize,
|
||||
op_log: Mutex<Vec<MockWarmOp>>,
|
||||
@@ -378,11 +384,29 @@ impl MockWarmBackend {
|
||||
*self.inner.put_remote_version.lock().await = remote_version;
|
||||
}
|
||||
|
||||
/// Make the next PUT persist its body remotely but lose its response.
|
||||
pub fn lose_next_put_response(&self) {
|
||||
self.inner.response_loss_after_put.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Override candidate probing for transition-recovery fail-closed tests.
|
||||
pub async fn set_transition_candidate_probe_override(&self, probe: Option<TransitionCandidateProbe>) {
|
||||
*self.inner.transition_candidate_probe_override.lock().await = probe;
|
||||
}
|
||||
|
||||
/// Reject non-empty remote versions before transition metadata is committed.
|
||||
pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
|
||||
self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Reject the next non-empty remote version validation without changing
|
||||
/// subsequent exact-version backend cleanup behavior.
|
||||
pub fn reject_next_non_empty_remote_version_validation(&self) {
|
||||
self.inner
|
||||
.reject_non_empty_remote_version_validations
|
||||
.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
/// Enable or disable a persistent remove failure for durability tests.
|
||||
pub fn set_remove_failure(&self, fail: bool) {
|
||||
self.inner.fail_remove.store(fail, Ordering::Release);
|
||||
@@ -444,6 +468,11 @@ impl MockWarmBackend {
|
||||
self.inner.remove_versions.lock().await.clone()
|
||||
}
|
||||
|
||||
/// Return the provider-authoritative view of a remote transition candidate.
|
||||
pub async fn probe_transition_candidate_state(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
self.probe_transition_candidate(object).await
|
||||
}
|
||||
|
||||
/// Number of `get` calls recorded — useful to assert restore reads hit the
|
||||
/// local copy rather than the remote tier.
|
||||
pub async fn get_count(&self) -> usize {
|
||||
@@ -575,12 +604,30 @@ impl MockWarmBackend {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn maybe_lose_put_response(&self) -> Result<(), std::io::Error> {
|
||||
if self.inner.response_loss_after_put.swap(false, Ordering::AcqRel) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::ConnectionReset,
|
||||
"mock warm backend lost PUT response after storing remote object",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl WarmBackend for MockWarmBackend {
|
||||
fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
|
||||
if self.inner.reject_non_empty_remote_versions.load(Ordering::Acquire) && !remote_version_id.is_empty() {
|
||||
if remote_version_id.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let reject_once = self
|
||||
.inner
|
||||
.reject_non_empty_remote_version_validations
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1))
|
||||
.is_ok();
|
||||
if reject_once || self.inner.reject_non_empty_remote_versions.load(Ordering::Acquire) {
|
||||
return Err(std::io::Error::other("mock warm backend requires an unversioned remote object"));
|
||||
}
|
||||
Ok(())
|
||||
@@ -599,6 +646,7 @@ impl WarmBackend for MockWarmBackend {
|
||||
object: object.to_string(),
|
||||
})
|
||||
.await;
|
||||
self.maybe_lose_put_response()?;
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
@@ -649,6 +697,7 @@ impl WarmBackend for MockWarmBackend {
|
||||
object: object.to_string(),
|
||||
})
|
||||
.await;
|
||||
self.maybe_lose_put_response()?;
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
@@ -725,6 +774,26 @@ impl WarmBackend for MockWarmBackend {
|
||||
self.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
self.precondition().await?;
|
||||
self.record(MockWarmOp::Probe {
|
||||
object: object.to_string(),
|
||||
})
|
||||
.await;
|
||||
if let Some(probe) = self.inner.transition_candidate_probe_override.lock().await.clone() {
|
||||
return Ok(probe);
|
||||
}
|
||||
let objects = self.inner.objects.lock().await;
|
||||
let Some(stored) = objects.get(object) else {
|
||||
return Ok(TransitionCandidateProbe::Missing);
|
||||
};
|
||||
if stored.remote_version_id.is_empty() {
|
||||
Ok(TransitionCandidateProbe::UnversionedPresent)
|
||||
} else {
|
||||
Ok(TransitionCandidateProbe::VersionedPresent(stored.remote_version_id.clone()))
|
||||
}
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.precondition().await?;
|
||||
self.record(MockWarmOp::InUse).await;
|
||||
@@ -909,3 +978,73 @@ pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, objec
|
||||
tokio::time::sleep(POLL_INTERVAL).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
|
||||
#[tokio::test]
|
||||
async fn mock_probe_distinguishes_missing_unversioned_and_versioned_candidates() {
|
||||
let backend = MockWarmBackend::new();
|
||||
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state("missing")
|
||||
.await
|
||||
.expect("probe missing candidate"),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
backend
|
||||
.put("unversioned", ReaderImpl::Body(Bytes::new()), 0)
|
||||
.await
|
||||
.expect("put unversioned candidate");
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state("unversioned")
|
||||
.await
|
||||
.expect("probe unversioned candidate"),
|
||||
TransitionCandidateProbe::UnversionedPresent
|
||||
);
|
||||
|
||||
let remote_version = Uuid::new_v4().to_string();
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
backend
|
||||
.put("versioned", ReaderImpl::Body(Bytes::new()), 0)
|
||||
.await
|
||||
.expect("put versioned candidate");
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state("versioned")
|
||||
.await
|
||||
.expect("probe versioned candidate"),
|
||||
TransitionCandidateProbe::VersionedPresent(remote_version)
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
backend
|
||||
.op_log()
|
||||
.await
|
||||
.into_iter()
|
||||
.filter(|op| matches!(op, MockWarmOp::Probe { .. }))
|
||||
.count(),
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mock_probe_preserves_fault_fail_closed_behavior() {
|
||||
let backend = MockWarmBackend::new();
|
||||
backend.set_reject_credentials(true).await;
|
||||
|
||||
let err = backend
|
||||
.probe_transition_candidate("remote-object")
|
||||
.await
|
||||
.expect_err("credential rejection must fail the authoritative probe");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::PermissionDenied);
|
||||
assert!(backend.op_log().await.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -21,14 +21,17 @@ use uuid::Uuid;
|
||||
use crate::config::com;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::services::tier::tier::TierDestinationId;
|
||||
use crate::storage_api_contracts::{list::ListOperations as _, object::HTTPPreconditions};
|
||||
use crate::store::ECStore;
|
||||
use crate::storage_api_contracts::{
|
||||
list::{ListOperations, StorageListObjectsV2Info},
|
||||
object::{EcstoreObjectIO, EcstoreObjectOperations, HTTPPreconditions},
|
||||
};
|
||||
|
||||
pub(crate) const TIER_MUTATION_INTENT_SCHEMA: &str = "rustfs-tier-mutation-intent-v1";
|
||||
pub(crate) const MAX_TIER_MUTATION_INTENT_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
|
||||
pub(crate) const TIER_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/records";
|
||||
pub(crate) const TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/coordinators";
|
||||
pub(crate) type TierMutationDigest = [u8; 32];
|
||||
|
||||
pub(crate) type Result<T> = std::result::Result<T, TierMutationIntentError>;
|
||||
@@ -293,21 +296,23 @@ impl TierMutationIntent {
|
||||
}
|
||||
|
||||
pub(crate) fn tier_mutation_intent_record_object_name(mutation_id: Uuid) -> Result<String> {
|
||||
tier_mutation_intent_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id)
|
||||
}
|
||||
|
||||
fn tier_mutation_intent_record_object_name_with_prefix(prefix: &str, mutation_id: Uuid) -> Result<String> {
|
||||
if mutation_id.is_nil() {
|
||||
return Err(TierMutationIntentError::Corrupt("mutation_id is nil"));
|
||||
}
|
||||
let mutation_key = mutation_id.simple().to_string();
|
||||
Ok(format!(
|
||||
"{}/{}/{}/{}.json",
|
||||
TIER_MUTATION_INTENT_RECORD_PREFIX,
|
||||
&mutation_key[..2],
|
||||
&mutation_key[2..4],
|
||||
mutation_key
|
||||
))
|
||||
Ok(format!("{}/{}/{}/{}.json", prefix, &mutation_key[..2], &mutation_key[2..4], mutation_key))
|
||||
}
|
||||
|
||||
pub(crate) fn tier_mutation_intent_id_from_record_object_name(object: &str) -> Result<Uuid> {
|
||||
let prefix = format!("{TIER_MUTATION_INTENT_RECORD_PREFIX}/");
|
||||
tier_mutation_intent_id_from_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, object)
|
||||
}
|
||||
|
||||
fn tier_mutation_intent_id_from_record_object_name_with_prefix(prefix: &str, object: &str) -> Result<Uuid> {
|
||||
let prefix = format!("{prefix}/");
|
||||
let suffix = object
|
||||
.strip_prefix(&prefix)
|
||||
.ok_or(TierMutationIntentError::Corrupt("intent record path has wrong prefix"))?;
|
||||
@@ -340,17 +345,43 @@ pub(crate) fn tier_mutation_intent_id_from_record_object_name(object: &str) -> R
|
||||
Uuid::parse_str(mutation_key).map_err(|_| TierMutationIntentError::Corrupt("intent record path has invalid uuid"))
|
||||
}
|
||||
|
||||
pub(crate) async fn save_tier_mutation_intent_record(api: Arc<ECStore>, intent: &TierMutationIntent) -> EcstoreResult<()> {
|
||||
let object = tier_mutation_intent_record_object_name(intent.mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
pub(crate) async fn save_tier_mutation_intent_record<S>(api: Arc<S>, intent: &TierMutationIntent) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let object = tier_mutation_intent_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, intent.mutation_id)
|
||||
.map_err(tier_mutation_intent_store_error)?;
|
||||
let data = intent.encode().map_err(tier_mutation_intent_store_error)?;
|
||||
com::save_config(api, &object, data).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_tier_mutation_intent_record_if_absent(
|
||||
api: Arc<ECStore>,
|
||||
pub(crate) async fn save_tier_coordinator_mutation_intent_record_if_absent<S>(
|
||||
api: Arc<S>,
|
||||
intent: &TierMutationIntent,
|
||||
) -> EcstoreResult<()> {
|
||||
let object = tier_mutation_intent_record_object_name(intent.mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
save_tier_mutation_intent_record_if_absent_with_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, intent).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_tier_mutation_intent_record_if_absent<S>(api: Arc<S>, intent: &TierMutationIntent) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
save_tier_mutation_intent_record_if_absent_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, intent).await
|
||||
}
|
||||
|
||||
async fn save_tier_mutation_intent_record_if_absent_with_prefix<S>(
|
||||
api: Arc<S>,
|
||||
prefix: &str,
|
||||
intent: &TierMutationIntent,
|
||||
) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let object = tier_mutation_intent_record_object_name_with_prefix(prefix, intent.mutation_id)
|
||||
.map_err(tier_mutation_intent_store_error)?;
|
||||
let data = intent.encode().map_err(tier_mutation_intent_store_error)?;
|
||||
com::save_config_with_opts(
|
||||
api,
|
||||
@@ -368,16 +399,34 @@ pub(crate) async fn save_tier_mutation_intent_record_if_absent(
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn load_tier_mutation_intent_record(api: Arc<ECStore>, mutation_id: Uuid) -> EcstoreResult<TierMutationIntent> {
|
||||
pub(crate) async fn load_tier_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<TierMutationIntent>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (intent, _) = load_tier_mutation_intent_record_with_etag(api, mutation_id).await?;
|
||||
Ok(intent)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_tier_mutation_intent_record_with_etag(
|
||||
api: Arc<ECStore>,
|
||||
pub(crate) async fn load_tier_mutation_intent_record_with_etag<S>(
|
||||
api: Arc<S>,
|
||||
mutation_id: Uuid,
|
||||
) -> EcstoreResult<(TierMutationIntent, String)> {
|
||||
let object = tier_mutation_intent_record_object_name(mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
) -> EcstoreResult<(TierMutationIntent, String)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
|
||||
}
|
||||
|
||||
async fn load_tier_mutation_intent_record_with_etag_at_prefix<S>(
|
||||
api: Arc<S>,
|
||||
prefix: &str,
|
||||
mutation_id: Uuid,
|
||||
) -> EcstoreResult<(TierMutationIntent, String)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let object =
|
||||
tier_mutation_intent_record_object_name_with_prefix(prefix, mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
let (data, object_info) = com::read_config_with_metadata(api, &object, &ObjectOptions::default()).await?;
|
||||
let etag = object_info
|
||||
.etag
|
||||
@@ -387,15 +436,34 @@ pub(crate) async fn load_tier_mutation_intent_record_with_etag(
|
||||
Ok((intent, etag))
|
||||
}
|
||||
|
||||
pub(crate) async fn save_tier_mutation_intent_record_if_current(
|
||||
api: Arc<ECStore>,
|
||||
pub(crate) async fn save_tier_mutation_intent_record_if_current<S>(
|
||||
api: Arc<S>,
|
||||
intent: &TierMutationIntent,
|
||||
current_etag: &str,
|
||||
) -> EcstoreResult<()> {
|
||||
) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
if current_etag.trim().is_empty() {
|
||||
return Err(Error::other("tier mutation intent current ETag is empty"));
|
||||
}
|
||||
let object = tier_mutation_intent_record_object_name(intent.mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
save_tier_mutation_intent_record_if_current_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, intent, current_etag).await
|
||||
}
|
||||
|
||||
async fn save_tier_mutation_intent_record_if_current_with_prefix<S>(
|
||||
api: Arc<S>,
|
||||
prefix: &str,
|
||||
intent: &TierMutationIntent,
|
||||
current_etag: &str,
|
||||
) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
if current_etag.trim().is_empty() {
|
||||
return Err(Error::other("tier mutation intent current ETag is empty"));
|
||||
}
|
||||
let object = tier_mutation_intent_record_object_name_with_prefix(prefix, intent.mutation_id)
|
||||
.map_err(tier_mutation_intent_store_error)?;
|
||||
let data = intent.encode().map_err(tier_mutation_intent_store_error)?;
|
||||
com::save_config_with_opts(
|
||||
api,
|
||||
@@ -413,35 +481,122 @@ pub(crate) async fn save_tier_mutation_intent_record_if_current(
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_tier_mutation_intent_record(api: Arc<ECStore>, mutation_id: Uuid) -> EcstoreResult<()> {
|
||||
let object = tier_mutation_intent_record_object_name(mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
pub(crate) async fn delete_tier_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectOperations,
|
||||
{
|
||||
delete_tier_mutation_intent_record_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_tier_coordinator_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectOperations,
|
||||
{
|
||||
delete_tier_mutation_intent_record_with_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
|
||||
}
|
||||
|
||||
async fn delete_tier_mutation_intent_record_with_prefix<S>(api: Arc<S>, prefix: &str, mutation_id: Uuid) -> EcstoreResult<()>
|
||||
where
|
||||
S: EcstoreObjectOperations,
|
||||
{
|
||||
let object =
|
||||
tier_mutation_intent_record_object_name_with_prefix(prefix, mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
match com::delete_config(api, &object).await {
|
||||
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn advance_tier_mutation_intent_record_idempotent(
|
||||
api: Arc<ECStore>,
|
||||
pub(crate) async fn advance_tier_mutation_intent_record_idempotent<S>(
|
||||
api: Arc<S>,
|
||||
mutation_id: Uuid,
|
||||
next: TierMutationIntentState,
|
||||
committed_config_etag: Option<String>,
|
||||
) -> EcstoreResult<(TierMutationIntent, bool)> {
|
||||
let (mut intent, current_etag) = load_tier_mutation_intent_record_with_etag(api.clone(), mutation_id).await?;
|
||||
) -> EcstoreResult<(TierMutationIntent, bool)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
advance_tier_mutation_intent_record_idempotent_at_prefix(
|
||||
api,
|
||||
TIER_MUTATION_INTENT_RECORD_PREFIX,
|
||||
mutation_id,
|
||||
next,
|
||||
committed_config_etag,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn advance_tier_coordinator_mutation_intent_record_idempotent<S>(
|
||||
api: Arc<S>,
|
||||
mutation_id: Uuid,
|
||||
next: TierMutationIntentState,
|
||||
committed_config_etag: Option<String>,
|
||||
) -> EcstoreResult<(TierMutationIntent, bool)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
advance_tier_mutation_intent_record_idempotent_at_prefix(
|
||||
api,
|
||||
TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX,
|
||||
mutation_id,
|
||||
next,
|
||||
committed_config_etag,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn advance_tier_mutation_intent_record_idempotent_at_prefix<S>(
|
||||
api: Arc<S>,
|
||||
prefix: &str,
|
||||
mutation_id: Uuid,
|
||||
next: TierMutationIntentState,
|
||||
committed_config_etag: Option<String>,
|
||||
) -> EcstoreResult<(TierMutationIntent, bool)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (mut intent, current_etag) =
|
||||
load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await?;
|
||||
let advanced = intent
|
||||
.advance_idempotent(next, committed_config_etag)
|
||||
.map_err(tier_mutation_intent_store_error)?;
|
||||
if advanced {
|
||||
save_tier_mutation_intent_record_if_current(api, &intent, ¤t_etag).await?;
|
||||
save_tier_mutation_intent_record_if_current_with_prefix(api, prefix, &intent, ¤t_etag).await?;
|
||||
}
|
||||
Ok((intent, advanced))
|
||||
}
|
||||
|
||||
pub(crate) async fn list_tier_mutation_intent_records(
|
||||
api: Arc<ECStore>,
|
||||
pub(crate) async fn list_tier_mutation_intent_records<S>(
|
||||
api: Arc<S>,
|
||||
limit: usize,
|
||||
marker: Option<String>,
|
||||
) -> EcstoreResult<TierMutationIntentRecordScan> {
|
||||
) -> EcstoreResult<TierMutationIntentRecordScan>
|
||||
where
|
||||
S: EcstoreObjectIO + ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
|
||||
{
|
||||
list_tier_mutation_intent_records_at_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, limit, marker).await
|
||||
}
|
||||
|
||||
pub(crate) async fn list_tier_coordinator_mutation_intent_records<S>(
|
||||
api: Arc<S>,
|
||||
limit: usize,
|
||||
marker: Option<String>,
|
||||
) -> EcstoreResult<TierMutationIntentRecordScan>
|
||||
where
|
||||
S: EcstoreObjectIO + ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
|
||||
{
|
||||
list_tier_mutation_intent_records_at_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, limit, marker).await
|
||||
}
|
||||
|
||||
async fn list_tier_mutation_intent_records_at_prefix<S>(
|
||||
api: Arc<S>,
|
||||
prefix: &str,
|
||||
limit: usize,
|
||||
marker: Option<String>,
|
||||
) -> EcstoreResult<TierMutationIntentRecordScan>
|
||||
where
|
||||
S: EcstoreObjectIO + ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
|
||||
{
|
||||
if limit == 0 {
|
||||
return Err(Error::other("tier mutation intent scan limit must be greater than zero"));
|
||||
}
|
||||
@@ -450,7 +605,7 @@ pub(crate) async fn list_tier_mutation_intent_records(
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
RUSTFS_META_BUCKET,
|
||||
TIER_MUTATION_INTENT_RECORD_PREFIX,
|
||||
prefix,
|
||||
marker,
|
||||
None,
|
||||
i32::try_from(limit).map_or(i32::MAX, |value| value),
|
||||
@@ -469,15 +624,15 @@ pub(crate) async fn list_tier_mutation_intent_records(
|
||||
|
||||
for object in list.objects {
|
||||
scan.scanned += 1;
|
||||
let mutation_id = match tier_mutation_intent_id_from_record_object_name(&object.name) {
|
||||
let mutation_id = match tier_mutation_intent_id_from_record_object_name_with_prefix(prefix, &object.name) {
|
||||
Ok(mutation_id) => mutation_id,
|
||||
Err(_) => {
|
||||
scan.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
|
||||
Ok(intent) => scan.intents.push(intent),
|
||||
match load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await {
|
||||
Ok((intent, _)) => scan.intents.push(intent),
|
||||
Err(Error::ConfigNotFound) => {}
|
||||
Err(_) => scan.failed += 1,
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ use std::sync::Arc;
|
||||
use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::tier::TierConfigMgr;
|
||||
use super::tier::{TierConfigMgr, tier_config_etag_matches};
|
||||
use super::tier_mutation_intent::{
|
||||
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
|
||||
load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
|
||||
@@ -140,13 +140,27 @@ async fn handle_commit(
|
||||
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
|
||||
let committed_config_etag = parse_commit_etag(canonical_payload)?;
|
||||
let tier_config_mgr = api.tier_config_mgr();
|
||||
let (intent, applied) = advance_tier_mutation_intent_record_idempotent(
|
||||
api,
|
||||
let (intent, applied) = match advance_tier_mutation_intent_record_idempotent(
|
||||
api.clone(),
|
||||
mutation_id,
|
||||
TierMutationIntentState::Committed,
|
||||
Some(committed_config_etag),
|
||||
Some(committed_config_etag.clone()),
|
||||
)
|
||||
.await?;
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(Error::ConfigNotFound)
|
||||
if tier_config_etag_matches(api, &committed_config_etag)
|
||||
.await
|
||||
.map_err(Error::other)? =>
|
||||
{
|
||||
return Ok(TierMutationPeerOutcome {
|
||||
state: TierMutationPeerState::Committed,
|
||||
applied: false,
|
||||
});
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
if intent.state == TierMutationIntentState::Committed {
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id).await;
|
||||
}
|
||||
|
||||
@@ -64,6 +64,15 @@ pub struct WarmBackendGetOpts {
|
||||
pub length: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum TransitionCandidateProbe {
|
||||
Missing,
|
||||
UnversionedPresent,
|
||||
VersionedPresent(String),
|
||||
Ambiguous,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait WarmBackend {
|
||||
async fn validate(&self) -> Result<(), std::io::Error> {
|
||||
@@ -101,6 +110,9 @@ pub trait WarmBackend {
|
||||
}
|
||||
self.remove(object, rv).await
|
||||
}
|
||||
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
Ok(TransitionCandidateProbe::Unsupported)
|
||||
}
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error>;
|
||||
}
|
||||
|
||||
@@ -615,6 +627,22 @@ mod tests {
|
||||
assert_eq!(removes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_transition_candidate_probe_is_unsupported() {
|
||||
let backend = RejectingValidationBackend {
|
||||
validations: Arc::new(AtomicUsize::new(0)),
|
||||
puts: Arc::new(AtomicUsize::new(0)),
|
||||
removes: Arc::new(AtomicUsize::new(0)),
|
||||
};
|
||||
|
||||
let probe = backend
|
||||
.probe_transition_candidate("remote-object")
|
||||
.await
|
||||
.expect("default candidate probe should be a safe capability response");
|
||||
|
||||
assert_eq!(probe, TransitionCandidateProbe::Unsupported);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() {
|
||||
let gets = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
@@ -29,7 +29,7 @@ use crate::client::{
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierMinIO,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use tracing::warn;
|
||||
@@ -126,6 +126,10 @@ impl WarmBackend for WarmBackendMinIO {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
self.0.probe_transition_candidate(object).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ use crate::client::{
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierR2,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use tracing::warn;
|
||||
@@ -126,6 +126,10 @@ impl WarmBackend for WarmBackendR2 {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
self.0.probe_transition_candidate(object).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ use crate::client::{
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierRustFS,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
|
||||
@@ -123,6 +123,10 @@ impl WarmBackend for WarmBackendRustFS {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
self.0.probe_transition_candidate(object).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
|
||||
@@ -24,8 +24,10 @@ use url::Url;
|
||||
|
||||
use crate::client::{
|
||||
api_get_options::GetObjectOptions,
|
||||
api_list::ListObjectsOptions,
|
||||
api_put_object::PutObjectOptions,
|
||||
api_remove::{RemoveObjectOptions, RemoveObjectResult},
|
||||
api_s3_datatypes::ListVersionsResult,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
@@ -34,11 +36,12 @@ use crate::error::ErrorResponse;
|
||||
use crate::error::error_resp_to_object_err;
|
||||
use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use s3s::dto::BucketVersioningStatus;
|
||||
|
||||
pub struct WarmBackendS3 {
|
||||
pub client: Arc<TransitionClient>,
|
||||
@@ -48,6 +51,27 @@ pub struct WarmBackendS3 {
|
||||
pub storage_class: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum RemoteBucketVersioning {
|
||||
Disabled,
|
||||
Suspended,
|
||||
Enabled,
|
||||
}
|
||||
|
||||
fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result<RemoteBucketVersioning, std::io::Error> {
|
||||
Ok(match status {
|
||||
Some(BucketVersioningStatus::ENABLED) => RemoteBucketVersioning::Enabled,
|
||||
Some(BucketVersioningStatus::SUSPENDED) => RemoteBucketVersioning::Suspended,
|
||||
Some(status) => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
format!("remote tier bucket returned unsupported versioning status {status}"),
|
||||
));
|
||||
}
|
||||
None => RemoteBucketVersioning::Disabled,
|
||||
})
|
||||
}
|
||||
|
||||
impl WarmBackendS3 {
|
||||
pub async fn new(conf: &TierS3, _tier: &str) -> Result<Self, std::io::Error> {
|
||||
Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await
|
||||
@@ -159,11 +183,57 @@ impl WarmBackendS3 {
|
||||
let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
|
||||
Ok((headers, reader))
|
||||
}
|
||||
|
||||
async fn remote_bucket_versioning(&self) -> Result<RemoteBucketVersioning, std::io::Error> {
|
||||
let config = self.client.get_bucket_versioning(&self.bucket).await?;
|
||||
remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
|
||||
}
|
||||
|
||||
async fn list_transition_candidate_versions(&self, object: &str) -> Result<ListVersionsResult, std::io::Error> {
|
||||
let mut opts = ListObjectsOptions::default();
|
||||
opts.set("prefix", &self.get_dest(object));
|
||||
opts.set("max-keys", "2");
|
||||
self.client.list_object_versions_query(&self.bucket, &opts, "", "", "").await
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_transition_candidate_versions(
|
||||
remote_object: &str,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
versions: &ListVersionsResult,
|
||||
) -> TransitionCandidateProbe {
|
||||
if versions.is_truncated {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
if versions.delete_markers.iter().any(|marker| marker.key == remote_object) {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
let mut exact_versions = versions.versions.iter().filter(|version| version.key == remote_object);
|
||||
let Some(version) = exact_versions.next() else {
|
||||
return TransitionCandidateProbe::Missing;
|
||||
};
|
||||
if exact_versions.next().is_some() {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
match bucket_versioning {
|
||||
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
|
||||
RemoteBucketVersioning::Suspended if version.version_id == "null" => {
|
||||
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version.version_id.is_empty() => {
|
||||
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::client::api_s3_datatypes::{ListVersionsResult, Version};
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
||||
@@ -181,6 +251,116 @@ mod tests {
|
||||
Err(err) => assert!(err.to_string().contains("not allowed")),
|
||||
}
|
||||
}
|
||||
|
||||
fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
|
||||
ListVersionsResult {
|
||||
versions: versions
|
||||
.iter()
|
||||
.map(|(key, version_id)| Version {
|
||||
key: (*key).to_string(),
|
||||
version_id: (*version_id).to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
delete_markers: delete_markers
|
||||
.iter()
|
||||
.map(|(key, version_id)| Version {
|
||||
key: (*key).to_string(),
|
||||
version_id: (*version_id).to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
is_truncated,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_classifier_is_fail_closed() {
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Disabled,
|
||||
&list_versions(&[], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Disabled,
|
||||
&list_versions(&[("archive/object", "")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::UnversionedPresent
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Suspended,
|
||||
&list_versions(&[("archive/object", "null")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("null".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a"), ("archive/object", "version-b")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[("archive/object", "marker-a")], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[], true),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_bucket_versioning_status_parser_fails_closed() {
|
||||
assert_eq!(
|
||||
remote_bucket_versioning_from_status(None).expect("absent status means disabled"),
|
||||
RemoteBucketVersioning::Disabled
|
||||
);
|
||||
assert_eq!(
|
||||
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::ENABLED)).expect("enabled status should parse"),
|
||||
RemoteBucketVersioning::Enabled
|
||||
);
|
||||
assert_eq!(
|
||||
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::SUSPENDED)).expect("suspended status should parse"),
|
||||
RemoteBucketVersioning::Suspended
|
||||
);
|
||||
let err = remote_bucket_versioning_from_status(Some("UnexpectedStatus"))
|
||||
.expect_err("unknown versioning status must fail closed");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -215,6 +395,16 @@ impl WarmBackend for WarmBackendS3 {
|
||||
self.remove_with_result(object, rv).await.map(|_| ())
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
||||
let versions = self.list_transition_candidate_versions(object).await?;
|
||||
Ok(classify_transition_candidate_versions(
|
||||
&self.get_dest(object),
|
||||
bucket_versioning,
|
||||
&versions,
|
||||
))
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
let result = self
|
||||
.core
|
||||
|
||||
@@ -3346,6 +3346,13 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
};
|
||||
|
||||
let mut upload_cleanup = TransitionUploadCleanup::new(tgt_client, &dest_obj, self.ctx.clone());
|
||||
advance_and_save_transition_transaction(
|
||||
transaction_api.as_ref(),
|
||||
&mut transaction,
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let remote_upload = {
|
||||
let lease = &upload_cleanup.lease;
|
||||
let recorded_candidate = &mut upload_cleanup.candidate;
|
||||
@@ -5906,7 +5913,6 @@ mod transition_upload_integrity_tests {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
|
||||
for position in [
|
||||
ShardCorruptionPosition::First,
|
||||
@@ -6078,7 +6084,7 @@ mod transition_upload_integrity_tests {
|
||||
let remote_version = Uuid::new_v4().to_string();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
backend.reject_next_non_empty_remote_version_validation();
|
||||
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
@@ -6106,7 +6112,7 @@ mod transition_upload_integrity_tests {
|
||||
let remote_version = Uuid::nil().to_string();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
backend.reject_next_non_empty_remote_version_validation();
|
||||
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
|
||||
@@ -537,7 +537,8 @@ mod tests {
|
||||
transition_transaction::{
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
|
||||
TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
|
||||
TransitionTransactionState, recover_transition_transaction_records, save_transition_transaction_record,
|
||||
TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
|
||||
save_transition_transaction_record,
|
||||
},
|
||||
},
|
||||
client::transition_api::ReaderImpl,
|
||||
@@ -545,8 +546,8 @@ mod tests {
|
||||
disk::RUSTFS_META_BUCKET,
|
||||
runtime::{global::set_object_store_resolver, sources as runtime_sources},
|
||||
services::tier::{
|
||||
test_util::{MockWarmBackend, TransitionCleanupStoreBarrier, register_mock_tier},
|
||||
tier::TierConfigMgr,
|
||||
test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
|
||||
tier::{TIER_CONFIG_FILE, TierConfigMgr},
|
||||
tier_mutation_intent::{
|
||||
TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
|
||||
TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
|
||||
@@ -554,7 +555,7 @@ mod tests {
|
||||
save_tier_mutation_intent_record, save_tier_mutation_intent_record_if_current,
|
||||
},
|
||||
tier_mutation_peer::{TierMutationPeerError, TierMutationPeerState, handle_tier_mutation_peer_request},
|
||||
warm_backend::WarmBackend,
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend},
|
||||
},
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, MakeBucketOptions},
|
||||
@@ -1589,6 +1590,47 @@ mod tests {
|
||||
assert_eq!(loaded.state, TierMutationIntentState::Committed);
|
||||
assert_eq!(loaded.committed_config_etag.as_deref(), Some("new-etag"));
|
||||
|
||||
store
|
||||
.tier_config_mgr()
|
||||
.read()
|
||||
.await
|
||||
.save_tiering_config(store.clone())
|
||||
.await
|
||||
.expect("tier config should persist for cleaned intent commit proof");
|
||||
let tier_config_info = store
|
||||
.get_object_info(
|
||||
RUSTFS_META_BUCKET,
|
||||
&format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("tier config object info should load");
|
||||
let tier_config_etag = tier_config_info.etag.expect("tier config should carry an ETag");
|
||||
delete_tier_mutation_intent_record(store.clone(), mutation_id)
|
||||
.await
|
||||
.expect("committed peer intent cleanup should persist");
|
||||
let cleaned_commit_retry = handle_tier_mutation_peer_request(
|
||||
store.clone(),
|
||||
TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Commit,
|
||||
mutation_id,
|
||||
tier_config_etag.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("commit retry after durable cleanup should be idempotently terminal");
|
||||
assert!(!cleaned_commit_retry.applied);
|
||||
assert_eq!(cleaned_commit_retry.state, TierMutationPeerState::Committed);
|
||||
let mismatched_cleaned_commit = handle_tier_mutation_peer_request(
|
||||
store.clone(),
|
||||
TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Commit,
|
||||
mutation_id,
|
||||
b"not-the-current-etag",
|
||||
)
|
||||
.await
|
||||
.expect_err("missing intent without a matching committed config ETag must fail closed");
|
||||
assert!(matches!(mismatched_cleaned_commit, TierMutationPeerError::Store(Error::ConfigNotFound)));
|
||||
|
||||
let abort_id = uuid::Uuid::new_v4();
|
||||
let abort_intent = tier_mutation_peer_test_intent(abort_id, "COLD-B", [4; 32]);
|
||||
let abort_prepare_payload = abort_intent.encode().expect("abort prepare intent should encode");
|
||||
@@ -1808,7 +1850,7 @@ mod tests {
|
||||
let tier_name = "CROSSCTXA";
|
||||
let backend = register_mock_tier(&ctx_a.tier_config_mgr(), tier_name).await;
|
||||
backend.set_put_remote_version(Some(uuid::Uuid::new_v4().to_string())).await;
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
backend.reject_next_non_empty_remote_version_validation();
|
||||
let remove_barrier = backend.arm_failing_remove_barrier().await;
|
||||
|
||||
let bucket = "transition-cleanup-context-a";
|
||||
@@ -2452,4 +2494,307 @@ mod tests {
|
||||
);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn transition_transaction_recovery_retains_unproven_remote_candidates() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-unproven", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXUNPROVEN";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
.backend_identity();
|
||||
let remote_version = uuid::Uuid::new_v4().to_string();
|
||||
let new_transaction = || {
|
||||
TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "absent-source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Unversioned,
|
||||
},
|
||||
tier_name: tier_name.to_string(),
|
||||
backend_fingerprint: backend_identity,
|
||||
not_after_unix_nanos: 1_780_000_000_000_000_000,
|
||||
})
|
||||
.expect("transaction should build")
|
||||
};
|
||||
|
||||
let upload_started = new_transaction();
|
||||
let mut local_commit_started = new_transaction();
|
||||
local_commit_started
|
||||
.advance(
|
||||
local_commit_started.fence(),
|
||||
TransitionTransactionState::Uploaded,
|
||||
Some(TransitionRemoteVersion::versioned(remote_version.clone())),
|
||||
)
|
||||
.expect("transaction should enter uploaded state");
|
||||
local_commit_started
|
||||
.advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None)
|
||||
.expect("transaction should enter local commit state");
|
||||
|
||||
backend.set_put_remote_version(Some(remote_version)).await;
|
||||
for transaction in [&upload_started, &local_commit_started] {
|
||||
let candidate = bytes::Bytes::from_static(b"unproven transition remote candidate");
|
||||
backend
|
||||
.put(
|
||||
&transaction.remote_object,
|
||||
ReaderImpl::Body(candidate.clone()),
|
||||
i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
|
||||
)
|
||||
.await
|
||||
.expect("mock backend should accept candidate");
|
||||
save_transition_transaction_record(store.clone(), transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
}
|
||||
|
||||
let stats = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("transition transaction recovery should run");
|
||||
|
||||
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (2, 0, 2, 0));
|
||||
assert_eq!(
|
||||
transition_transaction_record_count(store.clone()).await,
|
||||
2,
|
||||
"an upload without completion proof or unproven local commit must remain for authoritative reconcile"
|
||||
);
|
||||
assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate");
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert_eq!(backend.exact_remove_count(), 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn transition_transaction_recovery_deletes_provider_recovered_unknown_upload() {
|
||||
let versioned_remote = uuid::Uuid::new_v4().to_string();
|
||||
for (case, tier_name, remote_version) in [
|
||||
("missing", "TXPROBEMISSING", None),
|
||||
("unversioned", "TXPROBEUNVERSIONED", Some(String::new())),
|
||||
("versioned", "TXPROBEVERSIONED", Some(versioned_remote)),
|
||||
] {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
|
||||
temp_dir.path(),
|
||||
&format!("transition-transaction-probe-{case}"),
|
||||
&[4],
|
||||
))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
.backend_identity();
|
||||
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Unversioned,
|
||||
},
|
||||
tier_name: tier_name.to_string(),
|
||||
backend_fingerprint: backend_identity,
|
||||
not_after_unix_nanos: 1_780_000_000_000_000_000,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("transaction should enter unknown upload outcome state");
|
||||
|
||||
if let Some(version) = &remote_version {
|
||||
backend.set_put_remote_version(Some(version.clone())).await;
|
||||
let candidate = bytes::Bytes::from_static(b"provider-recovered transition remote candidate");
|
||||
backend
|
||||
.put(
|
||||
&transaction.remote_object,
|
||||
ReaderImpl::Body(candidate.clone()),
|
||||
i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
|
||||
)
|
||||
.await
|
||||
.expect("mock backend should accept candidate");
|
||||
}
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
|
||||
let stats = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("transition transaction recovery should run");
|
||||
|
||||
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
|
||||
assert_eq!(
|
||||
backend.object_count().await,
|
||||
0,
|
||||
"case {case}: recovered unknown upload candidate must be absent"
|
||||
);
|
||||
let removed = remote_version
|
||||
.map(|version| vec![(transaction.remote_object.clone(), version)])
|
||||
.unwrap_or_default();
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
removed,
|
||||
"case {case}: recovery must delete only provider-recovered candidates"
|
||||
);
|
||||
assert_eq!(
|
||||
backend.exact_remove_count(),
|
||||
usize::from(removed.first().is_some_and(|(_, version)| !version.is_empty()))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn transition_response_loss_persists_unknown_outcome_for_provider_recovery() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-response-loss", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXRESPONSELOSS";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let bucket = "transition-response-loss-bucket";
|
||||
let object = "source.bin";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("source bucket should be created");
|
||||
let payload = b"a response-lost tier PUT must remain recoverable".repeat(1024);
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let source = store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
backend.lose_next_put_response();
|
||||
|
||||
let error = store
|
||||
.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: source.etag.clone().expect("source object should have an ETag"),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: source.version_id.map(|version| version.to_string()),
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("a lost tier PUT response must fail the transition request");
|
||||
assert!(
|
||||
matches!(error, StorageError::Io(ref err) if err.kind() == std::io::ErrorKind::ConnectionReset),
|
||||
"the response-loss error must remain visible to the caller: {error:?}"
|
||||
);
|
||||
|
||||
let records = store
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
RUSTFS_META_BUCKET,
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX,
|
||||
None,
|
||||
None,
|
||||
10,
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("transition transaction records should be listable");
|
||||
assert_eq!(records.objects.len(), 1, "response loss must leave one durable transaction record");
|
||||
let transaction_id = records.objects[0]
|
||||
.name
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.and_then(|name| name.strip_suffix(".json"))
|
||||
.and_then(|name| uuid::Uuid::parse_str(name).ok())
|
||||
.expect("transaction record name should contain a UUID");
|
||||
let transaction = load_transition_transaction_record(store.clone(), transaction_id)
|
||||
.await
|
||||
.expect("response loss transaction record should load");
|
||||
assert_eq!(
|
||||
transaction.state,
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
"a response-lost PUT must not remain in UploadStarted"
|
||||
);
|
||||
assert!(
|
||||
backend.contains(&transaction.remote_object).await,
|
||||
"the test backend must retain the remote candidate"
|
||||
);
|
||||
|
||||
backend
|
||||
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
|
||||
.await;
|
||||
let unsupported_stats = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("unsupported provider recovery should fail closed");
|
||||
assert_eq!(
|
||||
(
|
||||
unsupported_stats.scanned,
|
||||
unsupported_stats.recovered,
|
||||
unsupported_stats.retained,
|
||||
unsupported_stats.failed
|
||||
),
|
||||
(1, 0, 1, 0),
|
||||
"an unsupported provider probe must retain the unknown upload"
|
||||
);
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
|
||||
assert!(
|
||||
backend.contains(&transaction.remote_object).await,
|
||||
"unsupported recovery must not delete the candidate"
|
||||
);
|
||||
assert_eq!(backend.remove_count().await, 0, "unsupported recovery must not attempt cleanup");
|
||||
|
||||
backend.set_transition_candidate_probe_override(None).await;
|
||||
let stats = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("provider-authoritative recovery should run");
|
||||
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
|
||||
assert_eq!(backend.object_count().await, 0, "recovery must delete the provider-confirmed candidate");
|
||||
let op_log = backend.op_log().await;
|
||||
assert!(
|
||||
op_log.iter().any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
|
||||
"response-loss recovery must enter the provider probe branch"
|
||||
);
|
||||
assert!(
|
||||
op_log.iter().any(|operation| matches!(operation, MockWarmOp::Put { .. })),
|
||||
"response-loss fixture must record that the remote PUT reached the backend"
|
||||
);
|
||||
let source_after = store
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("recovery must preserve the local source object");
|
||||
assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -872,6 +872,8 @@ impl ECStore {
|
||||
let idx = if opts.data_movement && opts.version_id.is_some() {
|
||||
self.select_data_movement_pool_idx(bucket, &object, data.size(), opts, false)
|
||||
.await?
|
||||
} else if opts.no_lock {
|
||||
self.get_pool_idx_no_lock(bucket, &object, data.size()).await?
|
||||
} else {
|
||||
self.get_pool_idx(bucket, &object, data.size()).await?
|
||||
};
|
||||
|
||||
+51
-15
@@ -172,21 +172,31 @@ pub fn oidc_plugin_authn_metrics_snapshot() -> OidcPluginAuthnMetricsSnapshot {
|
||||
OIDC_PLUGIN_AUTHN_METRICS.snapshot()
|
||||
}
|
||||
|
||||
/// Header names whose values may carry OIDC secrets (client credentials, cookies,
|
||||
/// bearer tokens). Their values are never emitted to logs, only their byte length.
|
||||
const SENSITIVE_HEADER_NAMES: [&str; 4] = ["authorization", "proxy-authorization", "cookie", "set-cookie"];
|
||||
|
||||
fn is_sensitive_header(name: &str) -> bool {
|
||||
SENSITIVE_HEADER_NAMES
|
||||
.iter()
|
||||
.any(|candidate| name.eq_ignore_ascii_case(candidate))
|
||||
}
|
||||
|
||||
fn format_http_headers(headers: &http::HeaderMap) -> String {
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
let value = value.to_str().unwrap_or("<non-utf8>");
|
||||
format!("{}={}", name.as_str(), value)
|
||||
if is_sensitive_header(name.as_str()) {
|
||||
format!("{}=<redacted len={}>", name.as_str(), value.as_bytes().len())
|
||||
} else {
|
||||
let value = value.to_str().unwrap_or("<non-utf8>");
|
||||
format!("{}={}", name.as_str(), value)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("; ")
|
||||
}
|
||||
|
||||
fn format_http_body(body: &[u8]) -> String {
|
||||
String::from_utf8_lossy(body).into_owned()
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct TokenResponseBodyShape {
|
||||
json_object: bool,
|
||||
@@ -326,7 +336,6 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
|
||||
let uri = parts.uri.to_string();
|
||||
if tracing::enabled!(tracing::Level::DEBUG) {
|
||||
let request_headers = format_http_headers(&parts.headers);
|
||||
let request_body = format_http_body(&body);
|
||||
debug!(
|
||||
event = EVENT_OIDC_HTTP,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
@@ -336,7 +345,6 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
|
||||
uri = %uri,
|
||||
request_headers = %request_headers,
|
||||
request_body_len = body.len(),
|
||||
request_body = %request_body,
|
||||
"oidc outbound http"
|
||||
);
|
||||
}
|
||||
@@ -387,7 +395,6 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
|
||||
})?;
|
||||
if tracing::enabled!(tracing::Level::DEBUG) {
|
||||
let response_headers = format_http_headers(&headers);
|
||||
let response_body = format_http_body(&body_bytes);
|
||||
debug!(
|
||||
event = EVENT_OIDC_HTTP,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
@@ -400,7 +407,6 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
|
||||
elapsed_ms,
|
||||
response_headers = %response_headers,
|
||||
response_body_len = body_bytes.len(),
|
||||
response_body = %response_body,
|
||||
"oidc outbound http"
|
||||
);
|
||||
}
|
||||
@@ -814,7 +820,6 @@ impl OidcSys {
|
||||
}
|
||||
RequestTokenError::Parse(parse_err, body) => {
|
||||
let shape = inspect_token_response_body(body);
|
||||
let response_body = format_http_body(body);
|
||||
error!(
|
||||
event = EVENT_OIDC_DIAGNOSTICS,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
@@ -837,12 +842,11 @@ impl OidcSys {
|
||||
response_has_error = shape.has_error,
|
||||
response_has_error_description = shape.has_error_description,
|
||||
response_looks_like_html = shape.looks_like_html,
|
||||
response_body = %response_body,
|
||||
error = %e,
|
||||
"oidc token exchange failed"
|
||||
);
|
||||
format!(
|
||||
"token exchange failed: {e}: stage=token_response_parse_failed, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, parse_error_path={}, response_body_len={}, response_json_keys={}, response_has_id_token={}, response_has_error={}, response_looks_like_html={}, response_body={}",
|
||||
"token exchange failed: {e}: stage=token_response_parse_failed, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, parse_error_path={}, response_body_len={}, response_json_keys={}, response_has_id_token={}, response_has_error={}, response_looks_like_html={}",
|
||||
session.provider_id,
|
||||
config.config_url,
|
||||
issuer,
|
||||
@@ -854,8 +858,7 @@ impl OidcSys {
|
||||
shape.json_keys,
|
||||
shape.has_id_token,
|
||||
shape.has_error,
|
||||
shape.looks_like_html,
|
||||
response_body
|
||||
shape.looks_like_html
|
||||
)
|
||||
}
|
||||
RequestTokenError::Other(message) => {
|
||||
@@ -1849,6 +1852,39 @@ mod tests {
|
||||
assert_eq!(extract_string_claim(&claims, "missing"), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_http_headers_redacts_sensitive_values() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert(http::header::AUTHORIZATION, "Basic Y2xpZW50OnNlY3JldA==".parse().unwrap());
|
||||
headers.insert(http::header::CONTENT_TYPE, "application/json".parse().unwrap());
|
||||
headers.insert(http::header::COOKIE, "session=super-secret".parse().unwrap());
|
||||
|
||||
let rendered = format_http_headers(&headers);
|
||||
|
||||
// Sensitive header values never appear; only their length is emitted.
|
||||
assert!(!rendered.contains("Y2xpZW50OnNlY3JldA=="), "authorization value leaked: {rendered}");
|
||||
assert!(!rendered.contains("super-secret"), "cookie value leaked: {rendered}");
|
||||
assert!(
|
||||
rendered.contains("authorization=<redacted len="),
|
||||
"expected redacted authorization: {rendered}"
|
||||
);
|
||||
assert!(rendered.contains("cookie=<redacted len="), "expected redacted cookie: {rendered}");
|
||||
// Non-sensitive header values are preserved for diagnostics.
|
||||
assert!(
|
||||
rendered.contains("content-type=application/json"),
|
||||
"content-type should be visible: {rendered}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_sensitive_header_is_case_insensitive() {
|
||||
assert!(is_sensitive_header("Authorization"));
|
||||
assert!(is_sensitive_header("PROXY-AUTHORIZATION"));
|
||||
assert!(is_sensitive_header("Set-Cookie"));
|
||||
assert!(!is_sensitive_header("content-type"));
|
||||
assert!(!is_sensitive_header("x-request-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_groups_claim_array() {
|
||||
let mut claims = HashMap::new();
|
||||
|
||||
@@ -1156,10 +1156,50 @@ pub struct SRStateEditReq {
|
||||
pub struct ResyncBucketStatus {
|
||||
#[serde(default)]
|
||||
pub bucket: String,
|
||||
#[serde(rename = "targetArn", default, skip_serializing_if = "String::is_empty")]
|
||||
pub target_arn: String,
|
||||
#[serde(default)]
|
||||
pub status: String,
|
||||
#[serde(rename = "errorDetail", skip_serializing_if = "String::is_empty", default)]
|
||||
pub err_detail: String,
|
||||
#[serde(
|
||||
rename = "createdAt",
|
||||
default,
|
||||
with = "time::serde::rfc3339::option",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub created_at: Option<OffsetDateTime>,
|
||||
#[serde(
|
||||
rename = "startedAt",
|
||||
default,
|
||||
with = "time::serde::rfc3339::option",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub started_at: Option<OffsetDateTime>,
|
||||
#[serde(
|
||||
rename = "updatedAt",
|
||||
default,
|
||||
with = "time::serde::rfc3339::option",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub updated_at: Option<OffsetDateTime>,
|
||||
#[serde(
|
||||
rename = "completedAt",
|
||||
default,
|
||||
with = "time::serde::rfc3339::option",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub completed_at: Option<OffsetDateTime>,
|
||||
#[serde(default, skip_serializing_if = "is_zero_u64")]
|
||||
pub generation: u64,
|
||||
#[serde(rename = "replicatedObjects", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub replicated_objects: u64,
|
||||
#[serde(rename = "replicatedBytes", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub replicated_bytes: u64,
|
||||
#[serde(rename = "failedObjects", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub failed_objects: u64,
|
||||
#[serde(rename = "failedBytes", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub failed_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
@@ -1170,10 +1210,74 @@ pub struct SRResyncOpStatus {
|
||||
pub resync_id: String,
|
||||
#[serde(default)]
|
||||
pub status: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub state: String,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub buckets: Vec<ResyncBucketStatus>,
|
||||
#[serde(rename = "errorDetail", skip_serializing_if = "String::is_empty", default)]
|
||||
pub err_detail: String,
|
||||
#[serde(
|
||||
rename = "createdAt",
|
||||
default,
|
||||
with = "time::serde::rfc3339::option",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub created_at: Option<OffsetDateTime>,
|
||||
#[serde(
|
||||
rename = "startedAt",
|
||||
default,
|
||||
with = "time::serde::rfc3339::option",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub started_at: Option<OffsetDateTime>,
|
||||
#[serde(
|
||||
rename = "updatedAt",
|
||||
default,
|
||||
with = "time::serde::rfc3339::option",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub updated_at: Option<OffsetDateTime>,
|
||||
#[serde(
|
||||
rename = "completedAt",
|
||||
default,
|
||||
with = "time::serde::rfc3339::option",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub completed_at: Option<OffsetDateTime>,
|
||||
#[serde(default, skip_serializing_if = "is_zero_u64")]
|
||||
pub generation: u64,
|
||||
#[serde(rename = "totalBuckets", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub total_buckets: u64,
|
||||
#[serde(rename = "pendingBuckets", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub pending_buckets: u64,
|
||||
#[serde(rename = "runningBuckets", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub running_buckets: u64,
|
||||
#[serde(rename = "completedBuckets", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub completed_buckets: u64,
|
||||
#[serde(rename = "failedBuckets", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub failed_buckets: u64,
|
||||
#[serde(rename = "canceledBuckets", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub canceled_buckets: u64,
|
||||
#[serde(rename = "replicatedObjects", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub replicated_objects: u64,
|
||||
#[serde(rename = "replicatedBytes", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub replicated_bytes: u64,
|
||||
#[serde(rename = "failedObjects", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub failed_objects: u64,
|
||||
#[serde(rename = "failedBytes", default, skip_serializing_if = "is_zero_u64")]
|
||||
pub failed_bytes: u64,
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub truncated: bool,
|
||||
#[serde(rename = "nextContinuationToken", default, skip_serializing_if = "String::is_empty")]
|
||||
pub next_continuation_token: String,
|
||||
}
|
||||
|
||||
fn is_zero_u64(value: &u64) -> bool {
|
||||
*value == 0
|
||||
}
|
||||
|
||||
fn is_false(value: &bool) -> bool {
|
||||
!*value
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
@@ -1202,7 +1306,7 @@ pub struct SiteNetPerfResult {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{PeerInfo, PeerSite};
|
||||
use super::{PeerInfo, PeerSite, SRResyncOpStatus};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const TEST_CA_CERT: &str = "-----BEGIN CERTIFICATE-----\ntest-ca\n-----END CERTIFICATE-----";
|
||||
@@ -1302,4 +1406,94 @@ mod tests {
|
||||
assert!(peer_debug.contains("skip_tls_verify: false"));
|
||||
assert!(peer_debug.contains("has_custom_ca: true"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resync_status_legacy_json_defaults_new_lifecycle_fields() {
|
||||
let legacy_json = json!({
|
||||
"op": "start",
|
||||
"id": "resync-1",
|
||||
"status": "success",
|
||||
"buckets": [{
|
||||
"bucket": "photos",
|
||||
"status": "success"
|
||||
}]
|
||||
});
|
||||
|
||||
let status: SRResyncOpStatus =
|
||||
serde_json::from_value(legacy_json.clone()).expect("legacy resync status should deserialize");
|
||||
|
||||
assert_eq!(status.generation, 0);
|
||||
assert!(status.state.is_empty());
|
||||
assert!(status.created_at.is_none());
|
||||
assert_eq!(status.total_buckets, 0);
|
||||
assert_eq!(status.replicated_objects, 0);
|
||||
assert!(!status.truncated);
|
||||
assert!(status.next_continuation_token.is_empty());
|
||||
assert!(status.buckets[0].created_at.is_none());
|
||||
assert!(status.buckets[0].target_arn.is_empty());
|
||||
assert_eq!(status.buckets[0].generation, 0);
|
||||
assert_eq!(status.buckets[0].replicated_bytes, 0);
|
||||
assert_eq!(serde_json::to_value(status).expect("legacy resync status should serialize"), legacy_json);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resync_status_lifecycle_fields_round_trip_with_exact_json_names() {
|
||||
let status_json = json!({
|
||||
"op": "status",
|
||||
"id": "resync-2",
|
||||
"status": "success",
|
||||
"state": "running",
|
||||
"createdAt": "2026-07-22T01:00:00Z",
|
||||
"startedAt": "2026-07-22T01:00:01Z",
|
||||
"updatedAt": "2026-07-22T01:01:00Z",
|
||||
"completedAt": "2026-07-22T01:02:00Z",
|
||||
"generation": 7,
|
||||
"totalBuckets": 6,
|
||||
"pendingBuckets": 1,
|
||||
"runningBuckets": 1,
|
||||
"completedBuckets": 1,
|
||||
"failedBuckets": 1,
|
||||
"canceledBuckets": 2,
|
||||
"replicatedObjects": 12,
|
||||
"replicatedBytes": 4096,
|
||||
"failedObjects": 3,
|
||||
"failedBytes": 512,
|
||||
"truncated": true,
|
||||
"nextContinuationToken": "bucket-page-2",
|
||||
"buckets": [{
|
||||
"bucket": "photos",
|
||||
"targetArn": "arn:rustfs:replication::peer-a:photos",
|
||||
"status": "failed",
|
||||
"errorDetail": "target unavailable",
|
||||
"createdAt": "2026-07-22T01:00:00Z",
|
||||
"startedAt": "2026-07-22T01:00:01Z",
|
||||
"updatedAt": "2026-07-22T01:01:00Z",
|
||||
"completedAt": "2026-07-22T01:02:00Z",
|
||||
"generation": 7,
|
||||
"replicatedObjects": 12,
|
||||
"replicatedBytes": 4096,
|
||||
"failedObjects": 3,
|
||||
"failedBytes": 512
|
||||
}]
|
||||
});
|
||||
|
||||
let status: SRResyncOpStatus =
|
||||
serde_json::from_value(status_json.clone()).expect("expanded resync status should deserialize");
|
||||
|
||||
assert_eq!(status.generation, 7);
|
||||
assert_eq!(status.state, "running");
|
||||
assert_eq!(status.total_buckets, 6);
|
||||
assert_eq!(status.completed_buckets, 1);
|
||||
assert_eq!(status.replicated_bytes, 4096);
|
||||
assert!(status.truncated);
|
||||
assert_eq!(status.next_continuation_token, "bucket-page-2");
|
||||
assert_eq!(status.buckets[0].generation, 7);
|
||||
assert_eq!(status.buckets[0].target_arn, "arn:rustfs:replication::peer-a:photos");
|
||||
assert_eq!(status.buckets[0].failed_objects, 3);
|
||||
assert!(status.buckets[0].completed_at.is_some());
|
||||
assert_eq!(
|
||||
serde_json::to_value(status).expect("expanded resync status should serialize"),
|
||||
status_json
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ serde = { workspace = true, features = ["derive"] }
|
||||
starshard = { workspace = true, features = ["rayon", "async", "serde"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "fs"] }
|
||||
tokio-util = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
url = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
@@ -62,6 +63,7 @@ axum = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["path"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
tempfile = { workspace = true }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
+141
-244
@@ -13,89 +13,83 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{
|
||||
Event, NotificationError, registry::TargetRegistry, resolve_notify_object_store_handle, rule_engine::NotifyRuleEngine,
|
||||
NotificationError,
|
||||
lifecycle::{NotificationRuntimeState, NotifyLifecycleCoordinator},
|
||||
registry::TargetRegistry,
|
||||
resolve_notify_object_store_handle,
|
||||
rule_engine::NotifyRuleEngine,
|
||||
runtime_facade::NotifyRuntimeFacade,
|
||||
with_notify_server_config_read_lock, with_notify_server_config_write_lock,
|
||||
};
|
||||
use rustfs_config::notify::{
|
||||
NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
|
||||
NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
|
||||
};
|
||||
use rustfs_config::server_config::{Config, KVS};
|
||||
use rustfs_targets::{Target, arn::TargetID};
|
||||
use std::sync::{Arc, LazyLock};
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Serializes the read-modify-write sequence over the persisted notify server
|
||||
/// config. The persisted config is a single process-global resource (there is
|
||||
/// only one backing object store), so without this guard two concurrent updates
|
||||
/// can both read the same base config, apply disjoint changes, and race their
|
||||
/// full-config writes — the later write silently overwrites the earlier one,
|
||||
/// losing updates. Holding this mutex across the whole read→modify→write makes
|
||||
/// concurrent updates apply serially so every change is preserved.
|
||||
///
|
||||
/// The lock is only ever acquired inside `update_server_config`; it never nests
|
||||
/// with the per-manager `config` RwLock (the in-memory reload runs after this
|
||||
/// guard is released), so it introduces no lock-ordering risk.
|
||||
static NOTIFY_CONFIG_RMW_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
|
||||
|
||||
const LOG_COMPONENT_NOTIFY: &str = "notify";
|
||||
const LOG_SUBSYSTEM_CONFIG: &str = "config";
|
||||
const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
|
||||
const EVENT_NOTIFY_CONFIG_UPDATE: &str = "notify_config_update";
|
||||
|
||||
#[derive(Debug)]
|
||||
enum NotifyConfigStoreError {
|
||||
Lock(String),
|
||||
StorageNotAvailable,
|
||||
Read(String),
|
||||
Save(String),
|
||||
}
|
||||
|
||||
async fn update_server_config<F>(modifier: F) -> Result<Option<Config>, NotifyConfigStoreError>
|
||||
async fn update_server_config<F>(
|
||||
modifier: F,
|
||||
lifecycle: NotifyLifecycleCoordinator,
|
||||
) -> Result<Option<crate::lifecycle::NotificationLifecycleTransition>, NotifyConfigStoreError>
|
||||
where
|
||||
F: FnMut(&mut Config) -> bool,
|
||||
F: FnMut(&mut Config) -> bool + Send + 'static,
|
||||
{
|
||||
let Some(store) = resolve_notify_object_store_handle() else {
|
||||
return Err(NotifyConfigStoreError::StorageNotAvailable);
|
||||
};
|
||||
|
||||
let store_for_read = store.clone();
|
||||
let store_for_save = store.clone();
|
||||
serialized_read_modify_write(
|
||||
modifier,
|
||||
move || async move {
|
||||
crate::read_notify_server_config_without_migrate(store)
|
||||
.await
|
||||
.map_err(NotifyConfigStoreError::Read)
|
||||
},
|
||||
move |config| async move {
|
||||
crate::save_notify_server_config(store_for_save, &config)
|
||||
.await
|
||||
.map_err(NotifyConfigStoreError::Save)
|
||||
},
|
||||
)
|
||||
with_notify_server_config_write_lock(store, move || {
|
||||
read_modify_write(
|
||||
modifier,
|
||||
move || async move {
|
||||
crate::read_notify_server_config_without_migrate_no_lock(store_for_read)
|
||||
.await
|
||||
.map_err(NotifyConfigStoreError::Read)
|
||||
},
|
||||
move |config| async move {
|
||||
crate::save_notify_server_config_no_lock(store_for_save, &config)
|
||||
.await
|
||||
.map_err(NotifyConfigStoreError::Save)
|
||||
},
|
||||
move |config| lifecycle.update_config(config),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(NotifyConfigStoreError::Lock)?
|
||||
}
|
||||
|
||||
/// Runs a `read → modify → write` over the persisted notify config while holding
|
||||
/// [`NOTIFY_CONFIG_RMW_LOCK`], so concurrent updates serialize and cannot clobber
|
||||
/// each other's changes (backlog#968). `read`/`save` are injected so the exact
|
||||
/// production serialization path can be exercised in tests without a live store.
|
||||
async fn serialized_read_modify_write<F, R, RFut, S, SFut>(
|
||||
async fn read_modify_write<F, R, RFut, S, SFut, P, T>(
|
||||
mut modifier: F,
|
||||
read: R,
|
||||
save: S,
|
||||
) -> Result<Option<Config>, NotifyConfigStoreError>
|
||||
publish: P,
|
||||
) -> Result<Option<T>, NotifyConfigStoreError>
|
||||
where
|
||||
F: FnMut(&mut Config) -> bool,
|
||||
R: FnOnce() -> RFut,
|
||||
RFut: std::future::Future<Output = Result<Config, NotifyConfigStoreError>>,
|
||||
S: FnOnce(Config) -> SFut,
|
||||
SFut: std::future::Future<Output = Result<(), NotifyConfigStoreError>>,
|
||||
P: FnOnce(Config) -> T,
|
||||
{
|
||||
// Hold the RMW lock across the entire read→modify→write so concurrent
|
||||
// updates serialize and cannot clobber each other's changes (backlog#968).
|
||||
let _rmw_guard = NOTIFY_CONFIG_RMW_LOCK.lock().await;
|
||||
|
||||
let mut new_config = read().await?;
|
||||
|
||||
if !modifier(&mut new_config) {
|
||||
@@ -104,7 +98,7 @@ where
|
||||
|
||||
save(new_config.clone()).await?;
|
||||
|
||||
Ok(Some(new_config))
|
||||
Ok(Some(publish(new_config)))
|
||||
}
|
||||
|
||||
pub(crate) fn notify_configuration_hint() -> String {
|
||||
@@ -142,9 +136,8 @@ pub fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) ->
|
||||
#[derive(Clone)]
|
||||
pub struct NotifyConfigManager {
|
||||
config: Arc<RwLock<Config>>,
|
||||
registry: Arc<TargetRegistry>,
|
||||
lifecycle: NotifyLifecycleCoordinator,
|
||||
rule_engine: NotifyRuleEngine,
|
||||
runtime_facade: NotifyRuntimeFacade,
|
||||
}
|
||||
|
||||
impl NotifyConfigManager {
|
||||
@@ -154,64 +147,21 @@ impl NotifyConfigManager {
|
||||
rule_engine: NotifyRuleEngine,
|
||||
runtime_facade: NotifyRuntimeFacade,
|
||||
) -> Self {
|
||||
let lifecycle = NotifyLifecycleCoordinator::new(config.clone(), registry, runtime_facade);
|
||||
Self {
|
||||
config,
|
||||
registry,
|
||||
lifecycle,
|
||||
rule_engine,
|
||||
runtime_facade,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn lifecycle(&self) -> NotifyLifecycleCoordinator {
|
||||
self.lifecycle.clone()
|
||||
}
|
||||
|
||||
pub async fn init(&self) -> Result<(), NotificationError> {
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "initializing",
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
|
||||
let config = {
|
||||
let guard = self.config.read().await;
|
||||
debug!(
|
||||
subsystem_count = guard.0.len(),
|
||||
"Initializing notification system with configuration summary"
|
||||
);
|
||||
guard.clone()
|
||||
};
|
||||
|
||||
let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
|
||||
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "targets_created",
|
||||
target_count = targets.len(),
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
if targets.is_empty() {
|
||||
debug!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "idle",
|
||||
reason = "no_targets_configured",
|
||||
hint = %notify_configuration_hint(),
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
}
|
||||
|
||||
let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
|
||||
self.runtime_facade.replace_targets(activation).await?;
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "initialized",
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
Ok(())
|
||||
let config = self.config.read().await.clone();
|
||||
self.lifecycle.set_mode(true, Some(config)).wait().await
|
||||
}
|
||||
|
||||
pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
|
||||
@@ -227,6 +177,7 @@ impl NotifyConfigManager {
|
||||
|
||||
let ttype = target_type.to_lowercase();
|
||||
let tname = target_id.id.to_lowercase();
|
||||
let log_target_id = target_id.clone();
|
||||
|
||||
// Guard against orphaning bucket notification rules (backlog#979). Removing a
|
||||
// target while a bucket rule still references it would leave a dangling
|
||||
@@ -240,7 +191,7 @@ impl NotifyConfigManager {
|
||||
)));
|
||||
}
|
||||
|
||||
self.update_config_and_reload(|config| {
|
||||
self.update_config_and_reload(move |config| {
|
||||
let mut changed = false;
|
||||
if let Some(targets_of_type) = config.0.get_mut(&ttype) {
|
||||
if targets_of_type.remove(&tname).is_some() {
|
||||
@@ -249,7 +200,7 @@ impl NotifyConfigManager {
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
action = "remove_target",
|
||||
target_id = %target_id,
|
||||
target_id = %log_target_id,
|
||||
result = "removed",
|
||||
"notify config update"
|
||||
);
|
||||
@@ -265,7 +216,7 @@ impl NotifyConfigManager {
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
action = "remove_target",
|
||||
target_id = %target_id,
|
||||
target_id = %log_target_id,
|
||||
result = "not_found",
|
||||
"notify config update"
|
||||
);
|
||||
@@ -287,7 +238,7 @@ impl NotifyConfigManager {
|
||||
);
|
||||
let ttype = target_type.to_lowercase();
|
||||
let tname = target_name.to_lowercase();
|
||||
self.update_config_and_reload(|config| {
|
||||
self.update_config_and_reload(move |config| {
|
||||
config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone());
|
||||
true
|
||||
})
|
||||
@@ -316,7 +267,7 @@ impl NotifyConfigManager {
|
||||
)));
|
||||
}
|
||||
|
||||
self.update_config_and_reload(|config| {
|
||||
self.update_config_and_reload(move |config| {
|
||||
let mut changed = false;
|
||||
if let Some(targets) = config.0.get_mut(&ttype) {
|
||||
if targets.remove(&tname).is_some() {
|
||||
@@ -332,8 +283,8 @@ impl NotifyConfigManager {
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
action = "remove_target_config",
|
||||
target_type = %target_type,
|
||||
target_name = %target_name,
|
||||
target_type = %ttype,
|
||||
target_name = %tname,
|
||||
result = "not_found",
|
||||
"notify config update"
|
||||
);
|
||||
@@ -348,81 +299,62 @@ impl NotifyConfigManager {
|
||||
}
|
||||
|
||||
pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "reloading",
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
self.lifecycle.set_mode(true, Some(new_config)).wait().await
|
||||
}
|
||||
|
||||
self.update_config(new_config.clone()).await;
|
||||
pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
|
||||
let Some(store) = resolve_notify_object_store_handle() else {
|
||||
return Err(NotificationError::StorageNotAvailable(
|
||||
"Failed to load target configuration: server storage not initialized".to_string(),
|
||||
));
|
||||
};
|
||||
self.reload_persisted_config_from_store(store).await
|
||||
}
|
||||
|
||||
// Stop the currently running replay workers *before* activating the new ones
|
||||
// (backlog#970). Each replay worker drains a per-target persisted store; if the
|
||||
// new workers start while the old ones are still running against the same
|
||||
// stores, both drain the same queues and re-deliver events. `replace_targets`
|
||||
// below also stops workers, but only after `activate_targets_with_replay` has
|
||||
// already spawned the new ones — so without this explicit stop-before-start
|
||||
// there is a window where old and new workers overlap. (The full "signal +
|
||||
// join" shutdown lives in the targets crate and is tracked under the same
|
||||
// issue; this reorders the notify-side lifecycle.)
|
||||
self.runtime_facade.stop_replay_workers().await;
|
||||
|
||||
let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
|
||||
.registry
|
||||
.create_targets_from_config(&new_config)
|
||||
.await
|
||||
.map_err(NotificationError::Target)?;
|
||||
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "targets_created",
|
||||
target_count = targets.len(),
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
if targets.is_empty() {
|
||||
debug!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "idle",
|
||||
reason = "no_targets_configured",
|
||||
hint = %notify_configuration_hint(),
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
|
||||
if self.lifecycle.state() == NotificationRuntimeState::Terminated {
|
||||
return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
|
||||
}
|
||||
|
||||
let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
|
||||
self.runtime_facade.replace_targets(activation).await?;
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "reloaded",
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
let read_store = store.clone();
|
||||
let config_cache = self.config.clone();
|
||||
let lifecycle = self.lifecycle.clone();
|
||||
let transition = with_notify_server_config_read_lock(store, move || async move {
|
||||
let config = crate::read_existing_notify_server_config_no_lock(read_store)
|
||||
.await
|
||||
.map_err(NotificationError::ReadConfig)?;
|
||||
Ok::<_, NotificationError>(if *config_cache.read().await == config && lifecycle.is_converged() {
|
||||
None
|
||||
} else {
|
||||
Some(lifecycle.update_config(config))
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(NotificationError::StorageNotAvailable)??;
|
||||
|
||||
if let Some(transition) = transition {
|
||||
transition.wait().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn update_config(&self, new_config: Config) {
|
||||
let mut config = self.config.write().await;
|
||||
*config = new_config;
|
||||
}
|
||||
|
||||
async fn update_config_and_reload<F>(&self, mut modifier: F) -> Result<(), NotificationError>
|
||||
async fn update_config_and_reload<F>(&self, modifier: F) -> Result<(), NotificationError>
|
||||
where
|
||||
F: FnMut(&mut Config) -> bool,
|
||||
F: FnMut(&mut Config) -> bool + Send + 'static,
|
||||
{
|
||||
let Some(new_config) = update_server_config(&mut modifier).await.map_err(|err| match err {
|
||||
NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
|
||||
"Failed to save target configuration: server storage not initialized".to_string(),
|
||||
),
|
||||
NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
|
||||
NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
|
||||
})?
|
||||
if self.lifecycle.state() == NotificationRuntimeState::Terminated {
|
||||
return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
|
||||
}
|
||||
let Some(transition) = update_server_config(modifier, self.lifecycle.clone())
|
||||
.await
|
||||
.map_err(|err| match err {
|
||||
NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err),
|
||||
NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
|
||||
"Failed to save target configuration: server storage not initialized".to_string(),
|
||||
),
|
||||
NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
|
||||
NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
|
||||
})?
|
||||
else {
|
||||
debug!(
|
||||
event = EVENT_NOTIFY_CONFIG_UPDATE,
|
||||
@@ -443,15 +375,15 @@ impl NotifyConfigManager {
|
||||
result = "updated",
|
||||
"notify config update"
|
||||
);
|
||||
self.reload_config(new_config).await
|
||||
transition.wait().await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{NotifyConfigManager, NotifyConfigStoreError, runtime_target_id_for_subsystem, serialized_read_modify_write};
|
||||
use crate::NotificationError;
|
||||
use super::{NotifyConfigManager, NotifyConfigStoreError, read_modify_write, runtime_target_id_for_subsystem};
|
||||
use crate::rules::RulesMap;
|
||||
use crate::{NotificationError, NotificationRuntimeState};
|
||||
use crate::{
|
||||
integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
|
||||
runtime_facade::NotifyRuntimeFacade,
|
||||
@@ -464,7 +396,10 @@ mod tests {
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::ReplayWorkerManager;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
|
||||
fn build_manager() -> NotifyConfigManager {
|
||||
@@ -474,9 +409,10 @@ mod tests {
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
|
||||
let target_list = notifier.target_list();
|
||||
let runtime_facade = NotifyRuntimeFacade::new(
|
||||
let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
|
||||
target_list,
|
||||
Arc::new(RwLock::new(ReplayWorkerManager::new())),
|
||||
notifier.dispatch_gate(),
|
||||
Arc::new(Semaphore::new(4)),
|
||||
metrics,
|
||||
);
|
||||
@@ -524,78 +460,39 @@ mod tests {
|
||||
.reload_config(Config::default())
|
||||
.await
|
||||
.expect("reload_config should succeed for empty targets");
|
||||
assert!(matches!(manager.lifecycle().state(), NotificationRuntimeState::TargetsEnabled { .. }));
|
||||
}
|
||||
|
||||
// Regression test for backlog#968: the read-modify-write over the persisted
|
||||
// notify config must be serialized. Many tasks concurrently add a distinct
|
||||
// target through the same production RMW path (`serialized_read_modify_write`,
|
||||
// which holds the global RMW lock across read→modify→write) against a shared
|
||||
// in-memory backend. Every update must survive — no lost updates. Without the
|
||||
// lock, concurrent tasks would read the same base config and clobber each
|
||||
// other's writes, leaving only a subset of targets.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn concurrent_config_updates_preserve_all_targets() {
|
||||
// Shared in-memory stand-in for the persisted config backend.
|
||||
let backend = Arc::new(RwLock::new(Config::default()));
|
||||
const TASKS: usize = 32;
|
||||
#[tokio::test]
|
||||
async fn read_modify_write_publishes_only_after_save() {
|
||||
let saved = Arc::new(AtomicBool::new(false));
|
||||
let saved_by_writer = saved.clone();
|
||||
let observed_by_publisher = saved.clone();
|
||||
|
||||
let mut handles = Vec::with_capacity(TASKS);
|
||||
for idx in 0..TASKS {
|
||||
let backend = backend.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
let ttype = NOTIFY_WEBHOOK_SUB_SYS.to_lowercase();
|
||||
let tname = format!("target-{idx}");
|
||||
|
||||
let read_backend = backend.clone();
|
||||
let save_backend = backend.clone();
|
||||
|
||||
let result = serialized_read_modify_write(
|
||||
|config: &mut Config| {
|
||||
config
|
||||
.0
|
||||
.entry(ttype.clone())
|
||||
.or_default()
|
||||
.insert(tname.clone(), KVS::default());
|
||||
true
|
||||
},
|
||||
move || async move {
|
||||
let snapshot = read_backend.read().await.clone();
|
||||
// Yield inside the critical section to widen the race window:
|
||||
// if the RMW were not serialized, other tasks would read this
|
||||
// same base snapshot and their writes would clobber ours.
|
||||
tokio::task::yield_now().await;
|
||||
Ok::<_, NotifyConfigStoreError>(snapshot)
|
||||
},
|
||||
move |config: Config| async move {
|
||||
*save_backend.write().await = config;
|
||||
Ok::<_, NotifyConfigStoreError>(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("serialized RMW should succeed");
|
||||
assert!(result.is_some(), "modifier reported a change, expected Some(config)");
|
||||
}));
|
||||
}
|
||||
|
||||
for handle in handles {
|
||||
handle.await.expect("update task should not panic");
|
||||
}
|
||||
|
||||
let final_config = backend.read().await;
|
||||
let webhook_targets = final_config
|
||||
.0
|
||||
.get(&NOTIFY_WEBHOOK_SUB_SYS.to_lowercase())
|
||||
.expect("webhook subsystem should exist after updates");
|
||||
|
||||
assert_eq!(
|
||||
webhook_targets.len(),
|
||||
TASKS,
|
||||
"all concurrent target additions must be preserved (no lost updates)"
|
||||
);
|
||||
for idx in 0..TASKS {
|
||||
let tname = format!("target-{idx}");
|
||||
assert!(webhook_targets.contains_key(&tname), "missing target {tname}: concurrent update was lost");
|
||||
}
|
||||
read_modify_write(
|
||||
|config| {
|
||||
config
|
||||
.0
|
||||
.entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
|
||||
.or_default()
|
||||
.insert("primary".to_string(), KVS::default());
|
||||
true
|
||||
},
|
||||
|| async { Ok::<_, NotifyConfigStoreError>(Config::default()) },
|
||||
move |_config| async move {
|
||||
saved_by_writer.store(true, Ordering::Release);
|
||||
Ok::<_, NotifyConfigStoreError>(())
|
||||
},
|
||||
move |_config| {
|
||||
assert!(
|
||||
observed_by_publisher.load(Ordering::Acquire),
|
||||
"publication must observe the completed save"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read-modify-write should succeed")
|
||||
.expect("changed config should publish");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use rustfs_targets::{TargetError, arn::TargetID};
|
||||
use std::io;
|
||||
use thiserror::Error;
|
||||
use tokio::task::JoinError;
|
||||
|
||||
/// Errors related to the notification system's lifecycle.
|
||||
#[derive(Debug, Error)]
|
||||
@@ -70,3 +71,24 @@ pub enum NotificationError {
|
||||
#[error("Storage not available: {0}")]
|
||||
StorageNotAvailable(String),
|
||||
}
|
||||
|
||||
pub(crate) fn transition_join_error(error: JoinError) -> NotificationError {
|
||||
let reason = if error.is_cancelled() { "cancelled" } else { "panicked" };
|
||||
NotificationError::Initialization(format!("Notification lifecycle transition task {reason}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::transition_join_error;
|
||||
|
||||
#[tokio::test]
|
||||
async fn transition_join_error_does_not_expose_panic_payload() {
|
||||
let join_error = tokio::spawn(async { panic!("do-not-expose-payload") })
|
||||
.await
|
||||
.expect_err("test task should panic");
|
||||
|
||||
let rendered = transition_join_error(join_error).to_string();
|
||||
assert!(rendered.contains("panicked"));
|
||||
assert!(!rendered.contains("do-not-expose-payload"));
|
||||
}
|
||||
}
|
||||
|
||||
+84
-24
@@ -14,38 +14,86 @@
|
||||
|
||||
use crate::{
|
||||
BucketNotificationConfig, Event, EventArgs, LifecycleError, NotificationError, NotificationMetricSnapshot,
|
||||
NotificationSystem, NotificationTargetMetricSnapshot,
|
||||
NotificationSystem, NotificationTargetMetricSnapshot, error::transition_join_error,
|
||||
};
|
||||
use rustfs_config::server_config::Config;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::sync::{Arc, LazyLock, Mutex, OnceLock, Weak};
|
||||
use tracing::error;
|
||||
|
||||
static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new();
|
||||
static LEGACY_INITIALIZATION: LazyLock<Mutex<Option<LegacyInitialization>>> = LazyLock::new(|| Mutex::new(None));
|
||||
|
||||
enum LegacyInitialization {
|
||||
Initializing(Weak<NotificationSystem>),
|
||||
Retryable(Weak<NotificationSystem>),
|
||||
Initialized,
|
||||
}
|
||||
const LOG_COMPONENT_NOTIFY: &str = "notify";
|
||||
const LOG_SUBSYSTEM_GLOBAL: &str = "global";
|
||||
const EVENT_NOTIFY_GLOBAL_STATE: &str = "notify_global_state";
|
||||
|
||||
/// Initialize the global notification system with the given configuration.
|
||||
/// This function should only be called once throughout the application life cycle.
|
||||
pub async fn initialize(config: Config) -> Result<(), NotificationError> {
|
||||
// `new` is synchronous and responsible for creating instances
|
||||
let system = NotificationSystem::new(config);
|
||||
// `init` is asynchronous and responsible for performing I/O-intensive initialization
|
||||
system.init().await?;
|
||||
fn notification_system_or_init(config: Config) -> Arc<NotificationSystem> {
|
||||
NOTIFICATION_SYSTEM
|
||||
.get_or_init(|| Arc::new(NotificationSystem::new(config)))
|
||||
.clone()
|
||||
}
|
||||
|
||||
match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(losing_system) => {
|
||||
// Another initializer won the race. `init()` above already started this
|
||||
// system's targets and replay workers, so simply dropping it would leak
|
||||
// those background tasks. Shut the losing instance down cleanly before
|
||||
// reporting the conflict (backlog#984).
|
||||
losing_system.shutdown().await;
|
||||
Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized))
|
||||
/// Initialize the global notification system with the given configuration.
|
||||
///
|
||||
/// This preserves the historical one-shot API contract. Server lifecycle code
|
||||
/// that needs idempotent reconciliation should use [`reconcile`] instead.
|
||||
pub async fn initialize(config: Config) -> Result<(), NotificationError> {
|
||||
let system = {
|
||||
let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
|
||||
match legacy.as_ref() {
|
||||
Some(LegacyInitialization::Retryable(system)) => {
|
||||
let Some(system) = system.upgrade() else {
|
||||
return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
|
||||
};
|
||||
if !NOTIFICATION_SYSTEM.get().is_some_and(|global| Arc::ptr_eq(global, &system)) {
|
||||
return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
|
||||
}
|
||||
*legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system)));
|
||||
system
|
||||
}
|
||||
Some(LegacyInitialization::Initializing(_)) | Some(LegacyInitialization::Initialized) => {
|
||||
return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
|
||||
}
|
||||
None => {
|
||||
if NOTIFICATION_SYSTEM.get().is_some() {
|
||||
return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
|
||||
}
|
||||
let system = Arc::new(NotificationSystem::new(config.clone()));
|
||||
if NOTIFICATION_SYSTEM.set(system.clone()).is_err() {
|
||||
return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
|
||||
}
|
||||
*legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system)));
|
||||
system
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let task_system = system.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = task_system.set_targets_enabled(true, Some(config)).await;
|
||||
let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
|
||||
if matches!(
|
||||
legacy.as_ref(),
|
||||
Some(LegacyInitialization::Initializing(current))
|
||||
if current.upgrade().is_some_and(|current| Arc::ptr_eq(¤t, &task_system))
|
||||
) {
|
||||
*legacy = Some(if result.is_ok() {
|
||||
LegacyInitialization::Initialized
|
||||
} else {
|
||||
LegacyInitialization::Retryable(Arc::downgrade(&task_system))
|
||||
});
|
||||
}
|
||||
result
|
||||
})
|
||||
.await
|
||||
.map_err(transition_join_error)?
|
||||
}
|
||||
|
||||
/// Initialize the global notification system only for live in-process consumers.
|
||||
@@ -54,12 +102,24 @@ pub async fn initialize(config: Config) -> Result<(), NotificationError> {
|
||||
/// ListenBucketNotification clients can receive live events even when external
|
||||
/// notification targets are disabled.
|
||||
pub fn initialize_live_events() -> Result<(), NotificationError> {
|
||||
let system = NotificationSystem::new(Config::new());
|
||||
|
||||
match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(_) => Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)),
|
||||
if NOTIFICATION_SYSTEM
|
||||
.set(Arc::new(NotificationSystem::new(Config::new())))
|
||||
.is_err()
|
||||
{
|
||||
return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensures the stable process-wide live-event container exists.
|
||||
pub fn ensure_live_events() -> Arc<NotificationSystem> {
|
||||
notification_system_or_init(Config::new())
|
||||
}
|
||||
|
||||
/// Ensures the stable singleton exists and reconciles its target runtime.
|
||||
pub async fn reconcile(config: Config) -> Result<(), NotificationError> {
|
||||
let system = notification_system_or_init(config.clone());
|
||||
system.set_targets_enabled(true, Some(config)).await
|
||||
}
|
||||
|
||||
/// Returns a handle to the global NotificationSystem instance.
|
||||
|
||||
@@ -16,7 +16,12 @@ use crate::notification_system_subscriber::NotificationSystemSubscriberView;
|
||||
use crate::notifier::{EventNotifier, TargetList};
|
||||
use crate::services::NotifyServices;
|
||||
use crate::{
|
||||
Event, error::NotificationError, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
|
||||
Event,
|
||||
error::NotificationError,
|
||||
lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState},
|
||||
pipeline::LiveEventHistory,
|
||||
registry::TargetRegistry,
|
||||
rule_engine::NotifyRuleEngine,
|
||||
rules::BucketNotificationConfig,
|
||||
};
|
||||
use hashbrown::HashMap;
|
||||
@@ -160,11 +165,12 @@ impl NotificationMetrics {
|
||||
|
||||
/// The notification system that integrates all components
|
||||
pub struct NotificationSystem {
|
||||
/// The event notifier
|
||||
/// Event dispatcher. Runtime target mutation remains lifecycle-owned.
|
||||
pub notifier: Arc<EventNotifier>,
|
||||
/// The target registry
|
||||
/// Target factory registry. Creating a target does not publish it.
|
||||
pub registry: Arc<TargetRegistry>,
|
||||
/// The current configuration
|
||||
/// The current cached configuration. Runtime publication must still go
|
||||
/// through the lifecycle methods on this type.
|
||||
pub config: Arc<RwLock<Config>>,
|
||||
services: NotifyServices,
|
||||
}
|
||||
@@ -220,10 +226,12 @@ impl NotificationSystem {
|
||||
self.services.runtime_view.get_active_targets().await
|
||||
}
|
||||
|
||||
/// Gets the complete Target list, including both active and inactive Targets.
|
||||
///
|
||||
/// # Return
|
||||
/// An `Arc<RwLock<TargetList>>` containing all Targets.
|
||||
pub async fn config_snapshot(&self) -> Config {
|
||||
self.config.read().await.clone()
|
||||
}
|
||||
|
||||
/// Gets a read-only runtime container handle. Public mutation methods on
|
||||
/// `TargetList` are intentionally unavailable outside this crate.
|
||||
pub async fn get_all_targets(&self) -> Arc<RwLock<TargetList>> {
|
||||
self.services.runtime_view.get_all_targets()
|
||||
}
|
||||
@@ -327,6 +335,43 @@ impl NotificationSystem {
|
||||
self.services.config_manager.reload_config(new_config).await
|
||||
}
|
||||
|
||||
/// Synchronously publishes a config generation without changing target mode.
|
||||
pub fn publish_config(&self, new_config: Config) -> NotificationLifecycleTransition {
|
||||
self.services.config_manager.lifecycle().update_config(new_config)
|
||||
}
|
||||
|
||||
/// Reconciles the cached and active configuration with the persisted
|
||||
/// server config without changing the target-runtime mode.
|
||||
pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
|
||||
self.services.config_manager.reload_persisted_config().await
|
||||
}
|
||||
|
||||
/// Reconciles from an explicitly selected storage context.
|
||||
pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
|
||||
self.services.config_manager.reload_persisted_config_from_store(store).await
|
||||
}
|
||||
|
||||
/// Enables or suspends configured notification targets without replacing
|
||||
/// the process-wide live-event container.
|
||||
pub async fn set_targets_enabled(&self, enabled: bool, config: Option<Config>) -> Result<(), NotificationError> {
|
||||
self.publish_targets_enabled(enabled, config).wait().await
|
||||
}
|
||||
|
||||
/// Synchronously accepts a target-runtime mode transition. The returned
|
||||
/// receipt can be awaited after the caller releases its persistence lock.
|
||||
pub fn publish_targets_enabled(&self, enabled: bool, config: Option<Config>) -> NotificationLifecycleTransition {
|
||||
self.services.config_manager.lifecycle().set_mode(enabled, config)
|
||||
}
|
||||
|
||||
pub fn runtime_lifecycle_state(&self) -> NotificationRuntimeState {
|
||||
self.services.config_manager.lifecycle().state()
|
||||
}
|
||||
|
||||
/// Returns whether the latest accepted lifecycle generation is active.
|
||||
pub fn runtime_lifecycle_is_converged(&self) -> bool {
|
||||
self.services.config_manager.lifecycle().is_converged()
|
||||
}
|
||||
|
||||
/// Loads the bucket notification configuration
|
||||
pub async fn load_bucket_notification_config(
|
||||
&self,
|
||||
@@ -365,9 +410,13 @@ impl NotificationSystem {
|
||||
self.services.runtime_view.runtime_status_snapshot().await
|
||||
}
|
||||
|
||||
// Add a method to shut down the system
|
||||
pub async fn shutdown(&self) {
|
||||
self.services.runtime_facade.shutdown().await;
|
||||
let _ = self.shutdown_checked().await;
|
||||
}
|
||||
|
||||
/// Irreversibly terminates the notification target runtime for this process.
|
||||
pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
|
||||
self.services.config_manager.lifecycle().terminate().wait().await
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ mod event;
|
||||
pub mod factory;
|
||||
mod global;
|
||||
pub mod integration;
|
||||
mod lifecycle;
|
||||
mod notification_system_subscriber;
|
||||
pub mod notifier;
|
||||
mod pipeline;
|
||||
@@ -42,10 +43,11 @@ pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
|
||||
pub use error::{LifecycleError, NotificationError};
|
||||
pub use event::{Event, EventArgs, EventArgsBuilder, NotifyObjectInfo};
|
||||
pub use global::{
|
||||
initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, notification_system,
|
||||
notification_target_metrics, notifier_global,
|
||||
ensure_live_events, initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot,
|
||||
notification_system, notification_target_metrics, notifier_global, reconcile,
|
||||
};
|
||||
pub use integration::{NotificationMetricSnapshot, NotificationSystem, NotificationTargetMetricSnapshot};
|
||||
pub use lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState};
|
||||
pub use pipeline::{LiveEventHistory, NotifyEventBridge, NotifyPipeline};
|
||||
pub use rule_engine::NotifyRuleEngine;
|
||||
pub use rules::BucketNotificationConfig;
|
||||
@@ -53,6 +55,9 @@ pub use runtime_facade::NotifyRuntimeFacade;
|
||||
pub use runtime_view::NotifyRuntimeView;
|
||||
pub use services::NotifyServices;
|
||||
pub use status_view::NotifyStatusView;
|
||||
pub use storage_api::NotifyStore;
|
||||
pub(crate) use storage_api::crate_boundary::{
|
||||
read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config,
|
||||
read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
|
||||
resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
|
||||
with_notify_server_config_write_lock,
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+642
-33
@@ -12,13 +12,18 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{error::NotificationError, event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
|
||||
use crate::error::NotificationError;
|
||||
use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
|
||||
use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
|
||||
use rustfs_targets::Target;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use rustfs_targets::target::EntityTarget;
|
||||
use rustfs_targets::{SharedTarget, Target, TargetRuntimeManager};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
use rustfs_targets::{SharedTarget, TargetRuntimeManager};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, LazyLock, Mutex as StdMutex, Weak};
|
||||
use tokio::sync::{RwLock, Semaphore, watch};
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
const LOG_COMPONENT_NOTIFY: &str = "notify";
|
||||
@@ -29,8 +34,101 @@ const EVENT_NOTIFY_DISPATCH_STARTED: &str = "notify_dispatch_started";
|
||||
const EVENT_NOTIFY_DISPATCH_COMPLETED: &str = "notify_dispatch_completed";
|
||||
const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
|
||||
|
||||
async fn wait_for_dispatch_tasks(handles: Vec<JoinHandle<()>>) {
|
||||
for handle in handles {
|
||||
if let Err(e) = handle.await {
|
||||
let reason = if e.is_cancelled() { "join_cancelled" } else { "join_panicked" };
|
||||
error!(
|
||||
event = EVENT_NOTIFY_DISPATCH_FAILED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
reason,
|
||||
"Notify dispatch task failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub type SharedNotifyTargetList = Arc<RwLock<TargetList>>;
|
||||
|
||||
static TARGET_LIST_DISPATCH_GATES: LazyLock<StdMutex<HashMap<usize, Weak<RwLock<()>>>>> =
|
||||
LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
|
||||
pub(crate) fn shared_dispatch_gate(target_list: &SharedNotifyTargetList, preferred: Option<Arc<RwLock<()>>>) -> Arc<RwLock<()>> {
|
||||
let key = Arc::as_ptr(target_list) as usize;
|
||||
let mut gates = TARGET_LIST_DISPATCH_GATES.lock().unwrap_or_else(|err| err.into_inner());
|
||||
gates.retain(|_, gate| gate.strong_count() != 0);
|
||||
if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) {
|
||||
return gate;
|
||||
}
|
||||
let gate = preferred.unwrap_or_else(|| Arc::new(RwLock::new(())));
|
||||
gates.insert(key, Arc::downgrade(&gate));
|
||||
gate
|
||||
}
|
||||
|
||||
pub(crate) struct DirectDispatchTracker {
|
||||
cancellation: CancellationToken,
|
||||
inflight: watch::Sender<usize>,
|
||||
}
|
||||
|
||||
impl DirectDispatchTracker {
|
||||
fn new() -> Self {
|
||||
let (inflight, _) = watch::channel(0);
|
||||
Self {
|
||||
cancellation: CancellationToken::new(),
|
||||
inflight,
|
||||
}
|
||||
}
|
||||
|
||||
fn acquire(self: &Arc<Self>) -> DirectDispatchLease {
|
||||
self.inflight.send_modify(|count| *count += 1);
|
||||
DirectDispatchLease {
|
||||
tracker: Arc::clone(self),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_idle(&self) {
|
||||
let mut inflight = self.inflight.subscribe();
|
||||
while *inflight.borrow_and_update() != 0 {
|
||||
if inflight.changed().await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cancel_pending(&self) {
|
||||
self.cancellation.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
struct DirectDispatchLease {
|
||||
tracker: Arc<DirectDispatchTracker>,
|
||||
}
|
||||
|
||||
impl DirectDispatchLease {
|
||||
fn cancellation(&self) -> CancellationToken {
|
||||
self.tracker.cancellation.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DirectDispatchLease {
|
||||
fn drop(&mut self) {
|
||||
self.tracker.inflight.send_modify(|count| {
|
||||
if let Some(next) = count.checked_sub(1) {
|
||||
*count = next;
|
||||
} else {
|
||||
error!(
|
||||
event = EVENT_NOTIFY_DISPATCH_FAILED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
reason = "direct_lease_underflow",
|
||||
"Notify direct dispatch lease accounting underflowed"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves the effective send concurrency (semaphore permit count).
|
||||
///
|
||||
/// A value of `0` would build a zero-permit semaphore, so `acquire` never
|
||||
@@ -63,6 +161,8 @@ fn coerce_send_concurrency(configured: usize) -> usize {
|
||||
|
||||
/// Manages event notification to targets based on rules
|
||||
pub struct EventNotifier {
|
||||
dispatch_gate: Arc<RwLock<()>>,
|
||||
enqueue_limiter: Arc<Semaphore>,
|
||||
metrics: Arc<NotificationMetrics>,
|
||||
rule_engine: NotifyRuleEngine,
|
||||
target_list: SharedNotifyTargetList,
|
||||
@@ -82,10 +182,14 @@ impl EventNotifier {
|
||||
/// Returns a new instance of EventNotifier.
|
||||
pub fn new(metrics: Arc<NotificationMetrics>, rule_engine: NotifyRuleEngine) -> Self {
|
||||
let max_inflight = resolve_send_concurrency();
|
||||
let target_list = Arc::new(RwLock::new(TargetList::new()));
|
||||
let dispatch_gate = shared_dispatch_gate(&target_list, None);
|
||||
EventNotifier {
|
||||
dispatch_gate,
|
||||
enqueue_limiter: Arc::new(Semaphore::new(max_inflight)),
|
||||
metrics,
|
||||
rule_engine,
|
||||
target_list: Arc::new(RwLock::new(TargetList::new())),
|
||||
target_list,
|
||||
send_limiter: Arc::new(Semaphore::new(max_inflight)),
|
||||
}
|
||||
}
|
||||
@@ -99,6 +203,10 @@ impl EventNotifier {
|
||||
Arc::clone(&self.target_list)
|
||||
}
|
||||
|
||||
pub(crate) fn dispatch_gate(&self) -> Arc<RwLock<()>> {
|
||||
self.dispatch_gate.clone()
|
||||
}
|
||||
|
||||
/// Returns a list of ARNs for the registered targets
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -115,12 +223,9 @@ impl EventNotifier {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Removes all targets
|
||||
pub async fn remove_all_bucket_targets(&self) {
|
||||
let mut target_list_guard = self.target_list.write().await;
|
||||
// The logic for sending cancel signals via stream_cancel_senders would be removed.
|
||||
// TargetList::clear_targets_only already handles calling target.close().
|
||||
target_list_guard.clear_targets_only().await; // Modified clear to not re-cancel
|
||||
target_list_guard.clear_targets_only().await;
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
@@ -155,7 +260,14 @@ impl EventNotifier {
|
||||
return;
|
||||
}
|
||||
let target_ids_len = target_ids.len();
|
||||
let mut handles = vec![];
|
||||
let mut deferred_handles = Vec::new();
|
||||
let mut direct_handles = Vec::new();
|
||||
|
||||
// A lifecycle writer holds this gate only while handing queue-store
|
||||
// ownership from one runtime generation to the next. Taking the read
|
||||
// guard before cloning targets means the writer both blocks new sends
|
||||
// and drains every save already using the old generation.
|
||||
let dispatch_guard = Arc::new(self.dispatch_gate.clone().read_owned().await);
|
||||
|
||||
// Use scope to limit the borrow scope of target_list
|
||||
let target_list_guard = self.target_list.read().await;
|
||||
@@ -185,10 +297,17 @@ impl EventNotifier {
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let limiter = self.send_limiter.clone();
|
||||
let is_deferred = target_for_task.store().is_some();
|
||||
let direct_dispatch_lease = (!is_deferred).then(|| target_list_guard.direct_dispatch_lease());
|
||||
let direct_cancellation = direct_dispatch_lease.as_ref().map(DirectDispatchLease::cancellation);
|
||||
let deferred_dispatch_guard = is_deferred.then(|| Arc::clone(&dispatch_guard));
|
||||
let limiter = if is_deferred {
|
||||
self.enqueue_limiter.clone()
|
||||
} else {
|
||||
self.send_limiter.clone()
|
||||
};
|
||||
let metrics = self.metrics.clone();
|
||||
let event_clone = event.clone();
|
||||
let is_deferred = target_for_task.store().is_some();
|
||||
let target_name_for_task = target_for_task.name(); // Get the name before generating the task
|
||||
debug!(
|
||||
event = EVENT_NOTIFY_DISPATCH_STARTED,
|
||||
@@ -207,9 +326,32 @@ impl EventNotifier {
|
||||
data: event_clone.as_ref().clone(),
|
||||
});
|
||||
let handle = tokio::spawn(async move {
|
||||
let _direct_dispatch_lease = direct_dispatch_lease;
|
||||
let _deferred_dispatch_guard = deferred_dispatch_guard;
|
||||
metrics.increment_processing();
|
||||
let _permit = match limiter.acquire_owned().await {
|
||||
Ok(p) => p,
|
||||
let permit = if let Some(cancellation) = direct_cancellation {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => {
|
||||
metrics.decrement_processing();
|
||||
metrics.increment_skipped();
|
||||
debug!(
|
||||
event = EVENT_NOTIFY_DISPATCH_SKIPPED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
target_id = %target_name_for_task,
|
||||
reason = "runtime_generation_replaced",
|
||||
"Skipped pending direct notify dispatch"
|
||||
);
|
||||
return;
|
||||
}
|
||||
permit = limiter.acquire_owned() => permit,
|
||||
}
|
||||
} else {
|
||||
limiter.acquire_owned().await
|
||||
};
|
||||
let _permit = match permit {
|
||||
Ok(permit) => permit,
|
||||
Err(e) => {
|
||||
error!(
|
||||
event = EVENT_NOTIFY_DISPATCH_FAILED,
|
||||
@@ -218,7 +360,7 @@ impl EventNotifier {
|
||||
target_id = %target_name_for_task,
|
||||
error = %e,
|
||||
reason = "permit_acquire_failed",
|
||||
"Failed to acquire notify send permit"
|
||||
"Failed to acquire notify dispatch permit"
|
||||
);
|
||||
metrics.increment_failed();
|
||||
return;
|
||||
@@ -260,7 +402,11 @@ impl EventNotifier {
|
||||
);
|
||||
}
|
||||
});
|
||||
handles.push(handle);
|
||||
if is_deferred {
|
||||
deferred_handles.push(handle);
|
||||
} else {
|
||||
direct_handles.push(handle);
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
event = EVENT_NOTIFY_DISPATCH_FAILED,
|
||||
@@ -276,19 +422,13 @@ impl EventNotifier {
|
||||
// target_list is automatically released here
|
||||
drop(target_list_guard);
|
||||
|
||||
// Wait for all tasks to be completed
|
||||
for handle in handles {
|
||||
if let Err(e) = handle.await {
|
||||
error!(
|
||||
event = EVENT_NOTIFY_DISPATCH_FAILED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
error = %e,
|
||||
reason = "join_failed",
|
||||
"Notify dispatch task failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Every store-backed save owns a share of the generation guard, so
|
||||
// caller cancellation cannot race lifecycle handoff with an enqueue.
|
||||
// Direct targets own no queue store and may finish against the detached
|
||||
// target while lifecycle progresses.
|
||||
drop(dispatch_guard);
|
||||
wait_for_dispatch_tasks(deferred_handles).await;
|
||||
wait_for_dispatch_tasks(direct_handles).await;
|
||||
debug!(
|
||||
event = EVENT_NOTIFY_DISPATCH_COMPLETED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
@@ -320,7 +460,7 @@ impl EventNotifier {
|
||||
target_list_guard.add(target)?;
|
||||
}
|
||||
|
||||
info!(
|
||||
tracing::info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
@@ -334,6 +474,7 @@ impl EventNotifier {
|
||||
|
||||
/// A thread-safe list of targets
|
||||
pub struct TargetList {
|
||||
direct_dispatches: Arc<DirectDispatchTracker>,
|
||||
/// Map of TargetID to Target
|
||||
runtime: TargetRuntimeManager<Event>,
|
||||
}
|
||||
@@ -348,6 +489,7 @@ impl TargetList {
|
||||
/// Creates a new TargetList
|
||||
pub fn new() -> Self {
|
||||
TargetList {
|
||||
direct_dispatches: Arc::new(DirectDispatchTracker::new()),
|
||||
runtime: TargetRuntimeManager::new(),
|
||||
}
|
||||
}
|
||||
@@ -435,6 +577,20 @@ impl TargetList {
|
||||
self.runtime.status_snapshot(replay_workers)
|
||||
}
|
||||
|
||||
fn direct_dispatch_lease(&self) -> DirectDispatchLease {
|
||||
self.direct_dispatches.acquire()
|
||||
}
|
||||
|
||||
pub(crate) fn replace_runtime(
|
||||
&mut self,
|
||||
replacement: TargetRuntimeManager<Event>,
|
||||
) -> (TargetRuntimeManager<Event>, Arc<DirectDispatchTracker>) {
|
||||
let runtime = std::mem::replace(&mut self.runtime, replacement);
|
||||
let direct_dispatches = std::mem::replace(&mut self.direct_dispatches, Arc::new(DirectDispatchTracker::new()));
|
||||
direct_dispatches.cancel_pending();
|
||||
(runtime, direct_dispatches)
|
||||
}
|
||||
|
||||
pub fn runtime_mut(&mut self) -> &mut TargetRuntimeManager<Event> {
|
||||
&mut self.runtime
|
||||
}
|
||||
@@ -458,7 +614,7 @@ mod tests {
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::StoreError;
|
||||
use rustfs_targets::{
|
||||
TargetError,
|
||||
ReplayWorkerManager, TargetError,
|
||||
store::{Key, QueueStore, Store},
|
||||
target::{EntityTarget, QueuedPayload, QueuedPayloadMeta},
|
||||
};
|
||||
@@ -466,6 +622,7 @@ mod tests {
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
#[tokio::test]
|
||||
async fn encoded_event_key_matches_raw_prefix_suffix_filter() {
|
||||
@@ -525,19 +682,44 @@ mod tests {
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestTarget {
|
||||
block_first_save: Option<(Arc<Notify>, Arc<Notify>)>,
|
||||
close_calls: Arc<AtomicUsize>,
|
||||
close_entered: Option<Arc<Notify>>,
|
||||
id: TargetID,
|
||||
enabled: bool,
|
||||
save_calls: Arc<AtomicUsize>,
|
||||
selected_calls: Arc<AtomicUsize>,
|
||||
store: Option<QueueStore<QueuedPayload>>,
|
||||
}
|
||||
|
||||
impl TestTarget {
|
||||
fn new(id: &str, name: &str, enabled: bool) -> Self {
|
||||
Self {
|
||||
block_first_save: None,
|
||||
close_calls: Arc::new(AtomicUsize::new(0)),
|
||||
close_entered: None,
|
||||
id: TargetID::new(id.to_string(), name.to_string()),
|
||||
enabled,
|
||||
save_calls: Arc::new(AtomicUsize::new(0)),
|
||||
selected_calls: Arc::new(AtomicUsize::new(0)),
|
||||
store: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_blocked_first_save(mut self, entered: Arc<Notify>, release: Arc<Notify>) -> Self {
|
||||
self.block_first_save = Some((entered, release));
|
||||
self
|
||||
}
|
||||
|
||||
fn with_store(mut self, store: QueueStore<QueuedPayload>) -> Self {
|
||||
self.store = Some(store);
|
||||
self
|
||||
}
|
||||
|
||||
fn with_close_observer(mut self, close_entered: Arc<Notify>) -> Self {
|
||||
self.close_entered = Some(close_entered);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -554,7 +736,13 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
|
||||
self.save_calls.fetch_add(1, Ordering::SeqCst);
|
||||
let call = self.save_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if call == 0
|
||||
&& let Some((entered, release)) = &self.block_first_save
|
||||
{
|
||||
entered.notify_one();
|
||||
release.notified().await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -563,11 +751,17 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<(), TargetError> {
|
||||
self.close_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(close_entered) = &self.close_entered {
|
||||
close_entered.notify_one();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
|
||||
None
|
||||
self.store
|
||||
.as_ref()
|
||||
.map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
|
||||
}
|
||||
|
||||
fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
|
||||
@@ -580,10 +774,425 @@ mod tests {
|
||||
}
|
||||
|
||||
fn is_enabled(&self) -> bool {
|
||||
self.selected_calls.fetch_add(1, Ordering::SeqCst);
|
||||
self.enabled
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_pause_drains_entered_deferred_dispatch_and_blocks_new_dispatch() {
|
||||
let metrics = Arc::new(NotificationMetrics::new());
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
|
||||
let save_entered = Arc::new(Notify::new());
|
||||
let save_release = Arc::new(Notify::new());
|
||||
let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
|
||||
let target = TestTarget::new("gated-target", "webhook", true)
|
||||
.with_blocked_first_save(save_entered.clone(), save_release.clone())
|
||||
.with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
|
||||
|
||||
let mut rules_map = RulesMap::new();
|
||||
rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
|
||||
rule_engine.set_bucket_rules("bucket", rules_map).await;
|
||||
notifier
|
||||
.target_list()
|
||||
.write()
|
||||
.await
|
||||
.add(Arc::new(target.clone()))
|
||||
.expect("target install should succeed");
|
||||
|
||||
let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
|
||||
notifier.target_list(),
|
||||
Arc::new(RwLock::new(ReplayWorkerManager::new())),
|
||||
notifier.dispatch_gate(),
|
||||
Arc::new(Semaphore::new(1)),
|
||||
metrics,
|
||||
);
|
||||
let first_dispatch = tokio::spawn({
|
||||
let notifier = notifier.clone();
|
||||
async move {
|
||||
notifier
|
||||
.send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
|
||||
.await;
|
||||
}
|
||||
});
|
||||
save_entered.notified().await;
|
||||
assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
let mut pause = Box::pin(facade.pause_dispatch());
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut pause => panic!("lifecycle pause crossed an in-flight dispatch"),
|
||||
_ = std::future::ready(()) => {}
|
||||
}
|
||||
|
||||
save_release.notify_one();
|
||||
first_dispatch.await.expect("first dispatch task should finish");
|
||||
let pause_guard = pause.await;
|
||||
|
||||
let replacement = TestTarget::new("gated-target", "webhook", true);
|
||||
{
|
||||
let target_list = notifier.target_list();
|
||||
let mut target_list = target_list.write().await;
|
||||
target_list.clear();
|
||||
target_list
|
||||
.add(Arc::new(replacement.clone()))
|
||||
.expect("replacement target install should succeed");
|
||||
}
|
||||
|
||||
let mut second_dispatch =
|
||||
Box::pin(notifier.send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut))));
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut second_dispatch => panic!("dispatch crossed the lifecycle pause"),
|
||||
_ = std::future::ready(()) => {}
|
||||
}
|
||||
assert_eq!(
|
||||
replacement.selected_calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"a paused dispatch must not select a target from the replacement generation early"
|
||||
);
|
||||
assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
drop(pause_guard);
|
||||
second_dispatch.await;
|
||||
assert_eq!(target.selected_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(replacement.selected_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_pause_does_not_wait_for_direct_network_dispatch() {
|
||||
let metrics = Arc::new(NotificationMetrics::new());
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
|
||||
let save_entered = Arc::new(Notify::new());
|
||||
let save_release = Arc::new(Notify::new());
|
||||
let target =
|
||||
TestTarget::new("direct-target", "webhook", true).with_blocked_first_save(save_entered.clone(), save_release.clone());
|
||||
|
||||
let mut rules_map = RulesMap::new();
|
||||
rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
|
||||
rule_engine.set_bucket_rules("bucket", rules_map).await;
|
||||
notifier
|
||||
.target_list()
|
||||
.write()
|
||||
.await
|
||||
.add(Arc::new(target))
|
||||
.expect("target install should succeed");
|
||||
|
||||
let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
|
||||
notifier.target_list(),
|
||||
Arc::new(RwLock::new(ReplayWorkerManager::new())),
|
||||
notifier.dispatch_gate(),
|
||||
Arc::new(Semaphore::new(1)),
|
||||
metrics,
|
||||
);
|
||||
let dispatch = tokio::spawn({
|
||||
let notifier = notifier.clone();
|
||||
async move {
|
||||
notifier
|
||||
.send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
|
||||
.await;
|
||||
}
|
||||
});
|
||||
save_entered.notified().await;
|
||||
|
||||
let mut pause = Box::pin(facade.pause_dispatch());
|
||||
let pause_guard = tokio::select! {
|
||||
biased;
|
||||
guard = &mut pause => guard,
|
||||
_ = std::future::ready(()) => panic!("a direct network send blocked lifecycle handoff"),
|
||||
};
|
||||
drop(pause_guard);
|
||||
save_release.notify_one();
|
||||
dispatch.await.expect("direct dispatch should finish after release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_cancels_permit_waiting_direct_dispatch_before_closing_target() {
|
||||
let metrics = Arc::new(NotificationMetrics::new());
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
let notifier = Arc::new(EventNotifier {
|
||||
send_limiter: Arc::new(Semaphore::new(1)),
|
||||
..EventNotifier::new(metrics.clone(), rule_engine.clone())
|
||||
});
|
||||
let first_entered = Arc::new(Notify::new());
|
||||
let first_release = Arc::new(Notify::new());
|
||||
let close_entered = Arc::new(Notify::new());
|
||||
let target = TestTarget::new("direct-target", "webhook", true)
|
||||
.with_blocked_first_save(first_entered.clone(), first_release.clone())
|
||||
.with_close_observer(close_entered);
|
||||
|
||||
let mut rules_map = RulesMap::new();
|
||||
rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
|
||||
rule_engine.set_bucket_rules("bucket", rules_map).await;
|
||||
notifier
|
||||
.target_list()
|
||||
.write()
|
||||
.await
|
||||
.add(Arc::new(target.clone()))
|
||||
.expect("target should install");
|
||||
|
||||
let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
|
||||
notifier.target_list(),
|
||||
Arc::new(RwLock::new(ReplayWorkerManager::new())),
|
||||
notifier.dispatch_gate(),
|
||||
Arc::new(Semaphore::new(1)),
|
||||
metrics.clone(),
|
||||
);
|
||||
let first = tokio::spawn({
|
||||
let notifier = notifier.clone();
|
||||
async move {
|
||||
notifier
|
||||
.send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
|
||||
.await;
|
||||
}
|
||||
});
|
||||
first_entered.notified().await;
|
||||
|
||||
// This task selects the old generation and acquires its lease before
|
||||
// waiting for the saturated direct-send permit.
|
||||
let second = tokio::spawn({
|
||||
let notifier = notifier.clone();
|
||||
async move {
|
||||
notifier
|
||||
.send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut)))
|
||||
.await;
|
||||
}
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
while target.selected_calls.load(Ordering::SeqCst) != 2 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("both direct sends should select the old generation");
|
||||
|
||||
let activation = facade.activate_targets_with_replay(Vec::new()).await;
|
||||
let mut replace = Box::pin(facade.replace_targets(activation));
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut replace => panic!("replacement closed a generation with selected direct sends: {result:?}"),
|
||||
_ = std::future::ready(()) => {}
|
||||
}
|
||||
assert_eq!(target.close_calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
first_release.notify_one();
|
||||
first.await.expect("first direct dispatch should finish");
|
||||
second.await.expect("permit-waiting direct dispatch should be cancelled");
|
||||
replace.await.expect("replacement should close after direct leases drain");
|
||||
assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(target.close_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(metrics.processing_count(), 0);
|
||||
assert_eq!(metrics.processed_count(), 1);
|
||||
assert_eq!(metrics.skipped_count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn caller_abort_does_not_release_deferred_generation_lease() {
|
||||
let metrics = Arc::new(NotificationMetrics::new());
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
|
||||
let save_entered = Arc::new(Notify::new());
|
||||
let save_release = Arc::new(Notify::new());
|
||||
let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
|
||||
let target = TestTarget::new("deferred", "webhook", true)
|
||||
.with_blocked_first_save(save_entered.clone(), save_release.clone())
|
||||
.with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
|
||||
|
||||
let mut rules_map = RulesMap::new();
|
||||
rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
|
||||
rule_engine.set_bucket_rules("bucket", rules_map).await;
|
||||
notifier
|
||||
.target_list()
|
||||
.write()
|
||||
.await
|
||||
.add(Arc::new(target))
|
||||
.expect("target should install");
|
||||
let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
|
||||
notifier.target_list(),
|
||||
Arc::new(RwLock::new(ReplayWorkerManager::new())),
|
||||
notifier.dispatch_gate(),
|
||||
Arc::new(Semaphore::new(1)),
|
||||
metrics,
|
||||
);
|
||||
|
||||
let dispatch = tokio::spawn({
|
||||
let notifier = notifier.clone();
|
||||
async move {
|
||||
notifier
|
||||
.send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
|
||||
.await;
|
||||
}
|
||||
});
|
||||
save_entered.notified().await;
|
||||
dispatch.abort();
|
||||
let _ = dispatch.await;
|
||||
|
||||
let mut pause = Box::pin(facade.pause_dispatch());
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut pause => panic!("caller abort released the deferred generation lease"),
|
||||
_ = std::future::ready(()) => {}
|
||||
}
|
||||
save_release.notify_one();
|
||||
let pause_guard = pause.await;
|
||||
drop(pause_guard);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deferred_enqueue_concurrency_is_bounded() {
|
||||
const LIMIT: usize = 2;
|
||||
const TARGETS: usize = 3;
|
||||
|
||||
let metrics = Arc::new(NotificationMetrics::new());
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
let notifier = Arc::new(EventNotifier {
|
||||
enqueue_limiter: Arc::new(Semaphore::new(LIMIT)),
|
||||
..EventNotifier::new(metrics, rule_engine.clone())
|
||||
});
|
||||
let entered = Arc::new(Notify::new());
|
||||
let release = Arc::new(Notify::new());
|
||||
let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
|
||||
let mut targets = Vec::new();
|
||||
let mut rules_map = RulesMap::new();
|
||||
for index in 0..TARGETS {
|
||||
let target = TestTarget::new(&format!("deferred-{index}"), "webhook", true)
|
||||
.with_blocked_first_save(entered.clone(), release.clone())
|
||||
.with_store(QueueStore::new(queue_dir.path().join(index.to_string()), 16, ".event"));
|
||||
rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
|
||||
notifier
|
||||
.target_list()
|
||||
.write()
|
||||
.await
|
||||
.add(Arc::new(target.clone()))
|
||||
.expect("target should install");
|
||||
targets.push(target);
|
||||
}
|
||||
rule_engine.set_bucket_rules("bucket", rules_map).await;
|
||||
|
||||
let dispatch = tokio::spawn({
|
||||
let notifier = notifier.clone();
|
||||
async move {
|
||||
notifier
|
||||
.send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
|
||||
.await;
|
||||
}
|
||||
});
|
||||
let total_calls = || {
|
||||
targets
|
||||
.iter()
|
||||
.map(|target| target.save_calls.load(Ordering::SeqCst))
|
||||
.sum::<usize>()
|
||||
};
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
while total_calls() != LIMIT {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the configured number of enqueues should enter");
|
||||
for _ in 0..10 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert_eq!(total_calls(), LIMIT, "enqueue concurrency exceeded its semaphore capacity");
|
||||
|
||||
release.notify_waiters();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
while total_calls() != TARGETS {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the waiting enqueue should enter after a permit is released");
|
||||
release.notify_waiters();
|
||||
dispatch.await.expect("all bounded enqueues should finish");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deferred_enqueue_does_not_wait_for_a_blocked_direct_send_permit() {
|
||||
let metrics = Arc::new(NotificationMetrics::new());
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
let notifier = Arc::new(EventNotifier {
|
||||
send_limiter: Arc::new(Semaphore::new(1)),
|
||||
..EventNotifier::new(metrics.clone(), rule_engine.clone())
|
||||
});
|
||||
let direct_entered = Arc::new(Notify::new());
|
||||
let direct_release = Arc::new(Notify::new());
|
||||
let direct =
|
||||
TestTarget::new("direct", "webhook", true).with_blocked_first_save(direct_entered.clone(), direct_release.clone());
|
||||
let deferred_entered = Arc::new(Notify::new());
|
||||
let deferred_release = Arc::new(Notify::new());
|
||||
let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
|
||||
let deferred = TestTarget::new("deferred", "webhook", true)
|
||||
.with_blocked_first_save(deferred_entered.clone(), deferred_release.clone())
|
||||
.with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
|
||||
|
||||
let mut direct_rules = RulesMap::new();
|
||||
direct_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), direct.id.clone());
|
||||
rule_engine.set_bucket_rules("direct-bucket", direct_rules).await;
|
||||
let mut deferred_rules = RulesMap::new();
|
||||
deferred_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), deferred.id.clone());
|
||||
rule_engine.set_bucket_rules("deferred-bucket", deferred_rules).await;
|
||||
{
|
||||
let target_list = notifier.target_list();
|
||||
let mut target_list = target_list.write().await;
|
||||
target_list.add(Arc::new(direct)).expect("direct target should install");
|
||||
target_list.add(Arc::new(deferred)).expect("deferred target should install");
|
||||
}
|
||||
|
||||
let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
|
||||
notifier.target_list(),
|
||||
Arc::new(RwLock::new(ReplayWorkerManager::new())),
|
||||
notifier.dispatch_gate(),
|
||||
Arc::new(Semaphore::new(1)),
|
||||
metrics,
|
||||
);
|
||||
let direct_dispatch = tokio::spawn({
|
||||
let notifier = notifier.clone();
|
||||
async move {
|
||||
notifier
|
||||
.send(Arc::new(Event::new_test_event("direct-bucket", "object", EventName::ObjectCreatedPut)))
|
||||
.await;
|
||||
}
|
||||
});
|
||||
direct_entered.notified().await;
|
||||
let deferred_dispatch = tokio::spawn({
|
||||
let notifier = notifier.clone();
|
||||
async move {
|
||||
notifier
|
||||
.send(Arc::new(Event::new_test_event("deferred-bucket", "object", EventName::ObjectCreatedPut)))
|
||||
.await;
|
||||
}
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), deferred_entered.notified())
|
||||
.await
|
||||
.expect("queue persistence must not wait behind a direct network send permit");
|
||||
|
||||
let mut pause = Box::pin(facade.pause_dispatch());
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut pause => panic!("lifecycle pause crossed the blocked deferred enqueue"),
|
||||
_ = std::future::ready(()) => {}
|
||||
}
|
||||
deferred_release.notify_one();
|
||||
deferred_dispatch
|
||||
.await
|
||||
.expect("deferred dispatch should finish after release");
|
||||
let pause_guard = tokio::select! {
|
||||
biased;
|
||||
guard = &mut pause => guard,
|
||||
_ = std::future::ready(()) => panic!("direct network delivery kept the lifecycle gate locked"),
|
||||
};
|
||||
drop(pause_guard);
|
||||
|
||||
direct_release.notify_one();
|
||||
direct_dispatch.await.expect("direct dispatch should finish after release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_send_event_skips_disabled_target() {
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
|
||||
@@ -38,6 +38,11 @@ impl TargetRegistry {
|
||||
TargetRegistry { plugins }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_plugins(plugins: TargetPluginRegistry<Event>) -> Self {
|
||||
Self { plugins }
|
||||
}
|
||||
|
||||
pub fn supports_target_type(&self, target_type: &str) -> bool {
|
||||
self.plugins.supports_target_type(target_type)
|
||||
}
|
||||
@@ -67,6 +72,15 @@ impl TargetRegistry {
|
||||
) -> Result<Vec<Box<dyn Target<Event> + Send + Sync>>, TargetError> {
|
||||
self.plugins.create_targets_from_config(config, NOTIFY_ROUTE_PREFIX).await
|
||||
}
|
||||
|
||||
pub(crate) async fn create_dormant_targets_from_config(
|
||||
&self,
|
||||
config: &Config,
|
||||
) -> Result<(Vec<Box<dyn Target<Event> + Send + Sync>>, Vec<String>), TargetError> {
|
||||
self.plugins
|
||||
.create_dormant_targets_from_config(config, NOTIFY_ROUTE_PREFIX)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -12,13 +12,23 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{Event, NotificationError, integration::NotificationMetrics, notifier::SharedNotifyTargetList};
|
||||
use rustfs_targets::{
|
||||
BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter, ReplayEvent, ReplayWorkerManager, RuntimeActivation, Target,
|
||||
use crate::{
|
||||
Event, NotificationError,
|
||||
error::transition_join_error,
|
||||
integration::NotificationMetrics,
|
||||
notifier::{DirectDispatchTracker, SharedNotifyTargetList, shared_dispatch_gate},
|
||||
};
|
||||
use rustfs_targets::{
|
||||
BuiltinPluginRuntimeAdapter, OpenedActivation, PluginRuntimeAdapter, PreparedActivation, ReplayEvent, ReplayWorkerManager,
|
||||
RuntimeActivation, Target, TargetRuntimeManager,
|
||||
};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
use tokio::sync::{OwnedRwLockWriteGuard, RwLock, Semaphore};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const LOG_COMPONENT_NOTIFY: &str = "notify";
|
||||
@@ -26,12 +36,25 @@ const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
|
||||
const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
|
||||
const EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED: &str = "notify_runtime_shutdown_failed";
|
||||
const EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED: &str = "notify_replay_retry_exhausted";
|
||||
const TARGET_CLOSE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
pub(crate) struct DetachedNotifyRuntime {
|
||||
direct_dispatches: Arc<DirectDispatchTracker>,
|
||||
runtime: TargetRuntimeManager<Event>,
|
||||
replay_workers: ReplayWorkerManager,
|
||||
}
|
||||
|
||||
// Multi-lock publication order: replay_workers -> target_list ->
|
||||
// publication_gate. The lifecycle may already hold dispatch_gate while
|
||||
// entering this facade; no path that needs these three locks may acquire
|
||||
// publication_gate before either runtime lock.
|
||||
#[derive(Clone)]
|
||||
pub struct NotifyRuntimeFacade {
|
||||
dispatch_gate: Arc<RwLock<()>>,
|
||||
legacy_terminated: Arc<AtomicBool>,
|
||||
target_list: SharedNotifyTargetList,
|
||||
replay_workers: Arc<RwLock<ReplayWorkerManager>>,
|
||||
runtime_adapter: Arc<dyn PluginRuntimeAdapter<Event>>,
|
||||
runtime_adapter: Arc<BuiltinPluginRuntimeAdapter<Event>>,
|
||||
}
|
||||
|
||||
impl NotifyRuntimeFacade {
|
||||
@@ -41,6 +64,18 @@ impl NotifyRuntimeFacade {
|
||||
concurrency_limiter: Arc<Semaphore>,
|
||||
metrics: Arc<NotificationMetrics>,
|
||||
) -> Self {
|
||||
let dispatch_gate = shared_dispatch_gate(&target_list, None);
|
||||
Self::new_with_dispatch_gate(target_list, replay_workers, dispatch_gate, concurrency_limiter, metrics)
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_dispatch_gate(
|
||||
target_list: SharedNotifyTargetList,
|
||||
replay_workers: Arc<RwLock<ReplayWorkerManager>>,
|
||||
dispatch_gate: Arc<RwLock<()>>,
|
||||
concurrency_limiter: Arc<Semaphore>,
|
||||
metrics: Arc<NotificationMetrics>,
|
||||
) -> Self {
|
||||
let dispatch_gate = shared_dispatch_gate(&target_list, Some(dispatch_gate));
|
||||
let replay_metrics = metrics;
|
||||
let runtime_adapter = BuiltinPluginRuntimeAdapter::new(
|
||||
Arc::new(move |event: ReplayEvent<Event>| {
|
||||
@@ -97,36 +132,166 @@ impl NotifyRuntimeFacade {
|
||||
);
|
||||
|
||||
Self {
|
||||
dispatch_gate,
|
||||
legacy_terminated: Arc::new(AtomicBool::new(false)),
|
||||
target_list,
|
||||
replay_workers,
|
||||
runtime_adapter: Arc::new(runtime_adapter),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn pause_dispatch(&self) -> OwnedRwLockWriteGuard<()> {
|
||||
self.dispatch_gate.clone().write_owned().await
|
||||
}
|
||||
|
||||
pub async fn activate_targets_with_replay(
|
||||
&self,
|
||||
targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
|
||||
) -> RuntimeActivation<Event> {
|
||||
self.runtime_adapter.activate_with_replay(targets).await
|
||||
// The compatibility pair must not start replacement replay before
|
||||
// replace_targets has stopped and joined the current generation.
|
||||
self.runtime_adapter.prepare_dormant_compat_activation(targets).await
|
||||
}
|
||||
|
||||
pub async fn replace_targets(&self, activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
|
||||
// Lock order: replay_workers -> target_list (matches notify AGENTS.md).
|
||||
pub(crate) async fn prepare_targets(
|
||||
&self,
|
||||
targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
|
||||
cancellation: &CancellationToken,
|
||||
) -> PreparedActivation<Event> {
|
||||
self.runtime_adapter.prepare_targets_cancellable(targets, cancellation).await
|
||||
}
|
||||
|
||||
pub(crate) async fn open_prepared_stores(
|
||||
&self,
|
||||
prepared: PreparedActivation<Event>,
|
||||
) -> Result<(OpenedActivation<Event>, PreparedActivation<Event>), NotificationError> {
|
||||
let runtime_adapter = self.runtime_adapter.clone();
|
||||
tokio::task::spawn_blocking(move || runtime_adapter.open_prepared_stores(prepared))
|
||||
.await
|
||||
.map_err(transition_join_error)
|
||||
}
|
||||
|
||||
pub(crate) async fn commit_prepared<Committed>(
|
||||
&self,
|
||||
opened: OpenedActivation<Event>,
|
||||
on_committed: Committed,
|
||||
) -> (DetachedNotifyRuntime, PreparedActivation<Event>)
|
||||
where
|
||||
Committed: FnOnce(bool),
|
||||
{
|
||||
// The lifecycle coordinator validates the generation immediately before
|
||||
// entering the non-cancellable handoff. Once old replay workers have
|
||||
// been joined, this generation must publish before a later accepted
|
||||
// intent can run; abandoning it here would leave the old runtime
|
||||
// visible without replay workers.
|
||||
let mut replay_workers = self.replay_workers.write().await;
|
||||
let mut target_list = self.target_list.write().await;
|
||||
self.runtime_adapter
|
||||
.replace_runtime_targets(target_list.runtime_mut(), &mut replay_workers, activation)
|
||||
.await
|
||||
.map_err(NotificationError::Target)?;
|
||||
|
||||
let (activation, rejected) = self.runtime_adapter.try_activate_prepared(opened);
|
||||
let fully_activated = rejected.failure_summary().is_none();
|
||||
let (runtime, replay_workers, direct_dispatches) =
|
||||
Self::swap_activation(&mut target_list, &mut replay_workers, activation);
|
||||
on_committed(fully_activated);
|
||||
(
|
||||
DetachedNotifyRuntime {
|
||||
direct_dispatches,
|
||||
runtime,
|
||||
replay_workers,
|
||||
},
|
||||
rejected,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn commit_disabled<Committed>(&self, on_committed: Committed) -> DetachedNotifyRuntime
|
||||
where
|
||||
Committed: FnOnce(),
|
||||
{
|
||||
// Lock order: replay_workers -> target_list. The lifecycle coordinator
|
||||
// crosses its publication barrier before stopping the old workers, so
|
||||
// this non-cancellable commit must publish even if a newer intent was
|
||||
// accepted while the workers joined.
|
||||
let mut replay_workers = self.replay_workers.write().await;
|
||||
let mut target_list = self.target_list.write().await;
|
||||
|
||||
let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
|
||||
let detached = DetachedNotifyRuntime {
|
||||
direct_dispatches,
|
||||
runtime,
|
||||
replay_workers: std::mem::take(&mut *replay_workers),
|
||||
};
|
||||
on_committed();
|
||||
detached
|
||||
}
|
||||
|
||||
pub async fn replace_targets(&self, mut activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
|
||||
// A caller may supply an activation created outside the compatibility
|
||||
// prepare method. Stop any already-running replacement workers before
|
||||
// entering the ordered handoff; the supported activate→replace pair is
|
||||
// dormant here and therefore never overlaps the old generation.
|
||||
self.runtime_adapter.stop_replay_workers(&mut activation.replay_workers).await;
|
||||
let dispatch_guard = self.pause_dispatch().await;
|
||||
if self.legacy_terminated.load(Ordering::Acquire) {
|
||||
drop(dispatch_guard);
|
||||
self.runtime_adapter
|
||||
.close_compat_activation(activation)
|
||||
.await
|
||||
.map_err(NotificationError::Target)?;
|
||||
return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
|
||||
}
|
||||
self.stop_active_replay_workers().await;
|
||||
let (activation, open_rejected, activation_rejected) = self.runtime_adapter.start_dormant_compat_activation(activation);
|
||||
let activation_failures = [open_rejected.failure_summary(), activation_rejected.failure_summary()]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>();
|
||||
let (old_runtime, old_replay_workers, old_direct_dispatches) = {
|
||||
// Lock order: replay_workers -> target_list (matches notify AGENTS.md).
|
||||
let mut replay_workers = self.replay_workers.write().await;
|
||||
let mut target_list = self.target_list.write().await;
|
||||
Self::swap_activation(&mut target_list, &mut replay_workers, activation)
|
||||
};
|
||||
drop(dispatch_guard);
|
||||
let close_old = self.close_detached_targets(DetachedNotifyRuntime {
|
||||
direct_dispatches: old_direct_dispatches,
|
||||
runtime: old_runtime,
|
||||
replay_workers: old_replay_workers,
|
||||
});
|
||||
let close_open_rejected = self.close_prepared(open_rejected);
|
||||
let close_activation_rejected = self.close_prepared(activation_rejected);
|
||||
let (close_old, close_open_rejected, close_activation_rejected) =
|
||||
tokio::join!(close_old, close_open_rejected, close_activation_rejected);
|
||||
close_old?;
|
||||
close_open_rejected?;
|
||||
close_activation_rejected?;
|
||||
if !activation_failures.is_empty() {
|
||||
return Err(NotificationError::Initialization(format!(
|
||||
"one or more notification targets failed to activate: {}",
|
||||
activation_failures.join("; ")
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn stop_replay_workers(&self) {
|
||||
let mut replay_workers = self.replay_workers.write().await;
|
||||
self.runtime_adapter.stop_replay_workers(&mut replay_workers).await;
|
||||
let _dispatch_guard = self.pause_dispatch().await;
|
||||
self.stop_active_replay_workers().await;
|
||||
}
|
||||
|
||||
pub(crate) async fn stop_active_replay_workers(&self) {
|
||||
let mut detached = {
|
||||
let mut replay_workers = self.replay_workers.write().await;
|
||||
std::mem::take(&mut *replay_workers)
|
||||
};
|
||||
self.runtime_adapter.stop_replay_workers(&mut detached).await;
|
||||
}
|
||||
|
||||
pub async fn shutdown(&self) {
|
||||
let _ = self.shutdown_checked().await;
|
||||
}
|
||||
|
||||
pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
|
||||
let _dispatch_guard = self.pause_dispatch().await;
|
||||
self.legacy_terminated.store(true, Ordering::Release);
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
@@ -135,7 +300,14 @@ impl NotifyRuntimeFacade {
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
|
||||
let active_targets = self.replay_workers.read().await.len();
|
||||
let (detached_runtime, detached_replay_workers, detached_direct_dispatches) = {
|
||||
// Lock order: replay_workers -> target_list (matches notify AGENTS.md).
|
||||
let mut replay_workers = self.replay_workers.write().await;
|
||||
let mut target_list = self.target_list.write().await;
|
||||
let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
|
||||
(runtime, std::mem::take(&mut *replay_workers), direct_dispatches)
|
||||
};
|
||||
let active_targets = detached_replay_workers.len();
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
@@ -145,25 +317,22 @@ impl NotifyRuntimeFacade {
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
|
||||
{
|
||||
// Lock order: replay_workers -> target_list (matches notify AGENTS.md).
|
||||
let mut replay_workers = self.replay_workers.write().await;
|
||||
let mut target_list = self.target_list.write().await;
|
||||
if let Err(err) = self
|
||||
.runtime_adapter
|
||||
.shutdown(target_list.runtime_mut(), &mut replay_workers)
|
||||
.await
|
||||
{
|
||||
tracing::error!(
|
||||
event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
error = %err,
|
||||
"Failed to shutdown notify runtime cleanly"
|
||||
);
|
||||
}
|
||||
let shutdown_result = self
|
||||
.close_detached(DetachedNotifyRuntime {
|
||||
direct_dispatches: detached_direct_dispatches,
|
||||
runtime: detached_runtime,
|
||||
replay_workers: detached_replay_workers,
|
||||
})
|
||||
.await;
|
||||
if let Err(err) = &shutdown_result {
|
||||
tracing::error!(
|
||||
event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
error = %err,
|
||||
"Failed to shutdown notify runtime cleanly"
|
||||
);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
@@ -172,6 +341,64 @@ impl NotifyRuntimeFacade {
|
||||
state = "stopped",
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
shutdown_result
|
||||
}
|
||||
|
||||
fn swap_activation(
|
||||
target_list: &mut crate::notifier::TargetList,
|
||||
replay_workers: &mut ReplayWorkerManager,
|
||||
activation: RuntimeActivation<Event>,
|
||||
) -> (TargetRuntimeManager<Event>, ReplayWorkerManager, Arc<DirectDispatchTracker>) {
|
||||
let mut replacement = TargetRuntimeManager::new();
|
||||
for target in activation.targets {
|
||||
replacement.add_arc(target);
|
||||
}
|
||||
let (runtime, direct_dispatches) = target_list.replace_runtime(replacement);
|
||||
(runtime, std::mem::replace(replay_workers, activation.replay_workers), direct_dispatches)
|
||||
}
|
||||
|
||||
pub(crate) async fn stop_detached_replay(&self, detached: &mut DetachedNotifyRuntime) {
|
||||
// Replay join is intentionally not wrapped in the target-close timeout:
|
||||
// returning before a worker is confirmed stopped could let a
|
||||
// replacement drain the same persistent queue concurrently.
|
||||
self.runtime_adapter.stop_replay_workers(&mut detached.replay_workers).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn close_detached_targets(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
|
||||
// Direct sends are allowed to finish after the replacement runtime is
|
||||
// published, but the old targets must remain open until every task that
|
||||
// selected that generation has released its lease.
|
||||
detached.direct_dispatches.wait_idle().await;
|
||||
match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, detached.runtime.clear_and_close()).await {
|
||||
Ok(close_errors) if close_errors.is_empty() => Ok(()),
|
||||
Ok(close_errors) => {
|
||||
let targets = close_errors
|
||||
.into_iter()
|
||||
.map(|(target_id, _)| target_id)
|
||||
.collect::<Vec<_>>()
|
||||
.join("; ");
|
||||
Err(NotificationError::Target(rustfs_targets::TargetError::Storage(format!(
|
||||
"Failed to close {targets}"
|
||||
))))
|
||||
}
|
||||
Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
|
||||
"Timed out closing replaced notification targets".to_string(),
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn close_detached(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
|
||||
self.stop_detached_replay(&mut detached).await;
|
||||
self.close_detached_targets(detached).await
|
||||
}
|
||||
|
||||
pub(crate) async fn close_prepared(&self, prepared: PreparedActivation<Event>) -> Result<(), NotificationError> {
|
||||
match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, self.runtime_adapter.close_prepared(prepared)).await {
|
||||
Ok(result) => result.map_err(NotificationError::Target),
|
||||
Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
|
||||
"Timed out closing superseded notification targets".to_string(),
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,26 +411,50 @@ mod tests {
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use rustfs_targets::store::{Key, Store};
|
||||
use rustfs_targets::store::{Key, QueueStore, Store};
|
||||
use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
|
||||
use rustfs_targets::{ReplayWorkerManager, SharedTarget, StoreError, Target, TargetError};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
use tokio::sync::{Notify, RwLock, Semaphore};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestTarget {
|
||||
close_entered: Option<Arc<Notify>>,
|
||||
close_error: bool,
|
||||
close_release: Option<Arc<Notify>>,
|
||||
close_calls: Arc<AtomicUsize>,
|
||||
id: TargetID,
|
||||
store: Option<QueueStore<QueuedPayload>>,
|
||||
}
|
||||
|
||||
impl TestTarget {
|
||||
fn new(id: &str, name: &str) -> Self {
|
||||
Self {
|
||||
close_entered: None,
|
||||
close_error: false,
|
||||
close_release: None,
|
||||
close_calls: Arc::new(AtomicUsize::new(0)),
|
||||
id: TargetID::new(id.to_string(), name.to_string()),
|
||||
store: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_blocking_close(mut self, entered: Arc<Notify>, release: Arc<Notify>) -> Self {
|
||||
self.close_entered = Some(entered);
|
||||
self.close_release = Some(release);
|
||||
self
|
||||
}
|
||||
|
||||
fn with_close_error(mut self) -> Self {
|
||||
self.close_error = true;
|
||||
self
|
||||
}
|
||||
|
||||
fn with_store(mut self, store: QueueStore<QueuedPayload>) -> Self {
|
||||
self.store = Some(store);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -229,11 +480,22 @@ mod tests {
|
||||
|
||||
async fn close(&self) -> Result<(), TargetError> {
|
||||
self.close_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(entered) = &self.close_entered {
|
||||
entered.notify_one();
|
||||
}
|
||||
if let Some(release) = &self.close_release {
|
||||
release.notified().await;
|
||||
}
|
||||
if self.close_error {
|
||||
return Err(TargetError::Storage("forced close failure".to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
|
||||
None
|
||||
self.store
|
||||
.as_ref()
|
||||
.map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
|
||||
}
|
||||
|
||||
fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
|
||||
@@ -254,7 +516,13 @@ mod tests {
|
||||
let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new()));
|
||||
let target_list = notifier.target_list();
|
||||
let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
|
||||
let facade = NotifyRuntimeFacade::new(target_list, replay_workers.clone(), Arc::new(Semaphore::new(4)), metrics);
|
||||
let facade = NotifyRuntimeFacade::new_with_dispatch_gate(
|
||||
target_list,
|
||||
replay_workers.clone(),
|
||||
notifier.dispatch_gate(),
|
||||
Arc::new(Semaphore::new(4)),
|
||||
metrics,
|
||||
);
|
||||
(facade, notifier, replay_workers)
|
||||
}
|
||||
|
||||
@@ -273,6 +541,26 @@ mod tests {
|
||||
assert_eq!(activation.replay_workers.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compatibility_activation_stays_dormant_until_ordered_replace() {
|
||||
let (facade, _, replay_workers) = build_facade();
|
||||
let queue_root = tempfile::tempdir().expect("queue root");
|
||||
let store = QueueStore::new_with_compression(queue_root.path(), 16, ".event", false);
|
||||
let target = TestTarget::new("primary", "webhook").with_store(store);
|
||||
|
||||
let activation = facade.activate_targets_with_replay(vec![Box::new(target)]).await;
|
||||
assert_eq!(activation.targets.len(), 1);
|
||||
assert_eq!(activation.replay_workers.len(), 0, "compatibility prepare must not start replay early");
|
||||
assert_eq!(replay_workers.read().await.len(), 0);
|
||||
|
||||
facade
|
||||
.replace_targets(activation)
|
||||
.await
|
||||
.expect("ordered compatibility replace should succeed");
|
||||
assert_eq!(replay_workers.read().await.len(), 1);
|
||||
facade.shutdown_checked().await.expect("test runtime should shut down");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_facade_replace_targets_commits_runtime_state() {
|
||||
let (facade, notifier, replay_workers) = build_facade();
|
||||
@@ -292,4 +580,129 @@ mod tests {
|
||||
assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]);
|
||||
assert_eq!(replay_workers.read().await.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_queries_do_not_wait_for_target_close() {
|
||||
let (facade, notifier, replay_workers) = build_facade();
|
||||
let close_entered = Arc::new(Notify::new());
|
||||
let close_release = Arc::new(Notify::new());
|
||||
let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), close_release.clone());
|
||||
facade
|
||||
.replace_targets(rustfs_targets::RuntimeActivation {
|
||||
replay_workers: ReplayWorkerManager::new(),
|
||||
targets: vec![Arc::new(target) as SharedTarget<Event>],
|
||||
})
|
||||
.await
|
||||
.expect("target install should succeed");
|
||||
|
||||
let shutdown = tokio::spawn({
|
||||
let facade = facade.clone();
|
||||
async move { facade.shutdown_checked().await }
|
||||
});
|
||||
close_entered.notified().await;
|
||||
|
||||
let target_list = notifier.target_list();
|
||||
assert!(target_list.try_read().is_ok(), "target list lock must not be held during close");
|
||||
assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during close");
|
||||
close_release.notify_one();
|
||||
shutdown
|
||||
.await
|
||||
.expect("shutdown task should not panic")
|
||||
.expect("shutdown should succeed");
|
||||
|
||||
let snapshot = NotifyRuntimeView::new(target_list, replay_workers)
|
||||
.runtime_status_snapshot()
|
||||
.await;
|
||||
assert_eq!(snapshot.target_count, 0);
|
||||
assert_eq!(snapshot.replay_worker_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_locks_are_released_while_replay_worker_joins() {
|
||||
let (facade, notifier, replay_workers) = build_facade();
|
||||
let cancel_received = Arc::new(Notify::new());
|
||||
let release = Arc::new(Notify::new());
|
||||
let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel(1);
|
||||
let join = tokio::spawn({
|
||||
let cancel_received = cancel_received.clone();
|
||||
let release = release.clone();
|
||||
async move {
|
||||
let _ = cancel_rx.recv().await;
|
||||
cancel_received.notify_one();
|
||||
release.notified().await;
|
||||
}
|
||||
});
|
||||
replay_workers
|
||||
.write()
|
||||
.await
|
||||
.insert_with_handle("blocked-worker".to_string(), cancel_tx, join);
|
||||
|
||||
let stop = tokio::spawn({
|
||||
let facade = facade.clone();
|
||||
async move { facade.stop_replay_workers().await }
|
||||
});
|
||||
cancel_received.notified().await;
|
||||
|
||||
assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during join");
|
||||
let target_list = notifier.target_list();
|
||||
assert!(target_list.try_read().is_ok(), "target list lock must not be held during replay join");
|
||||
|
||||
release.notify_one();
|
||||
stop.await.expect("stop task should finish");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_returns_close_error_after_detaching_runtime() {
|
||||
let (facade, notifier, replay_workers) = build_facade();
|
||||
let target = TestTarget::new("primary", "webhook").with_close_error();
|
||||
facade
|
||||
.replace_targets(rustfs_targets::RuntimeActivation {
|
||||
replay_workers: ReplayWorkerManager::new(),
|
||||
targets: vec![Arc::new(target) as SharedTarget<Event>],
|
||||
})
|
||||
.await
|
||||
.expect("target install should succeed");
|
||||
|
||||
let err = facade.shutdown_checked().await.expect_err("close failure should propagate");
|
||||
assert!(matches!(err, crate::NotificationError::Target(TargetError::Storage(_))));
|
||||
|
||||
let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
|
||||
.runtime_status_snapshot()
|
||||
.await;
|
||||
assert_eq!(snapshot.target_count, 0);
|
||||
assert_eq!(snapshot.replay_worker_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shutdown_bounds_a_target_that_never_closes() {
|
||||
let (facade, notifier, replay_workers) = build_facade();
|
||||
let close_entered = Arc::new(Notify::new());
|
||||
let never_release = Arc::new(Notify::new());
|
||||
let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), never_release);
|
||||
facade
|
||||
.replace_targets(rustfs_targets::RuntimeActivation {
|
||||
replay_workers: ReplayWorkerManager::new(),
|
||||
targets: vec![Arc::new(target) as SharedTarget<Event>],
|
||||
})
|
||||
.await
|
||||
.expect("target install should succeed");
|
||||
|
||||
let shutdown = tokio::spawn({
|
||||
let facade = facade.clone();
|
||||
async move { facade.shutdown_checked().await }
|
||||
});
|
||||
close_entered.notified().await;
|
||||
tokio::time::advance(super::TARGET_CLOSE_TIMEOUT).await;
|
||||
|
||||
let err = shutdown
|
||||
.await
|
||||
.expect("shutdown task should not panic")
|
||||
.expect_err("blocked close must time out");
|
||||
assert!(matches!(err, crate::NotificationError::Target(TargetError::Timeout(_))));
|
||||
let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
|
||||
.runtime_status_snapshot()
|
||||
.await;
|
||||
assert_eq!(snapshot.target_count, 0);
|
||||
assert_eq!(snapshot.replay_worker_count, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,7 +61,8 @@ impl NotifyRuntimeView {
|
||||
}
|
||||
|
||||
pub async fn snapshot_target_health(&self) -> Vec<RuntimeTargetHealthSnapshot> {
|
||||
self.target_list.read().await.runtime_health_snapshots().await
|
||||
let targets = self.target_list.read().await.values();
|
||||
rustfs_targets::health_snapshots_for_targets(targets).await
|
||||
}
|
||||
|
||||
pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot {
|
||||
@@ -84,7 +85,7 @@ mod tests {
|
||||
Arc,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::sync::{Notify, RwLock};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestTarget {
|
||||
@@ -93,6 +94,8 @@ mod tests {
|
||||
failed_messages: Arc<AtomicU64>,
|
||||
failed_store_length: u64,
|
||||
id: TargetID,
|
||||
health_started: Option<Arc<Notify>>,
|
||||
health_release: Option<Arc<Notify>>,
|
||||
total_messages: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
@@ -104,6 +107,8 @@ mod tests {
|
||||
failed_messages: Arc::new(AtomicU64::new(0)),
|
||||
failed_store_length: 0,
|
||||
id: TargetID::new(id.to_string(), name.to_string()),
|
||||
health_started: None,
|
||||
health_release: None,
|
||||
total_messages: Arc::new(AtomicU64::new(0)),
|
||||
}
|
||||
}
|
||||
@@ -123,6 +128,12 @@ mod tests {
|
||||
self
|
||||
}
|
||||
|
||||
fn with_health_gate(mut self, started: Arc<Notify>, release: Arc<Notify>) -> Self {
|
||||
self.health_started = Some(started);
|
||||
self.health_release = Some(release);
|
||||
self
|
||||
}
|
||||
|
||||
fn record_successes(&self, count: u64) {
|
||||
self.total_messages.store(count, Ordering::Relaxed);
|
||||
}
|
||||
@@ -142,6 +153,10 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn is_active(&self) -> Result<bool, TargetError> {
|
||||
if let (Some(started), Some(release)) = (&self.health_started, &self.health_release) {
|
||||
started.notify_one();
|
||||
release.notified().await;
|
||||
}
|
||||
Ok(self.active)
|
||||
}
|
||||
|
||||
@@ -268,4 +283,29 @@ mod tests {
|
||||
assert_eq!(status.target_count, 2);
|
||||
assert_eq!(status.replay_worker_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn health_probe_does_not_hold_the_target_list_read_lock() {
|
||||
let target_list = Arc::new(RwLock::new(TargetList::new()));
|
||||
let started = Arc::new(Notify::new());
|
||||
let release = Arc::new(Notify::new());
|
||||
let target = Arc::new(TestTarget::new("blocked", "webhook").with_health_gate(started.clone(), release.clone()));
|
||||
target_list
|
||||
.write()
|
||||
.await
|
||||
.add(target as Arc<dyn Target<Event> + Send + Sync>)
|
||||
.expect("test target should be added");
|
||||
let runtime_view = NotifyRuntimeView::new(target_list.clone(), Arc::new(RwLock::new(ReplayWorkerManager::new())));
|
||||
|
||||
let snapshot_task = tokio::spawn(async move { runtime_view.snapshot_target_health().await });
|
||||
started.notified().await;
|
||||
|
||||
let write_guard = tokio::time::timeout(std::time::Duration::from_secs(1), target_list.write())
|
||||
.await
|
||||
.expect("network health probe must not retain the target-list read lock");
|
||||
drop(write_guard);
|
||||
release.notify_one();
|
||||
|
||||
assert_eq!(snapshot_task.await.expect("snapshot task should finish").len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,7 +57,13 @@ impl NotifyServices {
|
||||
live_event_history: Arc<RwLock<LiveEventHistory>>,
|
||||
) -> Self {
|
||||
let runtime_view = NotifyRuntimeView::new(target_list.clone(), stream_cancellers.clone());
|
||||
let runtime_facade = NotifyRuntimeFacade::new(target_list, stream_cancellers, concurrency_limiter, metrics.clone());
|
||||
let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
|
||||
target_list,
|
||||
stream_cancellers,
|
||||
notifier.dispatch_gate(),
|
||||
concurrency_limiter,
|
||||
metrics.clone(),
|
||||
);
|
||||
let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone());
|
||||
let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view);
|
||||
let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history);
|
||||
|
||||
@@ -15,35 +15,70 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_ecstore::api::config::com::{
|
||||
read_config_without_migrate as read_notify_config_without_migrate_from_backend,
|
||||
save_server_config as save_notify_server_config_to_backend,
|
||||
read_config_without_migrate_no_lock as read_notify_config_without_migrate_from_backend_no_lock,
|
||||
read_existing_server_config_no_lock as read_existing_notify_config_from_backend_no_lock,
|
||||
save_server_config_no_lock as save_notify_server_config_to_backend_no_lock,
|
||||
with_server_config_read_lock as with_notify_server_config_read_lock_from_backend,
|
||||
with_server_config_write_lock as with_notify_server_config_write_lock_from_backend,
|
||||
};
|
||||
use rustfs_ecstore::api::runtime::object_store_handle as resolve_notify_object_store_handle_from_backend;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as NotifyStore;
|
||||
pub use rustfs_ecstore::api::storage::ECStore as NotifyStore;
|
||||
|
||||
pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
|
||||
resolve_notify_object_store_handle_from_backend()
|
||||
}
|
||||
|
||||
pub(crate) async fn read_notify_server_config_without_migrate(
|
||||
pub(crate) async fn read_notify_server_config_without_migrate_no_lock(
|
||||
store: Arc<NotifyStore>,
|
||||
) -> Result<rustfs_config::server_config::Config, String> {
|
||||
read_notify_config_without_migrate_from_backend(store)
|
||||
read_notify_config_without_migrate_from_backend_no_lock(store)
|
||||
.await
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
pub(crate) async fn save_notify_server_config(
|
||||
pub(crate) async fn read_existing_notify_server_config_no_lock(
|
||||
store: Arc<NotifyStore>,
|
||||
) -> Result<rustfs_config::server_config::Config, String> {
|
||||
read_existing_notify_config_from_backend_no_lock(store)
|
||||
.await
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
pub(crate) async fn save_notify_server_config_no_lock(
|
||||
store: Arc<NotifyStore>,
|
||||
config: &rustfs_config::server_config::Config,
|
||||
) -> Result<(), String> {
|
||||
save_notify_server_config_to_backend(store, config)
|
||||
save_notify_server_config_to_backend_no_lock(store, config)
|
||||
.await
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
pub(crate) async fn with_notify_server_config_write_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String>
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
with_notify_server_config_write_lock_from_backend(store, operation)
|
||||
.await
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
pub(crate) async fn with_notify_server_config_read_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String>
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
with_notify_server_config_read_lock_from_backend(store, operation)
|
||||
.await
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
pub(crate) mod crate_boundary {
|
||||
pub(crate) use super::{
|
||||
read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config,
|
||||
read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
|
||||
resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
|
||||
with_notify_server_config_write_lock,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_config::server_config::Config;
|
||||
use rustfs_notify::{NotificationError, NotificationRuntimeState, ensure_live_events, notification_system, reconcile};
|
||||
use std::sync::Arc;
|
||||
|
||||
fn assert_terminated(error: NotificationError) {
|
||||
assert!(matches!(
|
||||
error,
|
||||
NotificationError::Initialization(detail) if detail == "Notification runtime has terminated"
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn global_singleton_survives_suspend_but_not_process_termination() {
|
||||
let system = ensure_live_events();
|
||||
let same_system = ensure_live_events();
|
||||
assert!(Arc::ptr_eq(&system, &same_system));
|
||||
assert!(Arc::ptr_eq(
|
||||
&system,
|
||||
¬ification_system().expect("global notification system should exist")
|
||||
));
|
||||
|
||||
system
|
||||
.set_targets_enabled(true, Some(Config::new()))
|
||||
.await
|
||||
.expect("empty target runtime should enable");
|
||||
assert!(matches!(
|
||||
system.runtime_lifecycle_state(),
|
||||
NotificationRuntimeState::TargetsEnabled { .. }
|
||||
));
|
||||
|
||||
system
|
||||
.set_targets_enabled(false, None)
|
||||
.await
|
||||
.expect("disable should suspend targets without terminating the singleton");
|
||||
assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly);
|
||||
|
||||
system
|
||||
.set_targets_enabled(true, None)
|
||||
.await
|
||||
.expect("a suspended target runtime should be restartable");
|
||||
assert!(matches!(
|
||||
system.runtime_lifecycle_state(),
|
||||
NotificationRuntimeState::TargetsEnabled { .. }
|
||||
));
|
||||
|
||||
system
|
||||
.shutdown_checked()
|
||||
.await
|
||||
.expect("process shutdown should terminate the target runtime");
|
||||
assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated);
|
||||
|
||||
let lazy_system = ensure_live_events();
|
||||
assert!(Arc::ptr_eq(&system, &lazy_system));
|
||||
assert_terminated(
|
||||
reconcile(Config::new())
|
||||
.await
|
||||
.expect_err("lazy reconciliation must not restart a terminated process runtime"),
|
||||
);
|
||||
assert_terminated(
|
||||
lazy_system
|
||||
.reload_config(Config::new())
|
||||
.await
|
||||
.expect_err("config reload must not restart a terminated process runtime"),
|
||||
);
|
||||
assert_eq!(lazy_system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated);
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
|
||||
use rustfs_config::server_config::{Config, KVS};
|
||||
use rustfs_config::{ENABLE_KEY, WEBHOOK_ENDPOINT};
|
||||
use rustfs_notify::{initialize, notification_system};
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_legacy_initialize_can_retry_the_stable_singleton() {
|
||||
let mut invalid_target = KVS::new();
|
||||
invalid_target.insert(ENABLE_KEY.to_string(), "on".to_string());
|
||||
invalid_target.insert(WEBHOOK_ENDPOINT.to_string(), "not-a-url".to_string());
|
||||
let mut invalid_config = Config::new();
|
||||
invalid_config
|
||||
.0
|
||||
.entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
|
||||
.or_default()
|
||||
.insert("primary".to_string(), invalid_target);
|
||||
|
||||
initialize(invalid_config)
|
||||
.await
|
||||
.expect_err("invalid first target activation should fail");
|
||||
initialize(Config::new())
|
||||
.await
|
||||
.expect("legacy initialize should retry after the configuration is fixed");
|
||||
|
||||
notification_system()
|
||||
.expect("stable singleton should remain available")
|
||||
.shutdown_checked()
|
||||
.await
|
||||
.expect("test runtime should terminate");
|
||||
}
|
||||
@@ -154,7 +154,7 @@ impl TargetReplicationResyncStatus {
|
||||
}
|
||||
|
||||
fn marshal_wire_msg(&self, wr: &mut Vec<u8>) -> Result<()> {
|
||||
rmp::encode::write_map_len(wr, 11)?;
|
||||
rmp::encode::write_map_len(wr, 12)?;
|
||||
rmp::encode::write_str(wr, "st")?;
|
||||
write_msgp_time(wr, wire_time_or_default(self.start_time))?;
|
||||
rmp::encode::write_str(wr, "lst")?;
|
||||
@@ -177,6 +177,8 @@ impl TargetReplicationResyncStatus {
|
||||
rmp::encode::write_str(wr, &self.bucket)?;
|
||||
rmp::encode::write_str(wr, "obj")?;
|
||||
rmp::encode::write_str(wr, &self.object)?;
|
||||
rmp::encode::write_str(wr, "err")?;
|
||||
rmp::encode::write_str(wr, self.error.as_deref().unwrap_or_default())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -202,6 +204,10 @@ impl TargetReplicationResyncStatus {
|
||||
"rrc" => out.replicated_count = rmp::decode::read_int(rd)?,
|
||||
"bkt" => out.bucket = read_msgp_str(rd)?,
|
||||
"obj" => out.object = read_msgp_str(rd)?,
|
||||
"err" => {
|
||||
let error = read_msgp_str(rd)?;
|
||||
out.error = (!error.is_empty()).then_some(error);
|
||||
}
|
||||
_ => skip_msgp_value(rd)?,
|
||||
}
|
||||
}
|
||||
@@ -623,6 +629,7 @@ mod tests {
|
||||
bucket: "bucket-a".to_string(),
|
||||
object: "object-a".to_string(),
|
||||
replicated_count: 7,
|
||||
error: Some("durable failure".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
@@ -634,6 +641,7 @@ mod tests {
|
||||
assert_eq!(got.targets_map["arn:replication:a"].resync_id, "rid-1");
|
||||
assert_eq!(got.targets_map["arn:replication:a"].resync_status, ResyncStatusType::ResyncStarted);
|
||||
assert_eq!(got.targets_map["arn:replication:a"].replicated_count, 7);
|
||||
assert_eq!(got.targets_map["arn:replication:a"].error.as_deref(), Some("durable failure"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -833,7 +833,7 @@ mod serial_tests {
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
backend.set_put_remote_version(Some(Uuid::new_v4().to_string())).await;
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
backend.reject_next_non_empty_remote_version_validation();
|
||||
backend.set_remove_failure(true);
|
||||
let cleanup_store_barrier = TransitionCleanupStoreBarrier::install();
|
||||
|
||||
@@ -943,7 +943,7 @@ mod serial_tests {
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
let remote_version = Uuid::new_v4().to_string();
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
backend.reject_next_non_empty_remote_version_validation();
|
||||
backend.set_remove_failure(matches!(case, CleanupCase::FullyFailed));
|
||||
let put_barrier = backend.arm_put_barrier().await;
|
||||
let remove_barrier = if matches!(case, CleanupCase::RetryPersisted) {
|
||||
|
||||
@@ -22,6 +22,7 @@ pub enum CapabilityState {
|
||||
Supported,
|
||||
Unsupported,
|
||||
Disabled,
|
||||
#[serde(other)]
|
||||
#[default]
|
||||
Unknown,
|
||||
}
|
||||
@@ -133,4 +134,12 @@ mod tests {
|
||||
assert_eq!(decoded.1.reason.as_deref(), Some("target does not expose profiler"));
|
||||
assert!(!decoded.1.state.is_supported());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn future_capability_state_deserializes_as_unknown() {
|
||||
let status: CapabilityStatus = serde_json::from_str(r#"{"state":"future_state"}"#)
|
||||
.expect("future capability states should preserve conservative compatibility");
|
||||
|
||||
assert_eq!(status.state, CapabilityState::Unknown);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,8 @@ rustfs-tls-runtime = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["egress"] }
|
||||
async-trait = { workspace = true }
|
||||
async-nats = { workspace = true }
|
||||
futures-util = { workspace = true }
|
||||
async-nats = { workspace = true, features = ["server_2_10", "server_2_11", "server_2_12", "server_2_14", "service", "aws-lc-rs", "nkeys", "object-store", "kv", "websockets", "nuid"] }
|
||||
deadpool-postgres = { workspace = true }
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "tls12", "logging", "aws-lc-rs"] }
|
||||
@@ -26,6 +27,7 @@ lapin = { workspace = true, default-features = false, features = ["tokio", "rust
|
||||
libc = { workspace = true }
|
||||
pulsar = { workspace = true, default-features = false, features = ["tokio-rustls-runtime", "telemetry"] }
|
||||
regex = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
rumqttc = { workspace = true, features = ["websocket"] }
|
||||
redis = { workspace = true, features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
|
||||
@@ -38,6 +40,7 @@ sha2 = { workspace = true }
|
||||
snap = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
|
||||
tokio-util = { workspace = true }
|
||||
tokio-postgres = { workspace = true, default-features = false, features = ["runtime", "with-serde_json-1"] }
|
||||
tokio-postgres-rustls = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
@@ -23,7 +23,7 @@ use rustfs_config::{
|
||||
NATS_TLS_CLIENT_KEY, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_TLS_CA,
|
||||
PULSAR_TOPIC, PULSAR_USERNAME,
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::egress::OutboundPolicy;
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::str::FromStr;
|
||||
@@ -47,16 +47,13 @@ pub(super) fn split_env_field_and_instance(rest: &str, valid_fields: &HashSet<St
|
||||
.max_by_key(|(field, _)| field.len())
|
||||
}
|
||||
|
||||
pub(super) fn is_target_enabled(config: &KVS) -> bool {
|
||||
config
|
||||
.lookup(ENABLE_KEY)
|
||||
.map(|v| {
|
||||
EnableState::from_str(v.as_str())
|
||||
.ok()
|
||||
.map(|s| s.is_enabled())
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
pub(super) fn is_target_enabled(config: &KVS) -> Result<bool, TargetError> {
|
||||
let Some(value) = config.lookup(ENABLE_KEY) else {
|
||||
return Ok(false);
|
||||
};
|
||||
EnableState::from_str(value.as_str())
|
||||
.map(EnableState::is_enabled)
|
||||
.map_err(|_| TargetError::Configuration(format!("Invalid {ENABLE_KEY} value '{value}'")))
|
||||
}
|
||||
|
||||
pub(super) fn parse_target_bool(value: Option<&str>) -> Option<bool> {
|
||||
@@ -218,16 +215,20 @@ pub(super) fn validate_pulsar_broker_config(broker: &str, config: &KVS, default_
|
||||
}
|
||||
|
||||
pub(super) fn parse_url(value: &str, field_label: &str) -> Result<Url, TargetError> {
|
||||
Url::parse(value).map_err(|e| TargetError::Configuration(format!("Invalid {field_label}: {e} (value: '{value}')")))
|
||||
Url::parse(value).map_err(|e| TargetError::Configuration(format!("Invalid {field_label}: {e}")))
|
||||
}
|
||||
|
||||
pub(super) fn validate_outbound_http_url(value: &Url, field_label: &str) -> Result<(), TargetError> {
|
||||
validate_outbound_url(value).map_err(|e| TargetError::Configuration(format!("{field_label} is not allowed: {e}")))
|
||||
let policy =
|
||||
OutboundPolicy::from_env_cached().map_err(|err| TargetError::Configuration(format!("invalid outbound policy: {err}")))?;
|
||||
policy
|
||||
.validate_url(value)
|
||||
.map_err(|e| TargetError::Configuration(format!("{field_label} is not allowed: {e}")))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{parse_jetstream_enable, validate_nats_server_config, validate_pulsar_broker_config};
|
||||
use super::{parse_jetstream_enable, parse_url, validate_nats_server_config, validate_pulsar_broker_config};
|
||||
use async_nats::ServerAddr;
|
||||
use rustfs_config::server_config::KVS;
|
||||
use rustfs_config::{
|
||||
@@ -241,6 +242,12 @@ mod tests {
|
||||
ServerAddr::from_str("nats://127.0.0.1:4222").expect("valid nats address")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_url_error_does_not_echo_the_configured_value() {
|
||||
let err = parse_url("not a URL containing secret-token", "endpoint URL").expect_err("invalid URL should fail");
|
||||
assert!(!err.to_string().contains("secret-token"));
|
||||
}
|
||||
|
||||
// Absolute on Linux, macOS, and Windows. temp_dir needs no filesystem to exist for a
|
||||
// validation-only test, and Path::is_absolute stays true across platforms.
|
||||
fn nats_queue_dir() -> String {
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::loader::collect_merged_target_configs_from_env;
|
||||
use super::loader::{
|
||||
MergedTargetConfigRecord, collect_merged_target_configs_compat_from_env, collect_merged_target_configs_from_env,
|
||||
};
|
||||
use crate::TargetError;
|
||||
use crate::domain::TargetDomain;
|
||||
use rustfs_config::server_config::{Config, KVS};
|
||||
use std::collections::HashSet;
|
||||
@@ -86,6 +89,13 @@ pub fn normalize_target_plugin_instances(
|
||||
normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars())
|
||||
}
|
||||
|
||||
pub fn try_normalize_target_plugin_instances(
|
||||
config: &Config,
|
||||
descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
|
||||
) -> Result<Vec<TargetPluginInstanceRecord>, TargetError> {
|
||||
try_normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars())
|
||||
}
|
||||
|
||||
pub fn normalize_target_plugin_instances_from_env<I>(
|
||||
config: &Config,
|
||||
descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
|
||||
@@ -100,7 +110,7 @@ where
|
||||
.map(|field| (*field).to_string())
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
collect_merged_target_configs_from_env(
|
||||
collect_merged_target_configs_compat_from_env(
|
||||
config,
|
||||
descriptor.subsystem,
|
||||
descriptor.route_prefix,
|
||||
@@ -109,7 +119,42 @@ where
|
||||
env_vars,
|
||||
)
|
||||
.into_iter()
|
||||
.map(|record| TargetPluginInstanceRecord {
|
||||
.map(|record| target_plugin_instance_record(descriptor, record))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn try_normalize_target_plugin_instances_from_env<I>(
|
||||
config: &Config,
|
||||
descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
|
||||
env_vars: I,
|
||||
) -> Result<Vec<TargetPluginInstanceRecord>, TargetError>
|
||||
where
|
||||
I: IntoIterator<Item = (String, String)>,
|
||||
{
|
||||
let valid_fields = descriptor
|
||||
.valid_fields
|
||||
.iter()
|
||||
.map(|field| (*field).to_string())
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
Ok(collect_merged_target_configs_from_env(
|
||||
config,
|
||||
descriptor.subsystem,
|
||||
descriptor.route_prefix,
|
||||
descriptor.target_type,
|
||||
&valid_fields,
|
||||
env_vars,
|
||||
)?
|
||||
.into_iter()
|
||||
.map(|record| target_plugin_instance_record(descriptor, record))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn target_plugin_instance_record(
|
||||
descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
|
||||
record: MergedTargetConfigRecord,
|
||||
) -> TargetPluginInstanceRecord {
|
||||
TargetPluginInstanceRecord {
|
||||
domain: descriptor.domain,
|
||||
plugin_id: descriptor.plugin_id.to_string(),
|
||||
target_type: descriptor.target_type.to_string(),
|
||||
@@ -123,8 +168,7 @@ where
|
||||
has_env_instance: record.has_env_instance,
|
||||
},
|
||||
effective_config: record.effective_config,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn normalize_legacy_target_instances(
|
||||
@@ -149,8 +193,9 @@ where
|
||||
mod tests {
|
||||
use super::{
|
||||
TargetInstanceSourceClass, TargetPluginInstanceCompatDescriptor, normalize_legacy_target_instances_from_env,
|
||||
normalize_target_plugin_instances_from_env,
|
||||
try_normalize_target_plugin_instances_from_env,
|
||||
};
|
||||
use crate::TargetError;
|
||||
use crate::domain::TargetDomain;
|
||||
use crate::manifest::builtin_target_manifest;
|
||||
use rustfs_config::audit::{AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
|
||||
@@ -338,9 +383,50 @@ mod tests {
|
||||
let descriptor = notify_webhook_descriptor();
|
||||
let env = vec![("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "7".to_string())];
|
||||
|
||||
let canonical = normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone());
|
||||
let canonical = try_normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone())
|
||||
.expect("canonical normalization should succeed");
|
||||
let compatibility = normalize_legacy_target_instances_from_env(&cfg, &descriptor, env);
|
||||
|
||||
assert_eq!(canonical, compatibility);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_instances_rejects_invalid_enable_value() {
|
||||
let error = try_normalize_target_plugin_instances_from_env(
|
||||
&Config(HashMap::new()),
|
||||
¬ify_webhook_descriptor(),
|
||||
vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())],
|
||||
)
|
||||
.expect_err("invalid enable value must be propagated by the public normalizer");
|
||||
|
||||
match error {
|
||||
TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"),
|
||||
other => panic!("expected a configuration error, got {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_normalizer_keeps_valid_instance_when_one_enable_is_invalid() {
|
||||
let instances = normalize_legacy_target_instances_from_env(
|
||||
&Config(HashMap::new()),
|
||||
¬ify_webhook_descriptor(),
|
||||
vec![
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()),
|
||||
],
|
||||
);
|
||||
|
||||
assert_eq!(instances.len(), 2);
|
||||
let good = instances
|
||||
.iter()
|
||||
.find(|instance| instance.instance_id == "good")
|
||||
.expect("valid instance should remain present");
|
||||
let bad = instances
|
||||
.iter()
|
||||
.find(|instance| instance.instance_id == "bad")
|
||||
.expect("invalid legacy instance should remain visible");
|
||||
assert!(good.enabled);
|
||||
assert!(!bad.enabled);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::common::{is_target_enabled, split_env_field_and_instance};
|
||||
use crate::TargetError;
|
||||
use rustfs_config::server_config::{Config, KVS};
|
||||
use rustfs_config::{DEFAULT_DELIMITER, ENV_PREFIX};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
@@ -27,6 +28,31 @@ pub fn collect_target_configs(
|
||||
collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
|
||||
}
|
||||
|
||||
pub fn try_collect_target_configs(
|
||||
config: &Config,
|
||||
route_prefix: &str,
|
||||
target_type: &str,
|
||||
valid_fields: &HashSet<String>,
|
||||
) -> Result<Vec<(String, KVS)>, TargetError> {
|
||||
try_collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
|
||||
}
|
||||
|
||||
/// Collects enabled target instance configs with per-instance fault isolation.
|
||||
///
|
||||
/// Unlike [`try_collect_target_configs`], an instance with an invalid
|
||||
/// configuration (for example an unparseable `enable` value) does not abort the
|
||||
/// whole target type. The enabled `(id, config)` pairs are returned alongside a
|
||||
/// summary line for every instance that could not be collected, so callers can
|
||||
/// load the healthy instances while still surfacing the failed ones.
|
||||
pub fn collect_target_config_results(
|
||||
config: &Config,
|
||||
route_prefix: &str,
|
||||
target_type: &str,
|
||||
valid_fields: &HashSet<String>,
|
||||
) -> (Vec<(String, KVS)>, Vec<String>) {
|
||||
collect_target_config_results_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
|
||||
}
|
||||
|
||||
fn is_sensitive_target_field(field_name: &str) -> bool {
|
||||
let field_name = field_name.to_ascii_lowercase();
|
||||
field_name.contains("password")
|
||||
@@ -58,7 +84,23 @@ fn redact_target_field_value(field_name: &str, value: &str) -> String {
|
||||
return crate::target::mysql::redact_mysql_dsn(value);
|
||||
}
|
||||
if is_sensitive_target_field(field_name) {
|
||||
return "***redacted***".to_string();
|
||||
return crate::target::REDACTED_SECRET.to_string();
|
||||
}
|
||||
if field_name.eq_ignore_ascii_case(rustfs_config::WEBHOOK_ENDPOINT)
|
||||
|| field_name.eq_ignore_ascii_case(rustfs_config::AMQP_URL)
|
||||
{
|
||||
return url::Url::parse(value)
|
||||
.ok()
|
||||
.and_then(|endpoint| {
|
||||
let host = match endpoint.host()? {
|
||||
url::Host::Domain(host) => host.to_string(),
|
||||
url::Host::Ipv4(host) => host.to_string(),
|
||||
url::Host::Ipv6(host) => format!("[{host}]"),
|
||||
};
|
||||
let port = endpoint.port().map(|port| format!(":{port}")).unwrap_or_default();
|
||||
Some(format!("{}://{host}{port}", endpoint.scheme()))
|
||||
})
|
||||
.unwrap_or_else(|| crate::target::REDACTED_SECRET.to_string());
|
||||
}
|
||||
value.to_string()
|
||||
}
|
||||
@@ -123,7 +165,7 @@ pub fn collect_target_configs_from_env<I>(
|
||||
where
|
||||
I: IntoIterator<Item = (String, String)>,
|
||||
{
|
||||
collect_merged_target_configs_from_env(
|
||||
collect_merged_target_configs_compat_from_env(
|
||||
config,
|
||||
&format!("{route_prefix}{target_type}").to_lowercase(),
|
||||
route_prefix,
|
||||
@@ -137,6 +179,59 @@ where
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn try_collect_target_configs_from_env<I>(
|
||||
config: &Config,
|
||||
route_prefix: &str,
|
||||
target_type: &str,
|
||||
valid_fields: &HashSet<String>,
|
||||
env_vars: I,
|
||||
) -> Result<Vec<(String, KVS)>, TargetError>
|
||||
where
|
||||
I: IntoIterator<Item = (String, String)>,
|
||||
{
|
||||
Ok(collect_merged_target_configs_from_env(
|
||||
config,
|
||||
&format!("{route_prefix}{target_type}").to_lowercase(),
|
||||
route_prefix,
|
||||
target_type,
|
||||
valid_fields,
|
||||
env_vars,
|
||||
)?
|
||||
.into_iter()
|
||||
.filter(|record| record.enabled)
|
||||
.map(|record| (record.instance_id, record.effective_config))
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub fn collect_target_config_results_from_env<I>(
|
||||
config: &Config,
|
||||
route_prefix: &str,
|
||||
target_type: &str,
|
||||
valid_fields: &HashSet<String>,
|
||||
env_vars: I,
|
||||
) -> (Vec<(String, KVS)>, Vec<String>)
|
||||
where
|
||||
I: IntoIterator<Item = (String, String)>,
|
||||
{
|
||||
let mut configs = Vec::new();
|
||||
let mut failures = Vec::new();
|
||||
for result in collect_merged_target_config_results_from_env(
|
||||
config,
|
||||
&format!("{route_prefix}{target_type}").to_lowercase(),
|
||||
route_prefix,
|
||||
target_type,
|
||||
valid_fields,
|
||||
env_vars,
|
||||
) {
|
||||
match result {
|
||||
Ok(record) if record.enabled => configs.push((record.instance_id, record.effective_config)),
|
||||
Ok(_) => {}
|
||||
Err((record, err)) => failures.push(format!("{target_type}/{}: {err}", record.instance_id)),
|
||||
}
|
||||
}
|
||||
(configs, failures)
|
||||
}
|
||||
|
||||
pub(crate) fn collect_merged_target_configs_from_env<I>(
|
||||
config: &Config,
|
||||
section_name: &str,
|
||||
@@ -144,7 +239,52 @@ pub(crate) fn collect_merged_target_configs_from_env<I>(
|
||||
target_type: &str,
|
||||
valid_fields: &HashSet<String>,
|
||||
env_vars: I,
|
||||
) -> Result<Vec<MergedTargetConfigRecord>, TargetError>
|
||||
where
|
||||
I: IntoIterator<Item = (String, String)>,
|
||||
{
|
||||
collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars)
|
||||
.into_iter()
|
||||
.map(|result| result.map_err(|(_, err)| err))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn collect_merged_target_configs_compat_from_env<I>(
|
||||
config: &Config,
|
||||
section_name: &str,
|
||||
route_prefix: &str,
|
||||
target_type: &str,
|
||||
valid_fields: &HashSet<String>,
|
||||
env_vars: I,
|
||||
) -> Vec<MergedTargetConfigRecord>
|
||||
where
|
||||
I: IntoIterator<Item = (String, String)>,
|
||||
{
|
||||
collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars)
|
||||
.into_iter()
|
||||
.map(|result| match result {
|
||||
Ok(record) => record,
|
||||
Err((record, err)) => {
|
||||
warn!(
|
||||
target_type,
|
||||
instance_id = %record.instance_id,
|
||||
error = %err,
|
||||
"Treating target instance with invalid enable configuration as disabled"
|
||||
);
|
||||
record
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn collect_merged_target_config_results_from_env<I>(
|
||||
config: &Config,
|
||||
section_name: &str,
|
||||
route_prefix: &str,
|
||||
target_type: &str,
|
||||
valid_fields: &HashSet<String>,
|
||||
env_vars: I,
|
||||
) -> Vec<Result<MergedTargetConfigRecord, (MergedTargetConfigRecord, TargetError)>>
|
||||
where
|
||||
I: IntoIterator<Item = (String, String)>,
|
||||
{
|
||||
@@ -220,14 +360,28 @@ where
|
||||
let redacted_config = redacted_target_config(&merged_config);
|
||||
debug!(instance_id = %id, ?redacted_config, "Merged target configuration");
|
||||
}
|
||||
merged_configs.push(MergedTargetConfigRecord {
|
||||
instance_id: id,
|
||||
enabled: is_target_enabled(&merged_config),
|
||||
effective_config: merged_config,
|
||||
has_file_default,
|
||||
has_file_instance,
|
||||
has_env_default,
|
||||
has_env_instance,
|
||||
merged_configs.push(match is_target_enabled(&merged_config) {
|
||||
Ok(enabled) => Ok(MergedTargetConfigRecord {
|
||||
instance_id: id,
|
||||
enabled,
|
||||
effective_config: merged_config,
|
||||
has_file_default,
|
||||
has_file_instance,
|
||||
has_env_default,
|
||||
has_env_instance,
|
||||
}),
|
||||
Err(err) => Err((
|
||||
MergedTargetConfigRecord {
|
||||
instance_id: id,
|
||||
enabled: false,
|
||||
effective_config: merged_config,
|
||||
has_file_default,
|
||||
has_file_instance,
|
||||
has_env_default,
|
||||
has_env_instance,
|
||||
},
|
||||
err,
|
||||
)),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -237,9 +391,10 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
collect_env_target_instance_ids_from_env, collect_target_configs_from_env, redact_target_field_value,
|
||||
redacted_target_config,
|
||||
collect_env_target_instance_ids_from_env, collect_target_config_results_from_env, collect_target_configs_from_env,
|
||||
redact_target_field_value, redacted_target_config, try_collect_target_configs_from_env,
|
||||
};
|
||||
use crate::TargetError;
|
||||
use rustfs_config::notify::{
|
||||
ENV_NOTIFY_REDIS_ENABLE, ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS, ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE,
|
||||
ENV_NOTIFY_REDIS_URL, NOTIFY_REDIS_KEYS, NOTIFY_ROUTE_PREFIX,
|
||||
@@ -269,7 +424,7 @@ mod tests {
|
||||
|
||||
cfg.0.insert("notify_webhook".to_string(), subsystem);
|
||||
|
||||
let configs = collect_target_configs_from_env(
|
||||
let configs = try_collect_target_configs_from_env(
|
||||
&cfg,
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
"webhook",
|
||||
@@ -282,7 +437,8 @@ mod tests {
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENABLE".to_string(), "on".to_string()),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "42".to_string()),
|
||||
],
|
||||
);
|
||||
)
|
||||
.expect("valid env target");
|
||||
|
||||
let configs: HashMap<String, KVS> = configs.into_iter().collect();
|
||||
assert_eq!(configs.len(), 2);
|
||||
@@ -295,7 +451,7 @@ mod tests {
|
||||
#[test]
|
||||
fn collect_target_configs_discovers_enabled_instance_from_env() {
|
||||
let cfg = Config(HashMap::new());
|
||||
let configs = collect_target_configs_from_env(
|
||||
let configs = try_collect_target_configs_from_env(
|
||||
&cfg,
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
"webhook",
|
||||
@@ -307,7 +463,8 @@ mod tests {
|
||||
"https://example.com/from-env".to_string(),
|
||||
),
|
||||
],
|
||||
);
|
||||
)
|
||||
.expect("valid target configs");
|
||||
|
||||
assert_eq!(configs.len(), 1);
|
||||
assert_eq!(configs[0].0, "primary");
|
||||
@@ -323,7 +480,7 @@ mod tests {
|
||||
subsystem.insert("_".to_string(), default_kvs);
|
||||
cfg.0.insert("notify_webhook".to_string(), subsystem);
|
||||
|
||||
let configs = collect_target_configs_from_env(
|
||||
let configs = try_collect_target_configs_from_env(
|
||||
&cfg,
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
"webhook",
|
||||
@@ -332,7 +489,8 @@ mod tests {
|
||||
"RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SECONDARY".to_string(),
|
||||
"https://example.com/secondary".to_string(),
|
||||
)],
|
||||
);
|
||||
)
|
||||
.expect("valid target configs");
|
||||
|
||||
assert!(configs.is_empty());
|
||||
}
|
||||
@@ -361,7 +519,7 @@ mod tests {
|
||||
let cfg = Config(HashMap::new());
|
||||
let valid_fields = NOTIFY_REDIS_KEYS.iter().map(|key| (*key).to_string()).collect();
|
||||
|
||||
let configs = collect_target_configs_from_env(
|
||||
let configs = try_collect_target_configs_from_env(
|
||||
&cfg,
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
"redis",
|
||||
@@ -372,7 +530,8 @@ mod tests {
|
||||
(format!("{ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS}_PRIMARY"), "9".to_string()),
|
||||
(format!("{ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE}_PRIMARY"), "off".to_string()),
|
||||
],
|
||||
);
|
||||
)
|
||||
.expect("valid redis target config");
|
||||
|
||||
let configs: HashMap<String, KVS> = configs.into_iter().collect();
|
||||
let redis_config = configs.get("primary").expect("redis env target should be discovered");
|
||||
@@ -383,6 +542,99 @@ mod tests {
|
||||
assert_eq!(redis_config.lookup(REDIS_TLS_ALLOW_INSECURE).as_deref(), Some("off"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_target_configs_rejects_invalid_instance_enable_value() {
|
||||
let err = try_collect_target_configs_from_env(
|
||||
&Config(HashMap::new()),
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
"webhook",
|
||||
&HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
|
||||
vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())],
|
||||
)
|
||||
.expect_err("invalid enable value must not look like a disabled target");
|
||||
|
||||
match err {
|
||||
TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"),
|
||||
other => panic!("expected a configuration error, got {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_collection_keeps_valid_instances_when_one_enable_is_invalid() {
|
||||
let configs = collect_target_configs_from_env(
|
||||
&Config(HashMap::new()),
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
"webhook",
|
||||
&HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
|
||||
vec![
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()),
|
||||
],
|
||||
);
|
||||
|
||||
assert_eq!(configs.len(), 1);
|
||||
assert_eq!(configs[0].0, "good");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_target_config_results_isolates_invalid_instance_and_reports_it() {
|
||||
let (configs, failures) = collect_target_config_results_from_env(
|
||||
&Config(HashMap::new()),
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
"webhook",
|
||||
&HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
|
||||
vec![
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
|
||||
// "enable" is a common typo: EnableState accepts "enabled"/"on", not "enable".
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "enable".to_string()),
|
||||
],
|
||||
);
|
||||
|
||||
// The healthy instance still loads even though a sibling is malformed...
|
||||
assert_eq!(configs.len(), 1);
|
||||
assert_eq!(configs[0].0, "good");
|
||||
// ...and the malformed instance is surfaced (not silently dropped, not
|
||||
// aborting the whole target type) so callers can fail-report it.
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert!(
|
||||
failures[0].contains("webhook/bad") && failures[0].contains("enable"),
|
||||
"unexpected failure summary: {}",
|
||||
failures[0]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_target_config_results_does_not_report_validly_disabled_instances() {
|
||||
let (configs, failures) = collect_target_config_results_from_env(
|
||||
&Config(HashMap::new()),
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
"webhook",
|
||||
&HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
|
||||
vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "off".to_string())],
|
||||
);
|
||||
|
||||
assert!(configs.is_empty());
|
||||
assert!(failures.is_empty(), "a validly disabled instance is not a failure");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_target_configs_preserves_whitespace_padded_legacy_value() {
|
||||
let configs = try_collect_target_configs_from_env(
|
||||
&Config(HashMap::new()),
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
"webhook",
|
||||
&HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
|
||||
vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), " on ".to_string())],
|
||||
)
|
||||
.expect("the shared enable parser accepts surrounding whitespace");
|
||||
|
||||
assert_eq!(configs.len(), 1);
|
||||
assert_eq!(configs[0].0, "primary");
|
||||
assert_eq!(configs[0].1.lookup(ENABLE_KEY).as_deref(), Some(" on "));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_target_field_value_redacts_sensitive_fields() {
|
||||
assert_eq!(redact_target_field_value("password", "secret"), "***redacted***");
|
||||
@@ -396,6 +648,24 @@ mod tests {
|
||||
assert_eq!(redact_target_field_value("queue_limit", "1000"), "1000");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_target_field_value_strips_endpoint_path_and_query() {
|
||||
assert_eq!(
|
||||
redact_target_field_value("endpoint", "https://example.com/private/hook?token=secret"),
|
||||
"https://example.com"
|
||||
);
|
||||
assert_eq!(redact_target_field_value("endpoint", "not a URL with secret"), "***redacted***");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_target_field_value_strips_url_credentials() {
|
||||
assert_eq!(
|
||||
redact_target_field_value("url", "amqps://user:secret@broker.example/vhost"),
|
||||
"amqps://broker.example"
|
||||
);
|
||||
assert_eq!(redact_target_field_value("url", "not a URL with secret"), "***redacted***");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_dsn_string_partial_redaction() {
|
||||
let dsn = "rustfs:secret123@tcp(mysql.example.com:3306)/rustfs_events";
|
||||
@@ -435,7 +705,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
redacted,
|
||||
vec![
|
||||
("endpoint".to_string(), "https://example.com/hook".to_string()),
|
||||
("endpoint".to_string(), "https://example.com".to_string()),
|
||||
("password".to_string(), "***redacted***".to_string()),
|
||||
("client_key".to_string(), "***redacted***".to_string()),
|
||||
("auth_token".to_string(), "***redacted***".to_string()),
|
||||
|
||||
@@ -21,10 +21,12 @@ pub use instance::{
|
||||
LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
|
||||
TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
|
||||
normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
|
||||
try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env,
|
||||
};
|
||||
pub use loader::{
|
||||
collect_env_target_instance_ids, collect_env_target_instance_ids_from_env, collect_target_configs,
|
||||
collect_target_configs_from_env,
|
||||
collect_env_target_instance_ids, collect_env_target_instance_ids_from_env, collect_target_config_results,
|
||||
collect_target_config_results_from_env, collect_target_configs, collect_target_configs_from_env, try_collect_target_configs,
|
||||
try_collect_target_configs_from_env,
|
||||
};
|
||||
pub use target_args::{
|
||||
build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,
|
||||
|
||||
@@ -43,6 +43,7 @@ pub use config::{
|
||||
LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
|
||||
TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
|
||||
normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
|
||||
try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env,
|
||||
};
|
||||
pub use control_plane::{
|
||||
TargetPluginEnableState, TargetPluginExternalAction, TargetPluginExternalActionDecision, TargetPluginExternalActionError,
|
||||
@@ -65,10 +66,11 @@ pub use plugin::{
|
||||
TargetPluginRegistry, TargetRequestValidator, boxed_target,
|
||||
};
|
||||
pub use runtime::{
|
||||
ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
|
||||
RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, activate_targets_with_replay,
|
||||
OpenedActivation, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot,
|
||||
RuntimeTargetHealthReason, RuntimeTargetHealthSnapshot, RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget,
|
||||
TargetRuntimeManager, activate_targets_with_replay,
|
||||
adapter::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter},
|
||||
init_target_and_optionally_start_replay,
|
||||
health_snapshots_for_targets, init_target_and_optionally_start_replay,
|
||||
ops_diagnostics::{
|
||||
OpsDiagnosticsAccessDecision, OpsDiagnosticsReadRequest, OpsDiagnosticsRegistration, OpsDiagnosticsRegistry,
|
||||
OpsDiagnosticsRegistryError,
|
||||
@@ -85,6 +87,7 @@ pub use rustfs_s3_types::EventName;
|
||||
use serde::{Deserialize, Serialize};
|
||||
pub use sys::user_agent::*;
|
||||
pub use target::{Target, TargetDeliverySnapshot};
|
||||
pub use target::{TargetHealth, TargetHealthReason, TargetHealthState};
|
||||
|
||||
/// Represents a log of events for sending to targets
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
||||
+107
-11
@@ -14,8 +14,9 @@
|
||||
|
||||
use crate::{
|
||||
PluginRuntimeAdapter, RuntimeActivation, Target, TargetError,
|
||||
config::collect_target_configs,
|
||||
config::collect_target_config_results,
|
||||
manifest::{TargetPluginManifest, builtin_target_manifest},
|
||||
target::with_deferred_queue_store_open,
|
||||
};
|
||||
use hashbrown::HashMap;
|
||||
use rustfs_config::server_config::{Config, KVS};
|
||||
@@ -317,39 +318,77 @@ where
|
||||
config: &Config,
|
||||
route_prefix: &str,
|
||||
) -> Result<Vec<BoxedTarget<E>>, TargetError> {
|
||||
self.create_targets_from_config_with_store_mode(config, route_prefix, false)
|
||||
.await
|
||||
.map(|(targets, _)| targets)
|
||||
}
|
||||
|
||||
/// Creates targets while deferring queue-store open until runtime handoff.
|
||||
/// Unlike the compatibility activation API, lifecycle preparation reports
|
||||
/// any invalid or unconstructable configured instance so the originating
|
||||
/// Admin request cannot report a false success.
|
||||
pub async fn create_dormant_targets_from_config(
|
||||
&self,
|
||||
config: &Config,
|
||||
route_prefix: &str,
|
||||
) -> Result<(Vec<BoxedTarget<E>>, Vec<String>), TargetError> {
|
||||
self.create_targets_from_config_with_store_mode(config, route_prefix, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn create_targets_from_config_with_store_mode(
|
||||
&self,
|
||||
config: &Config,
|
||||
route_prefix: &str,
|
||||
defer_store_open: bool,
|
||||
) -> Result<(Vec<BoxedTarget<E>>, Vec<String>), TargetError> {
|
||||
let mut successful_targets = Vec::new();
|
||||
let mut failed_targets = 0usize;
|
||||
let mut failures = Vec::new();
|
||||
|
||||
for (target_type, plugin) in &self.plugins {
|
||||
info!(target_type = %target_type, "Start working on target type");
|
||||
for (id, merged_config) in collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set()) {
|
||||
// Per-instance fault isolation: an invalid instance (e.g. an
|
||||
// unparseable `enable` value) is recorded as a failure and skipped,
|
||||
// never aborting the remaining instances or other target types.
|
||||
let (collected, invalid_instances) =
|
||||
collect_target_config_results(config, route_prefix, target_type, plugin.valid_fields_set());
|
||||
for detail in invalid_instances {
|
||||
error!(target_type = %target_type, reason = "invalid_config", detail = %detail, "Skipping target instance with invalid configuration");
|
||||
failures.push(detail);
|
||||
}
|
||||
for (id, merged_config) in collected {
|
||||
info!(target_type = %target_type, instance_id = %id, "Target is enabled, ready to create");
|
||||
match self.create_target(target_type, id.clone(), &merged_config) {
|
||||
let created = if defer_store_open {
|
||||
with_deferred_queue_store_open(|| self.create_target(target_type, id.clone(), &merged_config))
|
||||
} else {
|
||||
self.create_target(target_type, id.clone(), &merged_config)
|
||||
};
|
||||
match created {
|
||||
Ok(target) => {
|
||||
info!(target_type = %target.id().name, instance_id = %id, "Create target successfully");
|
||||
successful_targets.push(target);
|
||||
}
|
||||
Err(err) => {
|
||||
failed_targets += 1;
|
||||
error!(target_type = %target_type, instance_id = %id, error = %err, "Failed to create target");
|
||||
Err(_) => {
|
||||
failures.push(format!("{target_type}/{id}: target construction failed"));
|
||||
error!(target_type = %target_type, instance_id = %id, reason = "construction_failed", "Failed to create target");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if failed_targets > 0 {
|
||||
if !failures.is_empty() {
|
||||
warn!(
|
||||
created = successful_targets.len(),
|
||||
failed = failed_targets,
|
||||
failed = failures.len(),
|
||||
"Some configured targets failed to create and were skipped"
|
||||
);
|
||||
}
|
||||
info!(
|
||||
count = successful_targets.len(),
|
||||
failed = failed_targets,
|
||||
failed = failures.len(),
|
||||
"All target processing completed"
|
||||
);
|
||||
Ok(successful_targets)
|
||||
Ok((successful_targets, failures))
|
||||
}
|
||||
|
||||
pub async fn create_activation_from_config<A>(
|
||||
@@ -475,4 +514,61 @@ mod tests {
|
||||
assert_eq!(activation.targets[0].id().to_string(), "primary:test");
|
||||
assert!(activation.replay_workers.is_empty());
|
||||
}
|
||||
|
||||
// Regression: a single instance with a malformed `enable` value must not
|
||||
// abort the remaining instances or unrelated target types. Before this fix
|
||||
// the collector short-circuited the whole create path, so one typo took
|
||||
// down every notify/audit target.
|
||||
#[tokio::test]
|
||||
async fn create_dormant_isolates_invalid_enable_and_still_loads_other_targets() {
|
||||
let mut registry = TargetPluginRegistry::<String>::new();
|
||||
for target_type in ["alpha", "beta"] {
|
||||
registry.register(TargetPluginDescriptor::new(
|
||||
target_type,
|
||||
&[ENABLE_KEY, "endpoint"],
|
||||
|_config| Ok(()),
|
||||
move |id, _config| {
|
||||
Ok(Box::new(TestTarget {
|
||||
id: crate::arn::TargetID::new(id, target_type.to_string()),
|
||||
}))
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
let mut cfg = Config(HashMap::new());
|
||||
|
||||
// alpha: one healthy instance plus one with a malformed `enable` value
|
||||
// ("enable" is a typo -- EnableState accepts "enabled"/"on", not "enable").
|
||||
let mut alpha = HashMap::new();
|
||||
let mut alpha_good = KVS::new();
|
||||
alpha_good.insert(ENABLE_KEY.to_string(), "on".to_string());
|
||||
alpha_good.insert("endpoint".to_string(), "https://example.com/alpha".to_string());
|
||||
alpha.insert("good".to_string(), alpha_good);
|
||||
let mut alpha_bad = KVS::new();
|
||||
alpha_bad.insert(ENABLE_KEY.to_string(), "enable".to_string());
|
||||
alpha.insert("bad".to_string(), alpha_bad);
|
||||
cfg.0.insert("notify_alpha".to_string(), alpha);
|
||||
|
||||
// beta: a healthy instance in a different target type must survive.
|
||||
let mut beta = HashMap::new();
|
||||
let mut beta_primary = KVS::new();
|
||||
beta_primary.insert(ENABLE_KEY.to_string(), "on".to_string());
|
||||
beta_primary.insert("endpoint".to_string(), "https://example.com/beta".to_string());
|
||||
beta.insert("primary".to_string(), beta_primary);
|
||||
cfg.0.insert("notify_beta".to_string(), beta);
|
||||
|
||||
let (targets, failures) = registry
|
||||
.create_dormant_targets_from_config(&cfg, "notify_")
|
||||
.await
|
||||
.expect("a malformed instance must not abort target creation");
|
||||
|
||||
let mut created: Vec<String> = targets.iter().map(|target| target.id().to_string()).collect();
|
||||
created.sort();
|
||||
assert_eq!(created, vec!["good:alpha".to_string(), "primary:beta".to_string()]);
|
||||
|
||||
// The malformed instance is surfaced (so an Admin write can't report a
|
||||
// false success) rather than silently dropped or fatally aborting.
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert!(failures[0].contains("alpha/bad"), "unexpected failure summary: {}", failures[0]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,21 +13,53 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::{
|
||||
ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
|
||||
TargetRuntimeManager, activate_targets_with_replay, init_target_and_optionally_start_replay, start_replay_worker,
|
||||
OpenedActivation, PrepareTargetResult, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation,
|
||||
RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot, TargetActivationFailure, TargetRuntimeManager, prepare_target,
|
||||
start_replay_worker,
|
||||
};
|
||||
use crate::plugin::PluginEvent;
|
||||
use crate::{Target, TargetError};
|
||||
use crate::{SharedTarget, Target, TargetError};
|
||||
use async_trait::async_trait;
|
||||
use rayon::prelude::*;
|
||||
use std::future::Future;
|
||||
use std::panic::{AssertUnwindSafe, catch_unwind};
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
|
||||
type ReplayStartObserver = Arc<dyn Fn(&str, bool) + Send + Sync>;
|
||||
|
||||
const MAX_PARALLEL_STORE_OPENS: usize = 4;
|
||||
|
||||
static STORE_OPEN_POOL: LazyLock<Result<rayon::ThreadPool, rayon::ThreadPoolBuildError>> = LazyLock::new(|| {
|
||||
rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(MAX_PARALLEL_STORE_OPENS)
|
||||
.thread_name(|index| format!("rustfs-target-store-open-{index}"))
|
||||
.build()
|
||||
});
|
||||
|
||||
enum StoreOpenOutcome<E>
|
||||
where
|
||||
E: PluginEvent,
|
||||
{
|
||||
Accepted(SharedTarget<E>),
|
||||
Rejected { panicked: bool, target: SharedTarget<E> },
|
||||
}
|
||||
|
||||
fn open_target_store<E>(target: SharedTarget<E>) -> StoreOpenOutcome<E>
|
||||
where
|
||||
E: PluginEvent,
|
||||
{
|
||||
match catch_unwind(AssertUnwindSafe(|| target.store().map(|store| store.open()))) {
|
||||
Ok(None | Some(Ok(()))) => StoreOpenOutcome::Accepted(target),
|
||||
Ok(Some(Err(_))) => StoreOpenOutcome::Rejected { panicked: false, target },
|
||||
Err(_) => StoreOpenOutcome::Rejected { panicked: true, target },
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared runtime contract for target plugins.
|
||||
#[async_trait]
|
||||
pub trait PluginRuntimeAdapter<E>: Send + Sync
|
||||
@@ -96,6 +128,234 @@ where
|
||||
stop_log_prefix: stop_log_prefix.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn prepare_targets(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> PreparedActivation<E> {
|
||||
self.prepare_targets_inner(targets, None).await
|
||||
}
|
||||
|
||||
pub async fn prepare_targets_cancellable(
|
||||
&self,
|
||||
targets: Vec<Box<dyn Target<E> + Send + Sync>>,
|
||||
cancellation: &CancellationToken,
|
||||
) -> PreparedActivation<E> {
|
||||
self.prepare_targets_inner(targets, Some(cancellation)).await
|
||||
}
|
||||
|
||||
async fn prepare_targets_inner(
|
||||
&self,
|
||||
targets: Vec<Box<dyn Target<E> + Send + Sync>>,
|
||||
cancellation: Option<&CancellationToken>,
|
||||
) -> PreparedActivation<E> {
|
||||
let mut prepared = Vec::with_capacity(targets.len());
|
||||
let mut failures = Vec::new();
|
||||
let mut rejected_targets = Vec::new();
|
||||
let mut targets = targets.into_iter();
|
||||
while let Some(target) = targets.next() {
|
||||
match prepare_target(target, cancellation).await {
|
||||
PrepareTargetResult::Ready(target) => prepared.push(target),
|
||||
PrepareTargetResult::Degraded { error, target } => {
|
||||
drop(error);
|
||||
tracing::warn!(
|
||||
target_id = %target.id(),
|
||||
reason = "initialization_failed",
|
||||
"Target initialization failed during lifecycle preparation"
|
||||
);
|
||||
failures.push(TargetActivationFailure {
|
||||
detail: format!("{}: initialization failed", target.id()),
|
||||
});
|
||||
prepared.push(target);
|
||||
}
|
||||
PrepareTargetResult::Failed { error, target } => {
|
||||
drop(error);
|
||||
let target_id = target.id().to_string();
|
||||
tracing::warn!(
|
||||
target_id,
|
||||
reason = "initialization_failed",
|
||||
"Target initialization failed during lifecycle preparation"
|
||||
);
|
||||
failures.push(TargetActivationFailure {
|
||||
detail: format!("{target_id}: initialization failed"),
|
||||
});
|
||||
rejected_targets.push(Arc::from(target));
|
||||
}
|
||||
PrepareTargetResult::Cancelled(target) => {
|
||||
prepared.push(Arc::from(target));
|
||||
prepared.extend(targets.map(Arc::from));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
PreparedActivation {
|
||||
failures,
|
||||
rejected_targets,
|
||||
targets: prepared,
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens queue stores only after the previous runtime generation has been
|
||||
/// quiesced. Targets whose stores cannot be opened retain the established
|
||||
/// fault-isolation behavior and are returned for lock-free shutdown.
|
||||
pub fn open_prepared_stores(&self, prepared: PreparedActivation<E>) -> (OpenedActivation<E>, PreparedActivation<E>) {
|
||||
let mut accepted = Vec::with_capacity(prepared.targets.len());
|
||||
let mut failures = prepared.failures;
|
||||
let mut rejected = prepared.rejected_targets;
|
||||
let outcomes = if prepared.targets.len() < 2 {
|
||||
prepared.targets.into_iter().map(open_target_store).collect()
|
||||
} else {
|
||||
match STORE_OPEN_POOL.as_ref() {
|
||||
// Vec's indexed parallel iterator preserves configuration
|
||||
// order in collect, keeping failure summaries deterministic.
|
||||
Ok(pool) => pool.install(|| prepared.targets.into_par_iter().map(open_target_store).collect::<Vec<_>>()),
|
||||
Err(err) => {
|
||||
tracing::warn!(error = %err, "Failed to create target store open pool; opening stores serially");
|
||||
prepared.targets.into_iter().map(open_target_store).collect()
|
||||
}
|
||||
}
|
||||
};
|
||||
for outcome in outcomes {
|
||||
match outcome {
|
||||
StoreOpenOutcome::Accepted(target) => accepted.push(target),
|
||||
StoreOpenOutcome::Rejected { panicked, target } => {
|
||||
if panicked {
|
||||
tracing::error!(
|
||||
target_id = %target.id(),
|
||||
reason = "store_open_panicked",
|
||||
"Target queue store panicked while opening during runtime handoff"
|
||||
);
|
||||
} else {
|
||||
tracing::error!(
|
||||
target_id = %target.id(),
|
||||
reason = "store_open_failed",
|
||||
"Failed to open target queue store during runtime handoff"
|
||||
);
|
||||
}
|
||||
failures.push(TargetActivationFailure {
|
||||
detail: format!("{}: queue store open failed", target.id()),
|
||||
});
|
||||
rejected.push(target);
|
||||
}
|
||||
}
|
||||
}
|
||||
(
|
||||
OpenedActivation { targets: accepted },
|
||||
PreparedActivation {
|
||||
failures,
|
||||
rejected_targets: rejected,
|
||||
targets: Vec::new(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub fn try_activate_prepared(&self, opened: OpenedActivation<E>) -> (RuntimeActivation<E>, PreparedActivation<E>) {
|
||||
let mut replay_workers = ReplayWorkerManager::new();
|
||||
let mut accepted = Vec::with_capacity(opened.targets.len());
|
||||
let mut failures = Vec::new();
|
||||
let mut rejected_targets = Vec::new();
|
||||
for target in opened.targets {
|
||||
let target_id = target.id().to_string();
|
||||
let replay = catch_unwind(AssertUnwindSafe(|| {
|
||||
target.store().filter(|_| target.is_enabled()).map(|store| {
|
||||
start_replay_worker(
|
||||
store.boxed_clone(),
|
||||
Arc::clone(&target),
|
||||
Arc::clone(&self.replay_hook),
|
||||
self.replay_semaphore.clone(),
|
||||
self.batch_timeout,
|
||||
self.idle_sleep,
|
||||
)
|
||||
})
|
||||
}));
|
||||
let replay = match replay {
|
||||
Ok(replay) => replay,
|
||||
Err(_) => {
|
||||
tracing::error!(
|
||||
target_id,
|
||||
reason = "replay_activation_panicked",
|
||||
"Target replay activation panicked during runtime handoff"
|
||||
);
|
||||
failures.push(TargetActivationFailure {
|
||||
detail: format!("{target_id}: replay activation failed"),
|
||||
});
|
||||
rejected_targets.push(target);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
(self.replay_start_observer)(&target_id, replay.is_some());
|
||||
if let Some((cancel_tx, join)) = replay {
|
||||
replay_workers.insert_with_handle(target_id, cancel_tx, join);
|
||||
}
|
||||
accepted.push(target);
|
||||
}
|
||||
|
||||
(
|
||||
RuntimeActivation {
|
||||
replay_workers,
|
||||
targets: accepted,
|
||||
},
|
||||
PreparedActivation {
|
||||
failures,
|
||||
rejected_targets,
|
||||
targets: Vec::new(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub async fn prepare_dormant_compat_activation(
|
||||
&self,
|
||||
targets: Vec<Box<dyn Target<E> + Send + Sync>>,
|
||||
) -> RuntimeActivation<E> {
|
||||
let PreparedActivation {
|
||||
failures,
|
||||
rejected_targets,
|
||||
targets,
|
||||
} = self.prepare_targets(targets).await;
|
||||
let rejected = PreparedActivation {
|
||||
failures,
|
||||
rejected_targets,
|
||||
targets: Vec::new(),
|
||||
};
|
||||
if let Err(err) = self.close_prepared(rejected).await {
|
||||
tracing::warn!(error = %err, "Failed to close targets rejected while preparing compatibility activation");
|
||||
}
|
||||
RuntimeActivation {
|
||||
replay_workers: ReplayWorkerManager::new(),
|
||||
targets,
|
||||
}
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn start_dormant_compat_activation(
|
||||
&self,
|
||||
activation: RuntimeActivation<E>,
|
||||
) -> (RuntimeActivation<E>, PreparedActivation<E>, PreparedActivation<E>) {
|
||||
let prepared = PreparedActivation {
|
||||
failures: Vec::new(),
|
||||
rejected_targets: Vec::new(),
|
||||
targets: activation.targets,
|
||||
};
|
||||
let (opened, open_rejected) = self.open_prepared_stores(prepared);
|
||||
let (activation, activation_rejected) = self.try_activate_prepared(opened);
|
||||
(activation, open_rejected, activation_rejected)
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub async fn close_compat_activation(&self, mut activation: RuntimeActivation<E>) -> Result<(), TargetError> {
|
||||
let mut runtime = TargetRuntimeManager::new();
|
||||
for target in activation.targets {
|
||||
runtime.add_arc(target);
|
||||
}
|
||||
self.shutdown(&mut runtime, &mut activation.replay_workers).await
|
||||
}
|
||||
|
||||
pub async fn close_prepared(&self, prepared: PreparedActivation<E>) -> Result<(), TargetError> {
|
||||
let mut runtime = TargetRuntimeManager::new();
|
||||
for target in prepared.targets.into_iter().chain(prepared.rejected_targets) {
|
||||
runtime.add_arc(target);
|
||||
}
|
||||
let mut replay_workers = ReplayWorkerManager::new();
|
||||
self.shutdown(&mut runtime, &mut replay_workers).await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -104,36 +364,16 @@ where
|
||||
E: PluginEvent,
|
||||
{
|
||||
async fn activate_with_replay(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> RuntimeActivation<E> {
|
||||
let replay_hook = Arc::clone(&self.replay_hook);
|
||||
let replay_start_observer = Arc::clone(&self.replay_start_observer);
|
||||
let replay_semaphore = self.replay_semaphore.clone();
|
||||
let batch_timeout = self.batch_timeout;
|
||||
let idle_sleep = self.idle_sleep;
|
||||
|
||||
activate_targets_with_replay(targets, move |target| {
|
||||
let replay_hook = Arc::clone(&replay_hook);
|
||||
let replay_start_observer = Arc::clone(&replay_start_observer);
|
||||
let replay_semaphore = replay_semaphore.clone();
|
||||
|
||||
async move {
|
||||
init_target_and_optionally_start_replay(
|
||||
target,
|
||||
move |target_id, has_replay| replay_start_observer(target_id, has_replay),
|
||||
move |store, target| {
|
||||
start_replay_worker(
|
||||
store,
|
||||
target,
|
||||
Arc::clone(&replay_hook),
|
||||
replay_semaphore.clone(),
|
||||
batch_timeout,
|
||||
idle_sleep,
|
||||
)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
})
|
||||
.await
|
||||
let prepared = self.prepare_targets(targets).await;
|
||||
let (opened, rejected) = self.open_prepared_stores(prepared);
|
||||
if let Err(err) = self.close_prepared(rejected).await {
|
||||
tracing::warn!(error = %err, "Failed to close targets whose queue stores could not be opened");
|
||||
}
|
||||
let (activation, rejected) = self.try_activate_prepared(opened);
|
||||
if let Err(err) = self.close_prepared(rejected).await {
|
||||
tracing::warn!(error = %err, "Failed to close targets rejected during replay activation");
|
||||
}
|
||||
activation
|
||||
}
|
||||
|
||||
async fn replace_runtime_targets(
|
||||
@@ -189,7 +429,7 @@ where
|
||||
if !close_errors.is_empty() {
|
||||
let detail = close_errors
|
||||
.into_iter()
|
||||
.map(|(target_id, err)| format!("{target_id}: {err}"))
|
||||
.map(|(target_id, _)| target_id)
|
||||
.collect::<Vec<_>>()
|
||||
.join("; ");
|
||||
return Err(TargetError::Storage(format!("Failed to close {detail}")));
|
||||
@@ -200,24 +440,143 @@ where
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter};
|
||||
use super::{BuiltinPluginRuntimeAdapter, MAX_PARALLEL_STORE_OPENS, PluginRuntimeAdapter};
|
||||
use crate::PluginEvent;
|
||||
use crate::arn::TargetID;
|
||||
use crate::store::{Key, QueueStore, Store};
|
||||
use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
|
||||
use crate::{StoreError, Target, TargetError};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
use std::time::Duration;
|
||||
use tempfile::tempdir;
|
||||
use tokio::sync::Notify;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
type TestStore = dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync;
|
||||
|
||||
type BeforeOpen = Arc<dyn Fn() + Send + Sync>;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestOpenStore {
|
||||
before_clone: BeforeOpen,
|
||||
before_open: BeforeOpen,
|
||||
store: QueueStore<QueuedPayload>,
|
||||
}
|
||||
|
||||
impl Store<QueuedPayload> for TestOpenStore {
|
||||
type Error = StoreError;
|
||||
type Key = Key;
|
||||
|
||||
fn open(&self) -> Result<(), Self::Error> {
|
||||
(self.before_open)();
|
||||
self.store.open()
|
||||
}
|
||||
|
||||
fn put(&self, item: Arc<QueuedPayload>) -> Result<Self::Key, Self::Error> {
|
||||
self.store.put(item)
|
||||
}
|
||||
|
||||
fn put_multiple(&self, items: Vec<QueuedPayload>) -> Result<Self::Key, Self::Error> {
|
||||
self.store.put_multiple(items)
|
||||
}
|
||||
|
||||
fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
|
||||
self.store.put_raw(data)
|
||||
}
|
||||
|
||||
fn get(&self, key: &Self::Key) -> Result<QueuedPayload, Self::Error> {
|
||||
self.store.get(key)
|
||||
}
|
||||
|
||||
fn get_multiple(&self, key: &Self::Key) -> Result<Vec<QueuedPayload>, Self::Error> {
|
||||
self.store.get_multiple(key)
|
||||
}
|
||||
|
||||
fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
|
||||
self.store.get_raw(key)
|
||||
}
|
||||
|
||||
fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
|
||||
self.store.del(key)
|
||||
}
|
||||
|
||||
fn delete(&self) -> Result<(), Self::Error> {
|
||||
self.store.delete()
|
||||
}
|
||||
|
||||
fn list(&self) -> Vec<Self::Key> {
|
||||
self.store.list()
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.store.len()
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
self.store.is_empty()
|
||||
}
|
||||
|
||||
fn boxed_clone(&self) -> Box<dyn Store<QueuedPayload, Error = Self::Error, Key = Self::Key> + Send + Sync> {
|
||||
(self.before_clone)();
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct StoreOpenGate {
|
||||
changed: Condvar,
|
||||
state: Mutex<StoreOpenGateState>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct StoreOpenGateState {
|
||||
active: usize,
|
||||
max_active: usize,
|
||||
released: bool,
|
||||
}
|
||||
|
||||
impl StoreOpenGate {
|
||||
fn enter(&self) {
|
||||
let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
|
||||
state.active += 1;
|
||||
state.max_active = state.max_active.max(state.active);
|
||||
self.changed.notify_all();
|
||||
while !state.released {
|
||||
state = self.changed.wait(state).unwrap_or_else(|err| err.into_inner());
|
||||
}
|
||||
state.active -= 1;
|
||||
}
|
||||
|
||||
fn wait_for_active(&self, expected: usize, timeout: Duration) -> bool {
|
||||
let state = self.state.lock().unwrap_or_else(|err| err.into_inner());
|
||||
let (state, _) = self
|
||||
.changed
|
||||
.wait_timeout_while(state, timeout, |state| state.max_active < expected)
|
||||
.unwrap_or_else(|err| err.into_inner());
|
||||
state.max_active >= expected
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
|
||||
state.released = true;
|
||||
self.changed.notify_all();
|
||||
}
|
||||
|
||||
fn max_active(&self) -> usize {
|
||||
self.state.lock().unwrap_or_else(|err| err.into_inner()).max_active
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestTarget {
|
||||
close_calls: Arc<AtomicUsize>,
|
||||
id: TargetID,
|
||||
init_calls: Arc<AtomicUsize>,
|
||||
init_entered: Option<Arc<Notify>>,
|
||||
init_fails: bool,
|
||||
store: Option<QueueStore<QueuedPayload>>,
|
||||
store: Option<Arc<TestStore>>,
|
||||
}
|
||||
|
||||
impl TestTarget {
|
||||
@@ -225,6 +584,8 @@ mod tests {
|
||||
Self {
|
||||
close_calls: Arc::new(AtomicUsize::new(0)),
|
||||
id: TargetID::new(id.to_string(), name.to_string()),
|
||||
init_calls: Arc::new(AtomicUsize::new(0)),
|
||||
init_entered: None,
|
||||
init_fails: false,
|
||||
store: None,
|
||||
}
|
||||
@@ -235,11 +596,16 @@ mod tests {
|
||||
self
|
||||
}
|
||||
|
||||
fn with_pending_init(mut self, init_entered: Arc<Notify>) -> Self {
|
||||
self.init_entered = Some(init_entered);
|
||||
self
|
||||
}
|
||||
|
||||
fn with_store(mut self) -> Self {
|
||||
let dir = tempdir().expect("tempdir should be created for queue store tests");
|
||||
let store = QueueStore::<QueuedPayload>::new(dir.path(), 16, ".queue");
|
||||
store.open().expect("queue store should open");
|
||||
self.store = Some(store);
|
||||
self.store = Some(Arc::new(store));
|
||||
self
|
||||
}
|
||||
}
|
||||
@@ -271,9 +637,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
|
||||
self.store
|
||||
.as_ref()
|
||||
.map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
|
||||
self.store.as_deref()
|
||||
}
|
||||
|
||||
fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
|
||||
@@ -281,6 +645,11 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn init(&self) -> Result<(), TargetError> {
|
||||
self.init_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(init_entered) = &self.init_entered {
|
||||
init_entered.notify_one();
|
||||
return std::future::pending().await;
|
||||
}
|
||||
if self.init_fails {
|
||||
return Err(TargetError::Configuration("forced init failure".to_string()));
|
||||
}
|
||||
@@ -334,6 +703,212 @@ mod tests {
|
||||
assert_eq!(activation.replay_workers.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_store_target_reports_init_failure_without_dropping_queue_runtime() {
|
||||
let adapter = builtin_adapter();
|
||||
let target = TestTarget::new("primary", "webhook").with_failed_init().with_store();
|
||||
|
||||
let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
|
||||
assert_eq!(prepared.targets.len(), 1);
|
||||
assert!(
|
||||
prepared
|
||||
.failure_summary()
|
||||
.is_some_and(|summary| summary.contains("initialization failed") && !summary.contains("forced init failure"))
|
||||
);
|
||||
|
||||
let (opened, rejected) = adapter.open_prepared_stores(prepared);
|
||||
assert!(rejected.failure_summary().is_some());
|
||||
let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
|
||||
assert!(activation_rejected.failure_summary().is_none());
|
||||
assert_eq!(activation.targets.len(), 1);
|
||||
assert_eq!(activation.replay_workers.len(), 1);
|
||||
activation.replay_workers.stop_all("stop degraded target replay worker").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellable_preparation_returns_current_and_remaining_targets_for_shutdown() {
|
||||
let adapter = builtin_adapter();
|
||||
let init_entered = Arc::new(Notify::new());
|
||||
let first = TestTarget::new("first", "webhook").with_pending_init(init_entered.clone());
|
||||
let first_close_calls = first.close_calls.clone();
|
||||
let second = TestTarget::new("second", "webhook");
|
||||
let second_close_calls = second.close_calls.clone();
|
||||
let second_init_calls = second.init_calls.clone();
|
||||
let cancellation = CancellationToken::new();
|
||||
let prepare_adapter = adapter.clone();
|
||||
let prepare_cancellation = cancellation.clone();
|
||||
let prepare = tokio::spawn(async move {
|
||||
prepare_adapter
|
||||
.prepare_targets_cancellable(vec![Box::new(first), Box::new(second)], &prepare_cancellation)
|
||||
.await
|
||||
});
|
||||
|
||||
init_entered.notified().await;
|
||||
cancellation.cancel();
|
||||
let prepared = tokio::time::timeout(Duration::from_secs(1), prepare)
|
||||
.await
|
||||
.expect("cancellation should interrupt target initialization")
|
||||
.expect("preparation task should finish");
|
||||
assert_eq!(prepared.targets.len(), 2);
|
||||
|
||||
adapter
|
||||
.close_prepared(prepared)
|
||||
.await
|
||||
.expect("cancelled targets should close");
|
||||
assert_eq!(first_close_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(second_close_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(second_init_calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_activation_opens_store_before_starting_replay() {
|
||||
let adapter = builtin_adapter();
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let queue_path = dir.path().join("queue");
|
||||
let mut target = TestTarget::new("primary", "webhook");
|
||||
target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&queue_path, 16, ".queue")));
|
||||
|
||||
let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
|
||||
assert!(!queue_path.exists(), "dormant preparation must not open the queue store");
|
||||
|
||||
let (opened, rejected) = adapter.open_prepared_stores(prepared);
|
||||
assert!(rejected.targets.is_empty());
|
||||
assert!(queue_path.is_dir(), "handoff must open the queue store before activation");
|
||||
let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
|
||||
assert!(activation_rejected.failure_summary().is_none());
|
||||
assert_eq!(activation.replay_workers.len(), 1);
|
||||
activation
|
||||
.replay_workers
|
||||
.stop_all("stop prepared activation test worker")
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn activation_closes_a_target_when_its_queue_store_cannot_open() {
|
||||
let adapter = builtin_adapter();
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let invalid_base = dir.path().join("not-a-directory");
|
||||
std::fs::write(&invalid_base, b"file").expect("invalid queue base should be created");
|
||||
let mut target = TestTarget::new("primary", "webhook");
|
||||
let close_calls = target.close_calls.clone();
|
||||
target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&invalid_base, 16, ".queue")));
|
||||
|
||||
let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
|
||||
|
||||
assert!(activation.targets.is_empty());
|
||||
assert!(activation.replay_workers.is_empty());
|
||||
assert_eq!(close_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_stores_open_with_bounded_parallelism_and_stable_order() {
|
||||
const TARGETS: usize = MAX_PARALLEL_STORE_OPENS * 2;
|
||||
|
||||
let adapter = builtin_adapter();
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let gate = Arc::new(StoreOpenGate::default());
|
||||
let mut targets: Vec<Box<dyn Target<String> + Send + Sync>> = Vec::with_capacity(TARGETS);
|
||||
let mut expected_ids = Vec::with_capacity(TARGETS);
|
||||
for index in 0..TARGETS {
|
||||
let mut target = TestTarget::new(&format!("target-{index}"), "webhook");
|
||||
expected_ids.push(target.id.to_string());
|
||||
let open_gate = gate.clone();
|
||||
target.store = Some(Arc::new(TestOpenStore {
|
||||
before_clone: Arc::new(|| {}),
|
||||
before_open: Arc::new(move || open_gate.enter()),
|
||||
store: QueueStore::new(dir.path().join(index.to_string()), 16, ".queue"),
|
||||
}));
|
||||
targets.push(Box::new(target));
|
||||
}
|
||||
|
||||
let prepared = adapter.prepare_targets(targets).await;
|
||||
let open_adapter = adapter.clone();
|
||||
let opening = tokio::task::spawn_blocking(move || open_adapter.open_prepared_stores(prepared));
|
||||
let wait_gate = gate.clone();
|
||||
let reached_bound =
|
||||
tokio::task::spawn_blocking(move || wait_gate.wait_for_active(MAX_PARALLEL_STORE_OPENS, Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("store-open observer should not panic");
|
||||
gate.release();
|
||||
let (opened, rejected) = opening.await.expect("bounded store opens should not panic");
|
||||
let opened_ids = opened
|
||||
.targets
|
||||
.iter()
|
||||
.map(|target| target.id().to_string())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert!(reached_bound, "store opens did not use the configured parallelism");
|
||||
assert_eq!(gate.max_active(), MAX_PARALLEL_STORE_OPENS);
|
||||
assert_eq!(opened_ids, expected_ids, "parallel store opens must preserve configuration order");
|
||||
assert!(rejected.targets.is_empty());
|
||||
assert!(rejected.failure_summary().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn panicking_store_open_rejects_and_closes_only_that_target() {
|
||||
let adapter = builtin_adapter();
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let mut target = TestTarget::new("panicking", "webhook");
|
||||
let close_calls = target.close_calls.clone();
|
||||
target.store = Some(Arc::new(TestOpenStore {
|
||||
before_clone: Arc::new(|| {}),
|
||||
before_open: Arc::new(|| panic!("forced store open panic: do-not-expose-payload")),
|
||||
store: QueueStore::new(dir.path(), 16, ".queue"),
|
||||
}));
|
||||
|
||||
let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
|
||||
let (opened, rejected) = adapter.open_prepared_stores(prepared);
|
||||
let summary = rejected
|
||||
.failure_summary()
|
||||
.expect("panicking store should report a generic activation failure");
|
||||
let (activation, activation_rejected) = adapter.try_activate_prepared(opened);
|
||||
|
||||
assert!(activation.targets.is_empty(), "a target without an open store must not become visible");
|
||||
assert!(
|
||||
activation.replay_workers.is_empty(),
|
||||
"a rejected target must not publish without a replay worker"
|
||||
);
|
||||
assert!(activation_rejected.failure_summary().is_none());
|
||||
assert!(summary.contains("queue store open failed"));
|
||||
assert!(!summary.contains("do-not-expose-payload"));
|
||||
adapter
|
||||
.close_prepared(rejected)
|
||||
.await
|
||||
.expect("a target rejected after a store panic should close");
|
||||
assert_eq!(close_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn panicking_store_clone_cannot_publish_target_without_replay_worker() {
|
||||
let adapter = builtin_adapter();
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let mut target = TestTarget::new("panicking-clone", "webhook");
|
||||
let close_calls = target.close_calls.clone();
|
||||
target.store = Some(Arc::new(TestOpenStore {
|
||||
before_clone: Arc::new(|| panic!("forced store clone panic: do-not-expose-payload")),
|
||||
before_open: Arc::new(|| {}),
|
||||
store: QueueStore::new(dir.path(), 16, ".queue"),
|
||||
}));
|
||||
|
||||
let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
|
||||
let (opened, open_rejected) = adapter.open_prepared_stores(prepared);
|
||||
assert!(open_rejected.failure_summary().is_none());
|
||||
let (activation, rejected) = adapter.try_activate_prepared(opened);
|
||||
let summary = rejected
|
||||
.failure_summary()
|
||||
.expect("panicking store clone should report a generic activation failure");
|
||||
|
||||
assert!(activation.targets.is_empty(), "a target without a replay worker must not become visible");
|
||||
assert!(activation.replay_workers.is_empty());
|
||||
assert!(summary.contains("replay activation failed"));
|
||||
assert!(!summary.contains("do-not-expose-payload"));
|
||||
adapter
|
||||
.close_prepared(rejected)
|
||||
.await
|
||||
.expect("a target rejected during replay activation should close");
|
||||
assert_eq!(close_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn builtin_adapter_shutdown_clears_runtime_and_replay_workers() {
|
||||
let adapter = builtin_adapter();
|
||||
|
||||
@@ -26,12 +26,23 @@ use crate::plugin::PluginEvent;
|
||||
use crate::store::{Key, Store, ensure_store_entry_raw_readable};
|
||||
use crate::target::QueuedPayload;
|
||||
use crate::target::TargetDeliverySnapshot;
|
||||
use crate::target::{TargetHealth, TargetHealthReason, TargetHealthState};
|
||||
use crate::{StoreError, TargetError};
|
||||
use futures_util::stream::{FuturesUnordered, StreamExt};
|
||||
use std::sync::Arc;
|
||||
use std::{collections::HashMap, fmt::Debug};
|
||||
use std::{future::Future, pin::Pin, time::Duration};
|
||||
use tokio::sync::{Semaphore, mpsc};
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
fn join_failure_reason(error: &tokio::task::JoinError) -> &'static str {
|
||||
if error.is_cancelled() {
|
||||
"join_cancelled"
|
||||
} else {
|
||||
"join_panicked"
|
||||
}
|
||||
}
|
||||
|
||||
/// Maximum number of replay attempts before a stored entry is exhausted. Each attempt runs one full
|
||||
/// send (one ack wait at the configured timeout for a JetStream entry), then a backoff sleep before
|
||||
@@ -68,11 +79,24 @@ pub(crate) fn inter_attempt_backoff_sum(attempts: usize) -> Duration {
|
||||
pub type SharedTarget<E> = Arc<dyn Target<E> + Send + Sync>;
|
||||
type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
|
||||
|
||||
/// Upper bound on how long [`ReplayWorkerManager::stop_all`] waits for a single
|
||||
/// replay worker to observe its cancel signal and exit before it is forcibly
|
||||
/// aborted. Workers observe cancellation promptly (including during retry
|
||||
/// backoff), so this only guards against a wedged task.
|
||||
const STOP_JOIN_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const HEALTH_COLLECTION_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const HEALTH_PROBE_CONCURRENCY: usize = 10;
|
||||
|
||||
pub(crate) enum PrepareTargetResult<E>
|
||||
where
|
||||
E: PluginEvent,
|
||||
{
|
||||
Ready(SharedTarget<E>),
|
||||
Degraded {
|
||||
error: TargetError,
|
||||
target: SharedTarget<E>,
|
||||
},
|
||||
Failed {
|
||||
error: TargetError,
|
||||
target: Box<dyn Target<E> + Send + Sync>,
|
||||
},
|
||||
Cancelled(Box<dyn Target<E> + Send + Sync>),
|
||||
}
|
||||
|
||||
/// Tracks a running replay worker: its cancel channel and, when the worker was
|
||||
/// spawned in-process, the [`JoinHandle`] used to await its exit on shutdown.
|
||||
@@ -113,7 +137,7 @@ impl ReplayWorkerManager {
|
||||
}
|
||||
|
||||
/// Registers a cancel channel together with the worker's join handle so
|
||||
/// `stop_all` can await the worker's exit (bounded by [`STOP_JOIN_TIMEOUT`]).
|
||||
/// `stop_all` can await the worker's exit.
|
||||
pub fn insert_with_handle(&mut self, target_id: String, cancel_tx: mpsc::Sender<()>, join: JoinHandle<()>) {
|
||||
self.cancellers.insert(
|
||||
target_id,
|
||||
@@ -140,37 +164,36 @@ impl ReplayWorkerManager {
|
||||
}
|
||||
|
||||
/// Stops every replay worker: it first signals cancellation to all of them,
|
||||
/// then awaits each worker's exit (bounded by [`STOP_JOIN_TIMEOUT`], after
|
||||
/// which the task is aborted). Signalling before joining lets all workers
|
||||
/// wind down concurrently, and joining guarantees no worker keeps draining
|
||||
/// the shared store after this returns — preventing duplicate delivery and
|
||||
/// orphaned tasks across reloads and shutdown.
|
||||
/// then strictly awaits each worker's exit. A worker already awaiting a
|
||||
/// delivery acknowledgement is allowed to finish; aborting it at an
|
||||
/// arbitrary deadline could leave an acknowledged queue entry undeleted and
|
||||
/// make the replacement worker deliver it again. Signalling before joining
|
||||
/// lets all workers wind down concurrently. Legacy joinless registrations
|
||||
/// can only be signalled.
|
||||
pub async fn stop_all(&mut self, log_prefix: &str) {
|
||||
let mut handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
|
||||
let handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
|
||||
let mut joins = std::collections::VecDeque::new();
|
||||
|
||||
// Phase 1: signal cancellation to all workers.
|
||||
for (target_id, handle) in &handles {
|
||||
for (target_id, handle) in handles {
|
||||
tracing::info!(target_id = %target_id, "{log_prefix}");
|
||||
let _ = handle.cancel_tx.send(()).await;
|
||||
let _ = handle.cancel_tx.try_send(());
|
||||
if let Some(join) = handle.join {
|
||||
joins.push_back((target_id, join));
|
||||
} else {
|
||||
tracing::warn!(
|
||||
target_id = %target_id,
|
||||
"Replay worker has no join handle; cancellation was signalled but exit cannot be verified"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: await each worker's exit, forcibly aborting any that overrun.
|
||||
for (target_id, handle) in handles.drain(..) {
|
||||
let Some(mut join) = handle.join else {
|
||||
continue;
|
||||
};
|
||||
match tokio::time::timeout(STOP_JOIN_TIMEOUT, &mut join).await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(err)) => {
|
||||
tracing::warn!(target_id = %target_id, error = %err, "Replay worker terminated abnormally");
|
||||
}
|
||||
Err(_) => {
|
||||
join.abort();
|
||||
tracing::warn!(
|
||||
target_id = %target_id,
|
||||
"Timed out awaiting replay worker exit; task aborted"
|
||||
);
|
||||
}
|
||||
// Phase 2: strict join. Delivery operations own their own protocol
|
||||
// deadlines; lifecycle must not invent a shorter deadline that turns an
|
||||
// acknowledgement race into duplicate delivery.
|
||||
while let Some((target_id, join)) = joins.pop_front() {
|
||||
if let Err(err) = join.await {
|
||||
tracing::warn!(target_id = %target_id, reason = join_failure_reason(&err), "Replay worker terminated abnormally");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -184,6 +207,55 @@ where
|
||||
pub targets: Vec<SharedTarget<E>>,
|
||||
}
|
||||
|
||||
/// Targets whose persistent queue stores are open and are ready to start
|
||||
/// replay. This distinct stage prevents activation from skipping store open.
|
||||
pub struct OpenedActivation<E>
|
||||
where
|
||||
E: PluginEvent,
|
||||
{
|
||||
pub(crate) targets: Vec<SharedTarget<E>>,
|
||||
}
|
||||
|
||||
struct TargetActivationFailure {
|
||||
detail: String,
|
||||
}
|
||||
|
||||
/// Targets that have completed initialization but have not started replay
|
||||
/// workers yet. Keeping preparation dormant lets lifecycle orchestration stop
|
||||
/// the previous workers before the replacement workers are spawned.
|
||||
pub struct PreparedActivation<E>
|
||||
where
|
||||
E: PluginEvent,
|
||||
{
|
||||
failures: Vec<TargetActivationFailure>,
|
||||
rejected_targets: Vec<SharedTarget<E>>,
|
||||
pub(crate) targets: Vec<SharedTarget<E>>,
|
||||
}
|
||||
|
||||
impl<E> PreparedActivation<E>
|
||||
where
|
||||
E: PluginEvent,
|
||||
{
|
||||
pub fn failure_summary(&self) -> Option<String> {
|
||||
if self.failures.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(
|
||||
self.failures
|
||||
.iter()
|
||||
.map(|failure| failure.detail.clone())
|
||||
.collect::<Vec<_>>()
|
||||
.join("; "),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn extend_creation_failures(&mut self, failures: impl IntoIterator<Item = String>) {
|
||||
self.failures
|
||||
.extend(failures.into_iter().map(|detail| TargetActivationFailure { detail }));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct RuntimeStatusSnapshot {
|
||||
pub replay_worker_count: usize,
|
||||
@@ -201,19 +273,16 @@ pub struct RuntimeTargetSnapshot {
|
||||
pub total_messages: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RuntimeTargetHealthState {
|
||||
Disabled,
|
||||
Error,
|
||||
Offline,
|
||||
Online,
|
||||
}
|
||||
pub type RuntimeTargetHealthState = TargetHealthState;
|
||||
pub type RuntimeTargetHealthReason = TargetHealthReason;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RuntimeTargetHealthSnapshot {
|
||||
pub account_id: String,
|
||||
pub enabled: bool,
|
||||
pub error_message: Option<String>,
|
||||
pub state: RuntimeTargetHealthState,
|
||||
pub reason: RuntimeTargetHealthReason,
|
||||
pub target_id: String,
|
||||
pub target_type: String,
|
||||
}
|
||||
@@ -339,17 +408,19 @@ where
|
||||
/// Surfacing them lets a caller fail an explicit shutdown while still tearing down the rest of the
|
||||
/// runtime.
|
||||
pub async fn clear_and_close(&mut self) -> Vec<(String, TargetError)> {
|
||||
let target_ids: Vec<String> = self.targets.keys().cloned().collect();
|
||||
let targets = std::mem::take(&mut self.targets);
|
||||
let mut closes = FuturesUnordered::new();
|
||||
for (target_id, target) in targets {
|
||||
closes.push(async move { (target_id, target.close().await) });
|
||||
}
|
||||
|
||||
let mut errors = Vec::new();
|
||||
for target_id in target_ids {
|
||||
if let Some(target) = self.targets.remove(&target_id)
|
||||
&& let Err(err) = target.close().await
|
||||
{
|
||||
while let Some((target_id, result)) = closes.next().await {
|
||||
if let Err(err) = result {
|
||||
tracing::error!(target_id = %target_id, error = %err, "Failed to close target during shutdown");
|
||||
errors.push((target_id, err));
|
||||
}
|
||||
}
|
||||
self.targets.clear();
|
||||
errors
|
||||
}
|
||||
|
||||
@@ -389,33 +460,57 @@ where
|
||||
}
|
||||
|
||||
pub async fn health_snapshots(&self) -> Vec<RuntimeTargetHealthSnapshot> {
|
||||
let mut snapshots = Vec::with_capacity(self.targets.len());
|
||||
for target in self.targets.values() {
|
||||
let enabled = target.is_enabled();
|
||||
let target_id = target.id();
|
||||
let (state, error_message) = if !enabled {
|
||||
(RuntimeTargetHealthState::Disabled, None)
|
||||
} else {
|
||||
match target.is_active().await {
|
||||
Ok(true) => (RuntimeTargetHealthState::Online, None),
|
||||
Ok(false) => (RuntimeTargetHealthState::Offline, None),
|
||||
Err(err) => (RuntimeTargetHealthState::Error, Some(err.to_string())),
|
||||
}
|
||||
};
|
||||
|
||||
snapshots.push(RuntimeTargetHealthSnapshot {
|
||||
enabled,
|
||||
error_message,
|
||||
state,
|
||||
target_id: target_id.to_string(),
|
||||
target_type: target_id.name,
|
||||
});
|
||||
}
|
||||
snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
|
||||
snapshots
|
||||
health_snapshots_for_targets(self.values()).await
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn health_snapshots_for_targets<E>(targets: Vec<SharedTarget<E>>) -> Vec<RuntimeTargetHealthSnapshot>
|
||||
where
|
||||
E: PluginEvent,
|
||||
{
|
||||
let deadline = tokio::time::Instant::now() + HEALTH_COLLECTION_TIMEOUT;
|
||||
let permits = Arc::new(Semaphore::new(HEALTH_PROBE_CONCURRENCY));
|
||||
let mut probes = FuturesUnordered::new();
|
||||
for target in targets {
|
||||
let permits = Arc::clone(&permits);
|
||||
let enabled = target.is_enabled();
|
||||
let target_id = target.id();
|
||||
probes.push(async move {
|
||||
let health = if !enabled {
|
||||
TargetHealth::disabled()
|
||||
} else if let Ok(Ok(_permit)) = tokio::time::timeout_at(deadline, permits.acquire_owned()).await {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
TargetHealth::error(TargetHealthReason::HealthCheckFailed)
|
||||
} else {
|
||||
match tokio::time::timeout_at(deadline, target.health()).await {
|
||||
Ok(health) => health,
|
||||
Err(_) => TargetHealth::error(TargetHealthReason::TimedOut),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
TargetHealth::error(TargetHealthReason::HealthCheckFailed)
|
||||
};
|
||||
(enabled, target_id, health)
|
||||
});
|
||||
}
|
||||
|
||||
let mut snapshots = Vec::with_capacity(probes.len());
|
||||
while let Some((enabled, target_id, health)) = probes.next().await {
|
||||
snapshots.push(RuntimeTargetHealthSnapshot {
|
||||
account_id: target_id.id.clone(),
|
||||
enabled,
|
||||
error_message: (health.state == TargetHealthState::Error).then(|| health.reason.as_str().to_string()),
|
||||
state: health.state,
|
||||
reason: health.reason,
|
||||
target_id: target_id.to_string(),
|
||||
target_type: target_id.name,
|
||||
});
|
||||
}
|
||||
|
||||
snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
|
||||
snapshots
|
||||
}
|
||||
|
||||
fn snapshot_from_delivery(target_id: TargetID, delivery: TargetDeliverySnapshot) -> RuntimeTargetSnapshot {
|
||||
RuntimeTargetSnapshot {
|
||||
failed_messages: delivery.failed_messages,
|
||||
@@ -440,21 +535,16 @@ where
|
||||
SharedTarget<E>,
|
||||
) -> (mpsc::Sender<()>, JoinHandle<()>),
|
||||
{
|
||||
let target_id = target.id().to_string();
|
||||
let has_store = target.store().is_some();
|
||||
|
||||
if let Err(err) = target.init().await {
|
||||
tracing::error!(target_id = %target_id, error = %err, "Failed to initialize target");
|
||||
if !has_store {
|
||||
let shared = match prepare_target(target, None).await {
|
||||
PrepareTargetResult::Ready(target) => target,
|
||||
PrepareTargetResult::Degraded { target, .. } => target,
|
||||
PrepareTargetResult::Failed { target, .. } => {
|
||||
let _ = target.close().await;
|
||||
return None;
|
||||
}
|
||||
tracing::warn!(
|
||||
target_id = %target_id,
|
||||
"Proceeding with store-backed target despite init failure"
|
||||
);
|
||||
}
|
||||
|
||||
let shared: SharedTarget<E> = Arc::from(target);
|
||||
PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"),
|
||||
};
|
||||
let target_id = shared.id().to_string();
|
||||
if !shared.is_enabled() {
|
||||
on_replay_start(&target_id, false);
|
||||
return Some((shared, None));
|
||||
@@ -467,6 +557,45 @@ where
|
||||
Some((shared, cancel))
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_target<E>(
|
||||
target: Box<dyn Target<E> + Send + Sync>,
|
||||
cancellation: Option<&CancellationToken>,
|
||||
) -> PrepareTargetResult<E>
|
||||
where
|
||||
E: PluginEvent,
|
||||
{
|
||||
let target_id = target.id().to_string();
|
||||
let has_store = target.store().is_some();
|
||||
|
||||
let init_result = match cancellation {
|
||||
Some(cancellation) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => return PrepareTargetResult::Cancelled(target),
|
||||
result = target.init() => result,
|
||||
}
|
||||
}
|
||||
None => target.init().await,
|
||||
};
|
||||
|
||||
if let Err(err) = init_result {
|
||||
tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target");
|
||||
if !has_store {
|
||||
return PrepareTargetResult::Failed { error: err, target };
|
||||
}
|
||||
tracing::warn!(
|
||||
target_id = %target_id,
|
||||
"Proceeding with store-backed target despite init failure"
|
||||
);
|
||||
return PrepareTargetResult::Degraded {
|
||||
error: err,
|
||||
target: Arc::from(target),
|
||||
};
|
||||
}
|
||||
|
||||
PrepareTargetResult::Ready(Arc::from(target))
|
||||
}
|
||||
|
||||
type ActivatedTarget<E> = (SharedTarget<E>, Option<(mpsc::Sender<()>, JoinHandle<()>)>);
|
||||
|
||||
pub async fn activate_targets_with_replay<E, F, Fut>(
|
||||
@@ -584,7 +713,11 @@ async fn stream_replay_worker<E>(
|
||||
}
|
||||
Ok(Ok(_)) => {}
|
||||
Err(join_err) => {
|
||||
tracing::warn!(target_id = %target.id(), error = %join_err, "The failed-events maintenance task failed to join");
|
||||
tracing::warn!(
|
||||
target_id = %target.id(),
|
||||
reason = join_failure_reason(&join_err),
|
||||
"The failed-events maintenance task failed to join"
|
||||
);
|
||||
}
|
||||
}
|
||||
last_prune = tokio::time::Instant::now();
|
||||
@@ -826,16 +959,18 @@ where
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::TargetRuntimeManager;
|
||||
use super::{HEALTH_PROBE_CONCURRENCY, TargetRuntimeManager, health_snapshots_for_targets};
|
||||
use crate::PluginEvent;
|
||||
use crate::StoreError;
|
||||
use crate::arn::TargetID;
|
||||
use crate::store::{Key, Store};
|
||||
use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
|
||||
use crate::{Target, TargetError};
|
||||
use crate::{SharedTarget, Target, TargetError};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{Notify, Semaphore};
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn seed_interval_start_backdates_by_one_interval() {
|
||||
@@ -900,14 +1035,50 @@ mod tests {
|
||||
#[derive(Clone)]
|
||||
struct TestTarget {
|
||||
id: TargetID,
|
||||
block_on_close: Arc<AtomicBool>,
|
||||
close_gate: Arc<Semaphore>,
|
||||
close_calls: Arc<AtomicUsize>,
|
||||
enabled: bool,
|
||||
health_delay: Duration,
|
||||
health_drops: Arc<AtomicUsize>,
|
||||
health_started: Arc<Notify>,
|
||||
close_started: Arc<Notify>,
|
||||
}
|
||||
|
||||
struct HealthDropGuard(Arc<AtomicUsize>);
|
||||
|
||||
impl Drop for HealthDropGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
impl TestTarget {
|
||||
fn new(id: &str, name: &str) -> Self {
|
||||
Self {
|
||||
id: TargetID::new(id.to_string(), name.to_string()),
|
||||
block_on_close: Arc::new(AtomicBool::new(false)),
|
||||
close_gate: Arc::new(Semaphore::new(0)),
|
||||
close_calls: Arc::new(AtomicUsize::new(0)),
|
||||
enabled: true,
|
||||
health_delay: Duration::ZERO,
|
||||
health_drops: Arc::new(AtomicUsize::new(0)),
|
||||
health_started: Arc::new(Notify::new()),
|
||||
close_started: Arc::new(Notify::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_health_delay(id: &str, delay: Duration) -> Self {
|
||||
Self {
|
||||
health_delay: delay,
|
||||
..Self::new(id, "webhook")
|
||||
}
|
||||
}
|
||||
|
||||
fn disabled(id: &str) -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
..Self::new(id, "webhook")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -922,6 +1093,9 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn is_active(&self) -> Result<bool, TargetError> {
|
||||
self.health_started.notify_one();
|
||||
let _drop_guard = HealthDropGuard(Arc::clone(&self.health_drops));
|
||||
tokio::time::sleep(self.health_delay).await;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -935,6 +1109,10 @@ mod tests {
|
||||
|
||||
async fn close(&self) -> Result<(), TargetError> {
|
||||
self.close_calls.fetch_add(1, Ordering::SeqCst);
|
||||
self.close_started.notify_one();
|
||||
if self.block_on_close.load(Ordering::SeqCst) {
|
||||
let _permit = self.close_gate.acquire().await.expect("close gate should remain open");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -947,7 +1125,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn is_enabled(&self) -> bool {
|
||||
true
|
||||
self.enabled
|
||||
}
|
||||
}
|
||||
|
||||
@@ -966,6 +1144,45 @@ mod tests {
|
||||
assert_eq!(close_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn runtime_manager_starts_all_target_closes_before_waiting_for_completion() {
|
||||
let mut manager = TargetRuntimeManager::<String>::new();
|
||||
let first = TestTarget::new("first", "webhook");
|
||||
let second = TestTarget::new("second", "webhook");
|
||||
let first_observer = first.clone();
|
||||
let second_observer = second.clone();
|
||||
manager.add_boxed(Box::new(first));
|
||||
manager.add_boxed(Box::new(second));
|
||||
|
||||
let first_close_key = manager
|
||||
.keys()
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("two targets should have a first close key");
|
||||
let (blocked, unblocked) = if first_close_key == first_observer.id.to_string() {
|
||||
(first_observer, second_observer)
|
||||
} else {
|
||||
(second_observer, first_observer)
|
||||
};
|
||||
blocked.block_on_close.store(true, Ordering::SeqCst);
|
||||
|
||||
let close_task = tokio::spawn(async move { manager.clear_and_close().await });
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), blocked.close_started.notified())
|
||||
.await
|
||||
.expect("the first target close should start");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), unblocked.close_started.notified())
|
||||
.await
|
||||
.expect("a blocked first close must not prevent the second close from starting");
|
||||
assert!(!close_task.is_finished(), "clear_and_close must still await the blocked target");
|
||||
|
||||
blocked.close_gate.add_permits(1);
|
||||
let errors = close_task.await.expect("clear_and_close task should join");
|
||||
assert!(errors.is_empty());
|
||||
assert_eq!(blocked.close_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(unblocked.close_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_manager_snapshots_targets() {
|
||||
let mut manager = TargetRuntimeManager::<String>::new();
|
||||
@@ -977,6 +1194,104 @@ mod tests {
|
||||
assert_eq!(snapshots[0].target_type, "webhook");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn health_snapshot_allows_a_four_second_probe() {
|
||||
let mut manager = TargetRuntimeManager::<String>::new();
|
||||
manager.add_boxed(Box::new(TestTarget::with_health_delay("slow", Duration::from_secs(4))));
|
||||
|
||||
let snapshots = manager.health_snapshots().await;
|
||||
|
||||
assert_eq!(snapshots.len(), 1);
|
||||
assert_eq!(snapshots[0].state, crate::TargetHealthState::Online);
|
||||
assert_eq!(snapshots[0].reason, crate::TargetHealthReason::Reachable);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn health_snapshot_times_out_after_five_seconds() {
|
||||
let mut manager = TargetRuntimeManager::<String>::new();
|
||||
manager.add_boxed(Box::new(TestTarget::with_health_delay("stalled", Duration::from_secs(6))));
|
||||
|
||||
let snapshots = manager.health_snapshots().await;
|
||||
|
||||
assert_eq!(snapshots.len(), 1);
|
||||
assert_eq!(snapshots[0].state, crate::TargetHealthState::Error);
|
||||
assert_eq!(snapshots[0].reason, crate::TargetHealthReason::TimedOut);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn health_collection_deadline_does_not_scale_with_target_count() {
|
||||
let mut manager = TargetRuntimeManager::<String>::new();
|
||||
for index in 0..24 {
|
||||
manager.add_boxed(Box::new(TestTarget::with_health_delay(
|
||||
&format!("stalled-{index}"),
|
||||
Duration::from_secs(30),
|
||||
)));
|
||||
}
|
||||
let started = tokio::time::Instant::now();
|
||||
|
||||
let snapshots = manager.health_snapshots().await;
|
||||
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(5));
|
||||
assert_eq!(snapshots.len(), 24);
|
||||
assert!(
|
||||
snapshots
|
||||
.iter()
|
||||
.all(|snapshot| snapshot.state == crate::TargetHealthState::Error)
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots
|
||||
.iter()
|
||||
.filter(|snapshot| snapshot.reason == crate::TargetHealthReason::TimedOut)
|
||||
.count(),
|
||||
HEALTH_PROBE_CONCURRENCY
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots
|
||||
.iter()
|
||||
.filter(|snapshot| snapshot.reason == crate::TargetHealthReason::HealthCheckFailed)
|
||||
.count(),
|
||||
24 - HEALTH_PROBE_CONCURRENCY
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn disabled_target_does_not_wait_for_probe_capacity() {
|
||||
let mut targets: Vec<SharedTarget<String>> = (0..HEALTH_PROBE_CONCURRENCY)
|
||||
.map(|index| {
|
||||
Arc::new(TestTarget::with_health_delay(&format!("stalled-{index}"), Duration::from_secs(30)))
|
||||
as SharedTarget<String>
|
||||
})
|
||||
.collect();
|
||||
targets.push(Arc::new(TestTarget::disabled("disabled")));
|
||||
|
||||
let snapshots = health_snapshots_for_targets(targets).await;
|
||||
let disabled = snapshots
|
||||
.iter()
|
||||
.find(|snapshot| snapshot.target_id == "disabled:webhook")
|
||||
.expect("disabled target snapshot");
|
||||
|
||||
assert_eq!(disabled.state, crate::TargetHealthState::Disabled);
|
||||
assert_eq!(disabled.reason, crate::TargetHealthReason::Disabled);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_health_collection_drops_in_flight_probe() {
|
||||
let target = TestTarget::with_health_delay("slow", Duration::from_secs(30));
|
||||
let health_drops = Arc::clone(&target.health_drops);
|
||||
let health_started = Arc::clone(&target.health_started);
|
||||
let targets: Vec<SharedTarget<String>> = vec![Arc::new(target)];
|
||||
let collector = tokio::spawn(health_snapshots_for_targets(targets));
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), health_started.notified())
|
||||
.await
|
||||
.expect("health probe should start");
|
||||
collector.abort();
|
||||
let join_error = collector.await.expect_err("health collector should be cancelled");
|
||||
|
||||
assert!(join_error.is_cancelled());
|
||||
assert_eq!(health_drops.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sleep_or_cancelled_returns_immediately_on_cancel() {
|
||||
let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel::<()>(1);
|
||||
@@ -1030,6 +1345,51 @@ mod tests {
|
||||
assert!(exited.load(Ordering::SeqCst), "stop_all must await the worker to completion");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn stop_all_does_not_abort_delivery_awaiting_acknowledgement() {
|
||||
use super::ReplayWorkerManager;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
let mut manager = ReplayWorkerManager::new();
|
||||
let acknowledgement = Arc::new(Notify::new());
|
||||
let worker_started = Arc::new(Notify::new());
|
||||
let exited = Arc::new(AtomicBool::new(false));
|
||||
let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel::<()>(1);
|
||||
let worker_acknowledgement = acknowledgement.clone();
|
||||
let worker_started_signal = worker_started.clone();
|
||||
let worker_exited = exited.clone();
|
||||
let join = tokio::spawn(async move {
|
||||
worker_started_signal.notify_one();
|
||||
let _ = cancel_rx.recv().await;
|
||||
// Model a protocol operation that has accepted the request but has
|
||||
// not returned its acknowledgement yet. Lifecycle must not abort
|
||||
// this future or the same durable entry can be sent twice.
|
||||
worker_acknowledgement.notified().await;
|
||||
worker_exited.store(true, Ordering::SeqCst);
|
||||
});
|
||||
manager.insert_with_handle("primary:webhook".to_string(), cancel_tx, join);
|
||||
worker_started.notified().await;
|
||||
|
||||
let mut stop = Box::pin(manager.stop_all("stopping ack-pending test worker"));
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut stop => panic!("stop_all returned before the pending acknowledgement"),
|
||||
_ = std::future::ready(()) => {}
|
||||
}
|
||||
tokio::time::advance(std::time::Duration::from_secs(60)).await;
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut stop => panic!("stop_all aborted an acknowledgement-pending delivery"),
|
||||
_ = std::future::ready(()) => {}
|
||||
}
|
||||
|
||||
acknowledgement.notify_one();
|
||||
stop.await;
|
||||
assert!(exited.load(Ordering::SeqCst));
|
||||
assert!(manager.is_empty());
|
||||
}
|
||||
|
||||
mod classifier {
|
||||
use super::super::{ReplayEvent, stream_replay_worker};
|
||||
use crate::arn::TargetID;
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::{
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error,
|
||||
TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, is_connectivity_error,
|
||||
open_target_queue_store, persist_queued_payload_to_store,
|
||||
},
|
||||
};
|
||||
@@ -32,13 +32,84 @@ use async_trait::async_trait;
|
||||
use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError, KafkaCode};
|
||||
use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SaslConfig, SecurityConfig};
|
||||
use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key};
|
||||
use std::{fmt, marker::PhantomData, sync::Arc, time::Duration};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::{fmt, future::Future, marker::PhantomData, sync::Arc, time::Duration};
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
pub(crate) const KAFKA_SASL_PLAIN: &str = "PLAIN";
|
||||
pub(crate) const KAFKA_SASL_SCRAM_SHA_256: &str = "SCRAM-SHA-256";
|
||||
pub(crate) const KAFKA_SASL_SCRAM_SHA_512: &str = "SCRAM-SHA-512";
|
||||
const KAFKA_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
struct KafkaDeliveryAttempt<'a> {
|
||||
armed: bool,
|
||||
poisoned: &'a AtomicBool,
|
||||
}
|
||||
|
||||
impl KafkaDeliveryAttempt<'_> {
|
||||
fn disarm(&mut self) {
|
||||
self.armed = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for KafkaDeliveryAttempt<'_> {
|
||||
fn drop(&mut self) {
|
||||
if self.armed {
|
||||
self.poisoned.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn kafka_delivery_timeout() -> TargetError {
|
||||
TargetError::Timeout(format!("Kafka delivery timed out after {KAFKA_DELIVERY_TIMEOUT:?}"))
|
||||
}
|
||||
|
||||
async fn with_serialized_kafka_delivery<P, T, Select, SelectFuture, Deliver, DeliveryFuture, Invalidate, InvalidateFuture>(
|
||||
delivery_lock: &Mutex<()>,
|
||||
delivery_poisoned: &AtomicBool,
|
||||
select_producer: Select,
|
||||
deliver: Deliver,
|
||||
invalidate: Invalidate,
|
||||
) -> Result<T, TargetError>
|
||||
where
|
||||
P: Send,
|
||||
T: Send,
|
||||
Select: FnOnce() -> SelectFuture + Send,
|
||||
SelectFuture: Future<Output = Result<P, TargetError>> + Send,
|
||||
Deliver: FnOnce(P) -> DeliveryFuture + Send,
|
||||
DeliveryFuture: Future<Output = Result<T, TargetError>> + Send,
|
||||
Invalidate: Fn() -> InvalidateFuture + Send,
|
||||
InvalidateFuture: Future<Output = ()> + Send,
|
||||
{
|
||||
let deadline = tokio::time::Instant::now() + KAFKA_DELIVERY_TIMEOUT;
|
||||
let _delivery_guard = tokio::time::timeout_at(deadline, delivery_lock.lock())
|
||||
.await
|
||||
.map_err(|_| kafka_delivery_timeout())?;
|
||||
let mut attempt = KafkaDeliveryAttempt {
|
||||
armed: true,
|
||||
poisoned: delivery_poisoned,
|
||||
};
|
||||
|
||||
if delivery_poisoned.load(Ordering::Acquire) {
|
||||
tokio::time::timeout_at(deadline, invalidate())
|
||||
.await
|
||||
.map_err(|_| kafka_delivery_timeout())?;
|
||||
delivery_poisoned.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
let result = tokio::time::timeout_at(deadline, async { deliver(select_producer().await?).await })
|
||||
.await
|
||||
.map_err(|_| kafka_delivery_timeout())?;
|
||||
if result.as_ref().is_err_and(is_connectivity_error) {
|
||||
tokio::time::timeout_at(deadline, invalidate())
|
||||
.await
|
||||
.map_err(|_| kafka_delivery_timeout())?;
|
||||
delivery_poisoned.store(false, Ordering::Release);
|
||||
}
|
||||
attempt.disarm();
|
||||
result
|
||||
}
|
||||
|
||||
/// Arguments for configuring a Kafka target
|
||||
#[derive(Clone)]
|
||||
@@ -233,6 +304,8 @@ where
|
||||
args: KafkaArgs,
|
||||
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
|
||||
producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
|
||||
delivery_lock: Arc<Mutex<()>>,
|
||||
delivery_poisoned: Arc<AtomicBool>,
|
||||
tls_state: Arc<Mutex<TargetTlsState>>,
|
||||
/// Adapter that bridges this target to the TLS reload coordinator.
|
||||
/// When `Some`, the target uses coordinator-managed material; when `None`,
|
||||
@@ -291,6 +364,8 @@ where
|
||||
args,
|
||||
store: queue_store,
|
||||
producer: Arc::new(Mutex::new(None)),
|
||||
delivery_lock: Arc::new(Mutex::new(())),
|
||||
delivery_poisoned: Arc::new(AtomicBool::new(false)),
|
||||
tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
|
||||
tls_adapter: None,
|
||||
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
|
||||
@@ -307,7 +382,7 @@ where
|
||||
};
|
||||
|
||||
let mut config = AsyncProducerConfig::new()
|
||||
.with_ack_timeout(Duration::from_secs(30))
|
||||
.with_ack_timeout(KAFKA_DELIVERY_TIMEOUT)
|
||||
.with_required_acks(acks);
|
||||
|
||||
if let Some(security) = self.args.security_config(true)? {
|
||||
@@ -388,20 +463,26 @@ where
|
||||
"Sending Kafka payload"
|
||||
);
|
||||
|
||||
let producer = self.get_or_build_producer().await?;
|
||||
|
||||
// Use "<bucket>/<object>" as the message key so all events for the same
|
||||
// object hash to the same partition and preserve per-object ordering
|
||||
// across multiple partitions (backlog#983).
|
||||
let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
|
||||
if let Err(err) = producer
|
||||
.send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
|
||||
.await
|
||||
{
|
||||
let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka");
|
||||
invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_producer()).await;
|
||||
return Err(mapped);
|
||||
}
|
||||
// rustfs-kafka-async does not validate response correlation IDs. Keep
|
||||
// producer selection, send, and timeout invalidation serialized so a
|
||||
// waiter cannot reuse a connection with an unread timed-out response.
|
||||
with_serialized_kafka_delivery(
|
||||
&self.delivery_lock,
|
||||
&self.delivery_poisoned,
|
||||
|| self.get_or_build_producer(),
|
||||
|producer| async move {
|
||||
// Use "<bucket>/<object>" as the message key so all events for the same
|
||||
// object hash to the same partition and preserve per-object ordering
|
||||
// across multiple partitions (backlog#983).
|
||||
let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
|
||||
producer
|
||||
.send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
|
||||
.await
|
||||
.map_err(|err| Self::map_kafka_error(err, "Failed to send message to Kafka"))
|
||||
},
|
||||
|| self.invalidate_cached_producer(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
debug!(target_id = %self.id, topic = %self.args.topic, "Event published to Kafka topic");
|
||||
self.delivery_counters.record_success();
|
||||
@@ -415,6 +496,8 @@ where
|
||||
args: self.args.clone(),
|
||||
store: self.store.as_ref().map(|s| s.boxed_clone()),
|
||||
producer: Arc::clone(&self.producer),
|
||||
delivery_lock: Arc::clone(&self.delivery_lock),
|
||||
delivery_poisoned: Arc::clone(&self.delivery_poisoned),
|
||||
tls_state: Arc::clone(&self.tls_state),
|
||||
tls_adapter: self.tls_adapter.clone(),
|
||||
delivery_counters: Arc::clone(&self.delivery_counters),
|
||||
@@ -559,6 +642,8 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
fn base_args() -> KafkaArgs {
|
||||
KafkaArgs {
|
||||
@@ -580,6 +665,158 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn timeout_invalidates_before_the_next_delivery_selects_a_producer() {
|
||||
let delivery_lock = Arc::new(Mutex::new(()));
|
||||
let delivery_poisoned = Arc::new(AtomicBool::new(false));
|
||||
let generation = Arc::new(AtomicUsize::new(1));
|
||||
let first_entered = Arc::new(Notify::new());
|
||||
|
||||
let first = {
|
||||
let delivery_lock = Arc::clone(&delivery_lock);
|
||||
let delivery_poisoned = Arc::clone(&delivery_poisoned);
|
||||
let generation = Arc::clone(&generation);
|
||||
let first_entered = Arc::clone(&first_entered);
|
||||
tokio::spawn(async move {
|
||||
with_serialized_kafka_delivery(
|
||||
&delivery_lock,
|
||||
&delivery_poisoned,
|
||||
{
|
||||
let generation = Arc::clone(&generation);
|
||||
move || async move { Ok(generation.load(Ordering::SeqCst)) }
|
||||
},
|
||||
move |selected| async move {
|
||||
assert_eq!(selected, 1);
|
||||
first_entered.notify_one();
|
||||
std::future::pending::<Result<usize, TargetError>>().await
|
||||
},
|
||||
move || {
|
||||
let generation = Arc::clone(&generation);
|
||||
async move { generation.store(2, Ordering::SeqCst) }
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
};
|
||||
|
||||
first_entered.notified().await;
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
let second = {
|
||||
let delivery_lock = Arc::clone(&delivery_lock);
|
||||
let delivery_poisoned = Arc::clone(&delivery_poisoned);
|
||||
let generation = Arc::clone(&generation);
|
||||
tokio::spawn(async move {
|
||||
with_serialized_kafka_delivery(
|
||||
&delivery_lock,
|
||||
&delivery_poisoned,
|
||||
{
|
||||
let generation = Arc::clone(&generation);
|
||||
move || async move { Ok(generation.load(Ordering::SeqCst)) }
|
||||
},
|
||||
|selected| async move { Ok(selected) },
|
||||
move || {
|
||||
let generation = Arc::clone(&generation);
|
||||
async move { generation.store(2, Ordering::SeqCst) }
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
};
|
||||
|
||||
assert!(matches!(
|
||||
first.await.expect("first delivery task should not panic"),
|
||||
Err(TargetError::Timeout(_))
|
||||
));
|
||||
assert_eq!(
|
||||
second
|
||||
.await
|
||||
.expect("second delivery task should not panic")
|
||||
.expect("second delivery should succeed"),
|
||||
2,
|
||||
"the waiter must select a fresh producer generation after timeout invalidation"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn delivery_deadline_includes_waiting_for_the_serialization_lock() {
|
||||
let delivery_lock = Arc::new(Mutex::new(()));
|
||||
let delivery_poisoned = AtomicBool::new(false);
|
||||
let selected = Arc::new(AtomicBool::new(false));
|
||||
let _held = delivery_lock.lock().await;
|
||||
|
||||
let error = with_serialized_kafka_delivery(
|
||||
&delivery_lock,
|
||||
&delivery_poisoned,
|
||||
{
|
||||
let selected = Arc::clone(&selected);
|
||||
move || async move {
|
||||
selected.store(true, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
|()| async { Ok(()) },
|
||||
|| async {},
|
||||
)
|
||||
.await
|
||||
.expect_err("lock admission must share the absolute delivery deadline");
|
||||
|
||||
assert!(matches!(error, TargetError::Timeout(_)));
|
||||
assert!(!selected.load(Ordering::SeqCst), "a timed-out waiter must not select a producer");
|
||||
assert!(!delivery_poisoned.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_delivery_poisons_the_connection_before_the_next_selection() {
|
||||
let delivery_lock = Arc::new(Mutex::new(()));
|
||||
let delivery_poisoned = Arc::new(AtomicBool::new(false));
|
||||
let generation = Arc::new(AtomicUsize::new(1));
|
||||
let first_entered = Arc::new(Notify::new());
|
||||
let first = {
|
||||
let delivery_lock = Arc::clone(&delivery_lock);
|
||||
let delivery_poisoned = Arc::clone(&delivery_poisoned);
|
||||
let first_entered = Arc::clone(&first_entered);
|
||||
tokio::spawn(async move {
|
||||
with_serialized_kafka_delivery(
|
||||
&delivery_lock,
|
||||
&delivery_poisoned,
|
||||
|| async { Ok(1usize) },
|
||||
move |_| async move {
|
||||
first_entered.notify_one();
|
||||
std::future::pending::<Result<(), TargetError>>().await
|
||||
},
|
||||
|| async {},
|
||||
)
|
||||
.await
|
||||
})
|
||||
};
|
||||
first_entered.notified().await;
|
||||
first.abort();
|
||||
assert!(first.await.expect_err("first delivery should be cancelled").is_cancelled());
|
||||
assert!(delivery_poisoned.load(Ordering::Acquire));
|
||||
|
||||
let selected = with_serialized_kafka_delivery(
|
||||
&delivery_lock,
|
||||
&delivery_poisoned,
|
||||
{
|
||||
let generation = Arc::clone(&generation);
|
||||
move || async move { Ok(generation.load(Ordering::SeqCst)) }
|
||||
},
|
||||
|selected| async move { Ok(selected) },
|
||||
{
|
||||
let generation = Arc::clone(&generation);
|
||||
move || {
|
||||
let generation = Arc::clone(&generation);
|
||||
async move { generation.store(2, Ordering::SeqCst) }
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the next delivery should recover from cancellation poisoning");
|
||||
|
||||
assert_eq!(selected, 2);
|
||||
assert!(!delivery_poisoned.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_empty_brokers() {
|
||||
let args = KafkaArgs {
|
||||
|
||||
@@ -19,12 +19,14 @@ use crate::{StoreError, TargetError, TargetLog};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_s3_types::EventName;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::cell::Cell;
|
||||
use std::fmt::Formatter;
|
||||
use std::future::Future;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::thread_local;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
pub mod amqp;
|
||||
@@ -70,6 +72,125 @@ pub struct TargetDeliveryCounters {
|
||||
total_messages: AtomicU64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TargetHealthState {
|
||||
Disabled,
|
||||
Error,
|
||||
Offline,
|
||||
Online,
|
||||
}
|
||||
|
||||
impl TargetHealthState {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Disabled => "disabled",
|
||||
Self::Error => "error",
|
||||
Self::Offline => "offline",
|
||||
Self::Online => "online",
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn status(self) -> &'static str {
|
||||
match self {
|
||||
Self::Online => "online",
|
||||
Self::Disabled | Self::Error | Self::Offline => "offline",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TargetHealthReason {
|
||||
AuthenticationFailed,
|
||||
ConfigurationInvalid,
|
||||
ConnectionRefused,
|
||||
Disabled,
|
||||
DnsFailure,
|
||||
HealthCheckFailed,
|
||||
InitializationFailed,
|
||||
NotLoadedInRuntime,
|
||||
Reachable,
|
||||
RequestFailed,
|
||||
TimedOut,
|
||||
TlsFailure,
|
||||
Unreachable,
|
||||
}
|
||||
|
||||
impl TargetHealthReason {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::AuthenticationFailed => "authentication_failed",
|
||||
Self::ConfigurationInvalid => "configuration_invalid",
|
||||
Self::ConnectionRefused => "connection_refused",
|
||||
Self::Disabled => "disabled",
|
||||
Self::DnsFailure => "dns_failure",
|
||||
Self::HealthCheckFailed => "health_check_failed",
|
||||
Self::InitializationFailed => "initialization_failed",
|
||||
Self::NotLoadedInRuntime => "not_loaded_in_runtime",
|
||||
Self::Reachable => "reachable",
|
||||
Self::RequestFailed => "request_failed",
|
||||
Self::TimedOut => "timed_out",
|
||||
Self::TlsFailure => "tls_failure",
|
||||
Self::Unreachable => "unreachable",
|
||||
}
|
||||
}
|
||||
|
||||
fn from_target_error(err: &TargetError) -> Self {
|
||||
match err {
|
||||
TargetError::Authentication(_) => Self::AuthenticationFailed,
|
||||
TargetError::Configuration(_) | TargetError::ParseError(_) => Self::ConfigurationInvalid,
|
||||
TargetError::Initialization(_) | TargetError::ServerNotInitialized(_) => Self::InitializationFailed,
|
||||
TargetError::Network(_) | TargetError::NotConnected => Self::Unreachable,
|
||||
TargetError::Request(_) => Self::RequestFailed,
|
||||
TargetError::Timeout(_) => Self::TimedOut,
|
||||
TargetError::Storage(_)
|
||||
| TargetError::JetStreamPublish { .. }
|
||||
| TargetError::Encoding(_)
|
||||
| TargetError::Serialization(_)
|
||||
| TargetError::InvalidARN(_)
|
||||
| TargetError::Unknown(_)
|
||||
| TargetError::Disabled
|
||||
| TargetError::Dropped(_)
|
||||
| TargetError::SaveConfig(_) => Self::HealthCheckFailed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct TargetHealth {
|
||||
pub state: TargetHealthState,
|
||||
pub reason: TargetHealthReason,
|
||||
}
|
||||
|
||||
impl TargetHealth {
|
||||
pub const fn disabled() -> Self {
|
||||
Self {
|
||||
state: TargetHealthState::Disabled,
|
||||
reason: TargetHealthReason::Disabled,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn error(reason: TargetHealthReason) -> Self {
|
||||
Self {
|
||||
state: TargetHealthState::Error,
|
||||
reason,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn offline(reason: TargetHealthReason) -> Self {
|
||||
Self {
|
||||
state: TargetHealthState::Offline,
|
||||
reason,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn online(reason: TargetHealthReason) -> Self {
|
||||
Self {
|
||||
state: TargetHealthState::Online,
|
||||
reason,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) type BoxedQueuedStore = Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>;
|
||||
|
||||
impl TargetDeliveryCounters {
|
||||
@@ -111,7 +232,25 @@ where
|
||||
/// Checks if the target is active and reachable
|
||||
async fn is_active(&self) -> Result<bool, TargetError>;
|
||||
|
||||
/// Saves an event (either sends it immediately or stores it for later)
|
||||
/// Returns a credential-free, machine-readable health result.
|
||||
async fn health(&self) -> TargetHealth {
|
||||
if !self.is_enabled() {
|
||||
return TargetHealth::disabled();
|
||||
}
|
||||
|
||||
match self.is_active().await {
|
||||
Ok(true) => TargetHealth::online(TargetHealthReason::Reachable),
|
||||
Ok(false) => TargetHealth::offline(TargetHealthReason::Unreachable),
|
||||
Err(err) => TargetHealth::error(TargetHealthReason::from_target_error(&err)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Saves an event (either sends it immediately or stores it for later).
|
||||
///
|
||||
/// A target whose [`Self::store`] returns `Some` must only persist the event
|
||||
/// here; network delivery belongs to its replay worker. Runtime lifecycle
|
||||
/// handoff drains these durable enqueues while allowing a direct network
|
||||
/// send to finish against a detached target.
|
||||
async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError>;
|
||||
|
||||
/// Sends an event from the store using the queued raw body and metadata.
|
||||
@@ -605,6 +744,24 @@ pub(crate) fn open_target_queue_store(
|
||||
Ok(store.map(|store| Box::new(store) as BoxedQueuedStore))
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
static DEFER_QUEUE_STORE_OPEN: Cell<bool> = const { Cell::new(false) };
|
||||
}
|
||||
|
||||
pub(crate) fn with_deferred_queue_store_open<T>(operation: impl FnOnce() -> T) -> T {
|
||||
struct Reset(bool);
|
||||
|
||||
impl Drop for Reset {
|
||||
fn drop(&mut self) {
|
||||
DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.set(self.0));
|
||||
}
|
||||
}
|
||||
|
||||
let previous = DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.replace(true));
|
||||
let _reset = Reset(previous);
|
||||
operation()
|
||||
}
|
||||
|
||||
/// Opens the queue store and returns the concrete QueueStore, so a target that needs its typed
|
||||
/// failed-store capability holds it directly rather than through the type-erased Store handle.
|
||||
pub(crate) fn open_target_queue_store_typed(
|
||||
@@ -625,9 +782,11 @@ pub(crate) fn open_target_queue_store_typed(
|
||||
TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
|
||||
};
|
||||
let store = QueueStore::<QueuedPayload>::new(queue_dir, queue_limit, extension);
|
||||
store
|
||||
.open()
|
||||
.map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
|
||||
if !DEFER_QUEUE_STORE_OPEN.with(Cell::get) {
|
||||
store
|
||||
.open()
|
||||
.map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
|
||||
}
|
||||
|
||||
Ok(Some(store))
|
||||
}
|
||||
@@ -649,6 +808,26 @@ pub(crate) fn is_connectivity_error(err: &TargetError) -> bool {
|
||||
matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
|
||||
}
|
||||
|
||||
/// Applies an absolute deadline to one protocol delivery attempt.
|
||||
///
|
||||
/// Target clients expose different timeout controls, and several of them only
|
||||
/// place a timeout value in the wire request without bounding the local socket
|
||||
/// future. Keeping the outer deadline here gives every caller the same typed,
|
||||
/// retryable timeout without changing the target-specific error mapping.
|
||||
pub(crate) async fn with_delivery_deadline<T, F>(
|
||||
deadline: Duration,
|
||||
operation: &'static str,
|
||||
delivery: F,
|
||||
) -> Result<T, TargetError>
|
||||
where
|
||||
F: Future<Output = Result<T, TargetError>>,
|
||||
{
|
||||
match tokio::time::timeout(deadline, delivery).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(TargetError::Timeout(format!("{operation} timed out after {deadline:?}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn invalidate_cache_on_connectivity_error<F, Fut>(err: &TargetError, invalidate: F)
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
@@ -1137,6 +1316,29 @@ mod tests {
|
||||
let _ = fs::remove_file(base);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deferred_queue_store_creation_does_not_touch_the_filesystem() {
|
||||
let base = std::env::temp_dir().join(format!("rustfs-target-store-deferred-{}", Uuid::new_v4()));
|
||||
fs::write(&base, b"not-a-directory").expect("failed to create file base");
|
||||
let target_id = TargetID::new("target-a".to_string(), ChannelTargetType::Kafka.as_str().to_string());
|
||||
|
||||
let store = with_deferred_queue_store_open(|| {
|
||||
open_target_queue_store(
|
||||
base.to_str().unwrap(),
|
||||
100,
|
||||
TargetType::NotifyEvent,
|
||||
ChannelTargetType::Kafka.as_str(),
|
||||
&target_id,
|
||||
"deferred open",
|
||||
)
|
||||
})
|
||||
.expect("deferred construction must not open the queue directory")
|
||||
.expect("non-empty queue directory should create a dormant store");
|
||||
|
||||
assert!(store.open().is_err(), "the invalid path must fail when handoff explicitly opens it");
|
||||
let _ = fs::remove_file(base);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persist_queued_payload_to_store_writes_encoded_payload() {
|
||||
let store = MockQueuedStore::new(false);
|
||||
@@ -1182,6 +1384,19 @@ mod tests {
|
||||
assert!(!is_connectivity_error(&TargetError::Serialization("serialization".to_string())));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn delivery_deadline_cuts_off_a_stalled_protocol_operation() {
|
||||
let error = with_delivery_deadline(
|
||||
Duration::from_secs(30),
|
||||
"test delivery",
|
||||
std::future::pending::<Result<(), TargetError>>(),
|
||||
)
|
||||
.await
|
||||
.expect_err("a stalled delivery must hit its hard deadline");
|
||||
|
||||
assert!(matches!(error, TargetError::Timeout(message) if message == "test delivery timed out after 30s"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalidate_cache_on_connectivity_error_only_runs_for_connectivity_failures() {
|
||||
let marker = Arc::new(AtomicBool::new(false));
|
||||
|
||||
@@ -765,11 +765,6 @@ where
|
||||
|
||||
#[instrument(skip(self, body, meta), fields(target_id = %self.id))]
|
||||
async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
|
||||
let client_guard = self.client.lock().await;
|
||||
let client = client_guard
|
||||
.as_ref()
|
||||
.ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
|
||||
|
||||
debug!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
@@ -790,10 +785,22 @@ where
|
||||
// silently dropped the event while its durable copy was already deleted
|
||||
// (backlog#971). Error classification now matches on the typed error
|
||||
// instead of substring matching on the display string.
|
||||
let notice = match client.publish_tracked(&self.args.topic, self.args.qos, false, body).await {
|
||||
Ok(notice) => notice,
|
||||
Err(e) => {
|
||||
let err = classify_mqtt_client_error(&e);
|
||||
let notice = match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, async {
|
||||
let client_guard = self.client.lock().await;
|
||||
let client = client_guard
|
||||
.as_ref()
|
||||
.ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
|
||||
let notice = client
|
||||
.publish_tracked(&self.args.topic, self.args.qos, false, body)
|
||||
.await
|
||||
.map_err(|error| classify_mqtt_client_error(&error))?;
|
||||
drop(client_guard);
|
||||
Ok(notice)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(Ok(notice)) => notice,
|
||||
Ok(Err(err)) => {
|
||||
warn!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
@@ -801,18 +808,29 @@ where
|
||||
target_id = %self.id,
|
||||
state = "publish_failed",
|
||||
reason = "enqueue_error",
|
||||
error = %e,
|
||||
error = %err,
|
||||
"mqtt delivery state"
|
||||
);
|
||||
mark_target_disconnected_on_connectivity_error(&self.connected, &err);
|
||||
return Err(err);
|
||||
}
|
||||
Err(_) => {
|
||||
warn!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "publish_failed",
|
||||
reason = "enqueue_timeout",
|
||||
"mqtt delivery state"
|
||||
);
|
||||
// Admission can time out because the local bounded request
|
||||
// channel is full while the MQTT session remains connected.
|
||||
// Only protocol/client failures are evidence of disconnect.
|
||||
return Err(TargetError::Timeout("MQTT publish enqueue timed out".to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
// Release the client lock before awaiting the broker acknowledgement so a
|
||||
// slow/hung broker never blocks other senders from queueing publishes.
|
||||
drop(client_guard);
|
||||
|
||||
match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, notice.wait_completion_async()).await {
|
||||
Ok(Ok(())) => {
|
||||
debug!(
|
||||
@@ -1708,9 +1726,9 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTlsConfig, PublishNoticeError, QoS,
|
||||
classify_mqtt_client_error, classify_mqtt_notice_error, next_reconnect_backoff, reconnect_supervisor,
|
||||
validate_mqtt_broker_url,
|
||||
AsyncClient, ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTarget, MQTTTlsConfig,
|
||||
MqttOptions, PublishNoticeError, QoS, QueuedPayloadMeta, classify_mqtt_client_error, classify_mqtt_notice_error,
|
||||
next_reconnect_backoff, reconnect_supervisor, validate_mqtt_broker_url,
|
||||
};
|
||||
use crate::error::TargetError;
|
||||
use crate::target::{REDACTED_SECRET, TargetType};
|
||||
@@ -1720,6 +1738,23 @@ mod tests {
|
||||
use tokio::sync::mpsc;
|
||||
use url::Url;
|
||||
|
||||
fn base_mqtt_args() -> MQTTArgs {
|
||||
MQTTArgs {
|
||||
enable: true,
|
||||
broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
|
||||
topic: "rustfs/events".to_string(),
|
||||
qos: QoS::AtLeastOnce,
|
||||
username: String::new(),
|
||||
password: String::new(),
|
||||
tls: MQTTTlsConfig::default(),
|
||||
max_reconnect_interval: Duration::from_secs(1),
|
||||
keep_alive: Duration::from_secs(30),
|
||||
queue_dir: String::new(),
|
||||
queue_limit: 0,
|
||||
target_type: TargetType::NotifyEvent,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mqtt_client_error_classified_as_not_connected() {
|
||||
// A publish that cannot be handed to the event loop means the client is
|
||||
@@ -1752,6 +1787,38 @@ mod tests {
|
||||
assert!(matches!(classify_mqtt_notice_error(&err), TargetError::Request(_)));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn enqueue_timeout_keeps_a_live_session_connected() {
|
||||
let target = MQTTTarget::<String>::new("mqtt:test".to_string(), base_mqtt_args()).expect("target should build");
|
||||
let (client, _event_loop) = AsyncClient::builder(MqttOptions::new("mqtt-timeout-test", ("localhost", 1883)))
|
||||
.capacity(1)
|
||||
.build();
|
||||
client
|
||||
.publish("fill", QoS::AtLeastOnce, false, b"fill".as_slice())
|
||||
.await
|
||||
.expect("first publish should fill the local channel");
|
||||
*target.client.lock().await = Some(client);
|
||||
target.connected.store(true, Ordering::SeqCst);
|
||||
let meta = QueuedPayloadMeta::new(
|
||||
rustfs_s3_types::EventName::ObjectCreatedPut,
|
||||
"bucket".to_string(),
|
||||
"object".to_string(),
|
||||
"application/json",
|
||||
2,
|
||||
);
|
||||
|
||||
let error = target
|
||||
.send_body(b"{}".to_vec(), &meta)
|
||||
.await
|
||||
.expect_err("a full local request channel should hit the enqueue deadline");
|
||||
|
||||
assert!(matches!(error, TargetError::Timeout(_)));
|
||||
assert!(
|
||||
target.connected.load(Ordering::SeqCst),
|
||||
"local admission pressure is not evidence that the MQTT session disconnected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn next_reconnect_backoff_doubles_until_capped() {
|
||||
let mut backoff = MQTT_RECONNECT_BACKOFF_MIN;
|
||||
@@ -1864,21 +1931,13 @@ mod tests {
|
||||
#[test]
|
||||
fn debug_redacts_mqtt_secret_fields() {
|
||||
let args = MQTTArgs {
|
||||
enable: true,
|
||||
broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
|
||||
topic: "rustfs/events".to_string(),
|
||||
qos: QoS::AtLeastOnce,
|
||||
username: "mqtt-user".to_string(),
|
||||
password: "mqtt-password".to_string(),
|
||||
tls: MQTTTlsConfig {
|
||||
client_key_path: "/etc/rustfs/mqtt.key".to_string(),
|
||||
..MQTTTlsConfig::default()
|
||||
},
|
||||
max_reconnect_interval: Duration::from_secs(1),
|
||||
keep_alive: Duration::from_secs(30),
|
||||
queue_dir: String::new(),
|
||||
queue_limit: 0,
|
||||
target_type: TargetType::NotifyEvent,
|
||||
..base_mqtt_args()
|
||||
};
|
||||
|
||||
let rendered = format!("{args:?}");
|
||||
|
||||
@@ -25,7 +25,7 @@ use crate::{
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType, build_queued_payload, delete_stored_payload, is_connectivity_error, open_target_queue_store,
|
||||
persist_queued_payload_to_store, redacted_secret,
|
||||
persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -47,6 +47,8 @@ use uuid::Uuid;
|
||||
/// `TargetError::Timeout`, a connectivity error, so the payload stays queued
|
||||
/// for replay.
|
||||
const MYSQL_CONN_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
/// Absolute ceiling for one INSERT, including pool checkout and server execution.
|
||||
const MYSQL_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Name of the optional idempotency-key column / primary key. Present on tables
|
||||
/// created by this target; absent on legacy two-column tables.
|
||||
@@ -784,29 +786,33 @@ where
|
||||
"Inserting MySQL event"
|
||||
);
|
||||
|
||||
let pool = self.get_or_init_pool().await?;
|
||||
// At this point the pool has already been initialized (get_or_init_pool
|
||||
// succeeded above), so get_conn() failures are always transient: the
|
||||
// connection was lost or the pool is temporarily exhausted.
|
||||
let mut conn = checkout_conn(&pool).await?;
|
||||
|
||||
let event_time = extract_event_time(body)?;
|
||||
let event_data =
|
||||
std::str::from_utf8(body).map_err(|e| TargetError::Serialization(format!("Event body is not valid UTF-8: {e}")))?;
|
||||
|
||||
let quoted_table = quote_table_name(&self.args.table)?;
|
||||
with_delivery_deadline(MYSQL_DELIVERY_TIMEOUT, "MySQL delivery", async {
|
||||
let pool = self.get_or_init_pool().await?;
|
||||
// At this point the pool has already been initialized (get_or_init_pool
|
||||
// succeeded above), so get_conn() failures are always transient: the
|
||||
// connection was lost or the pool is temporarily exhausted.
|
||||
let mut conn = checkout_conn(&pool).await?;
|
||||
|
||||
if self.idempotency_supported.load(Ordering::Relaxed) {
|
||||
let sql = mysql_insert_sql_with_event_id("ed_table);
|
||||
conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
|
||||
.await
|
||||
.map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
|
||||
} else {
|
||||
let sql = mysql_insert_sql_legacy("ed_table);
|
||||
conn.exec_drop(sql, (event_time.as_str(), event_data))
|
||||
.await
|
||||
.map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
|
||||
}
|
||||
if self.idempotency_supported.load(Ordering::Relaxed) {
|
||||
let sql = mysql_insert_sql_with_event_id("ed_table);
|
||||
conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
|
||||
.await
|
||||
.map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
|
||||
} else {
|
||||
let sql = mysql_insert_sql_legacy("ed_table);
|
||||
conn.exec_drop(sql, (event_time.as_str(), event_data))
|
||||
.await
|
||||
.map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
|
||||
self.delivery_counters.record_success();
|
||||
debug!(target_id = %self.id, "MySQL event inserted");
|
||||
|
||||
@@ -25,7 +25,7 @@ use crate::{
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
|
||||
open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret,
|
||||
open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -52,6 +52,8 @@ use publish_error::{classify_nats_flush_error, classify_nats_publish_error};
|
||||
pub(crate) use jetstream::resolve_dedup_id;
|
||||
pub(crate) use validation::{validate_jetstream_settings, validate_jetstream_stream};
|
||||
|
||||
const NATS_CORE_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NATSArgs {
|
||||
pub enable: bool,
|
||||
@@ -397,9 +399,21 @@ where
|
||||
}
|
||||
|
||||
async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
|
||||
let client = self.get_or_connect().await?;
|
||||
if let Err(e) = client.publish(self.args.subject.clone(), body.into()).await {
|
||||
let err = classify_nats_publish_error(&e);
|
||||
let result = with_delivery_deadline(NATS_CORE_DELIVERY_TIMEOUT, "NATS delivery", async {
|
||||
let client = self.get_or_connect().await?;
|
||||
client
|
||||
.publish(self.args.subject.clone(), body.into())
|
||||
.await
|
||||
.map_err(|err| classify_nats_publish_error(&err))?;
|
||||
|
||||
// publish only enqueues the message on the client's outbound channel. Flush to confirm the
|
||||
// message reached the server before delivery is treated as successful (backlog#971).
|
||||
client.flush().await.map_err(|err| classify_nats_flush_error(&err))?;
|
||||
Ok(())
|
||||
})
|
||||
.await;
|
||||
|
||||
if let Err(err) = result {
|
||||
if is_connectivity_error(&err) {
|
||||
self.invalidate_cached_client_connection().await;
|
||||
self.connected.store(false, Ordering::SeqCst);
|
||||
@@ -407,15 +421,6 @@ where
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// publish only enqueues the message on the client's outbound channel. Flush to confirm the
|
||||
// message reached the server before delivery is treated as successful (backlog#971).
|
||||
if let Err(e) = client.flush().await {
|
||||
let err = classify_nats_flush_error(&e);
|
||||
self.invalidate_cached_client_connection().await;
|
||||
self.connected.store(false, Ordering::SeqCst);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
self.delivery_counters.record_success();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ use crate::{
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store, redacted_optional_secret,
|
||||
redacted_secret,
|
||||
redacted_secret, with_delivery_deadline,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -70,6 +70,8 @@ const POSTGRES_POOL_RECYCLE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
/// Absolute ceiling on a single checkout, wrapping `pool.get()` in a Tokio
|
||||
/// timeout as a belt-and-suspenders guard on top of the deadpool timeouts.
|
||||
const POSTGRES_POOL_CHECKOUT_HARD_LIMIT: Duration = Duration::from_secs(20);
|
||||
/// Absolute ceiling for one SQL delivery, including pool checkout and execution.
|
||||
const POSTGRES_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Returns `true` for any `s3:ObjectRemoved:*` event.
|
||||
///
|
||||
@@ -730,30 +732,29 @@ where
|
||||
|
||||
let key = resolve_payload_key(&payload, meta);
|
||||
|
||||
let result = match self.args.format {
|
||||
// For the single-row `namespace` format, an object removal must
|
||||
// delete the row rather than UPSERT it, otherwise stale state
|
||||
// lingers in the table after the object is gone.
|
||||
PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
|
||||
client.execute(&self.namespace_delete_sql, &[&key]).await
|
||||
with_delivery_deadline(POSTGRES_DELIVERY_TIMEOUT, "PostgreSQL delivery", async {
|
||||
match self.args.format {
|
||||
// For the single-row `namespace` format, an object removal must
|
||||
// delete the row rather than UPSERT it, otherwise stale state
|
||||
// lingers in the table after the object is gone.
|
||||
PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
|
||||
client.execute(&self.namespace_delete_sql, &[&key]).await
|
||||
}
|
||||
PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
|
||||
PostgresFormat::Access => {
|
||||
let event_name_str = meta.event_name.to_string();
|
||||
let queued_at_ms = meta.queued_at_unix_ms as i64;
|
||||
client
|
||||
.execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
|
||||
.await
|
||||
}
|
||||
}
|
||||
PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
|
||||
PostgresFormat::Access => {
|
||||
let event_name_str = meta.event_name.to_string();
|
||||
let queued_at_ms = meta.queued_at_unix_ms as i64;
|
||||
client
|
||||
.execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
|
||||
.await
|
||||
}
|
||||
};
|
||||
.map_err(|err| map_pg_error(&err, "PostgreSQL insert failed"))
|
||||
})
|
||||
.await?;
|
||||
|
||||
match result {
|
||||
Ok(_) => {
|
||||
self.delivery_counters.record_success();
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => Err(map_pg_error(&err, "PostgreSQL insert failed")),
|
||||
}
|
||||
self.delivery_counters.record_success();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Probes the table from `init()`. Failure is non-fatal when a queue is
|
||||
|
||||
@@ -24,8 +24,9 @@ use crate::{
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
|
||||
persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
|
||||
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
|
||||
open_target_queue_store, persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
|
||||
with_delivery_deadline,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -39,11 +40,15 @@ use std::fmt;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Mutex as AsyncMutex;
|
||||
use tracing::{info, instrument};
|
||||
use tracing::{info, instrument, warn};
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
const PULSAR_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT: Duration = Duration::from_secs(1);
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PulsarArgs {
|
||||
pub enable: bool,
|
||||
@@ -276,6 +281,23 @@ where
|
||||
self.tls_state.lock().reset();
|
||||
}
|
||||
|
||||
async fn clear_failed_delivery_state(&self) {
|
||||
match tokio::time::timeout(PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT, self.producer.lock()).await {
|
||||
Ok(mut producer) => {
|
||||
producer.take();
|
||||
}
|
||||
Err(_) => {
|
||||
warn!(
|
||||
target_id = %self.id,
|
||||
reason = "producer_cleanup_lock_timeout",
|
||||
"Timed out clearing the Pulsar producer after a failed delivery"
|
||||
);
|
||||
}
|
||||
}
|
||||
self.clear_cached_client();
|
||||
self.connected.store(false, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
async fn get_or_connect_client(&self) -> Result<Pulsar<TokioExecutor>, TargetError> {
|
||||
// When a TLS reload adapter is attached, it drives client rebuilds
|
||||
// in the background. The inline per-send fingerprint check is skipped.
|
||||
@@ -334,20 +356,30 @@ where
|
||||
}
|
||||
|
||||
async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
|
||||
self.init_producer().await?;
|
||||
let mut guard = self.producer.lock().await;
|
||||
let producer = guard
|
||||
.as_mut()
|
||||
.ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
|
||||
let receipt = producer
|
||||
.send_non_blocking(body)
|
||||
.await
|
||||
.map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
|
||||
receipt
|
||||
.await
|
||||
.map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
|
||||
self.delivery_counters.record_success();
|
||||
Ok(())
|
||||
let result = with_delivery_deadline(PULSAR_DELIVERY_TIMEOUT, "Pulsar delivery", async {
|
||||
self.init_producer().await?;
|
||||
let mut guard = self.producer.lock().await;
|
||||
let producer = guard
|
||||
.as_mut()
|
||||
.ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
|
||||
let receipt = producer
|
||||
.send_non_blocking(body)
|
||||
.await
|
||||
.map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
|
||||
receipt
|
||||
.await
|
||||
.map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
|
||||
self.delivery_counters.record_success();
|
||||
Ok(())
|
||||
})
|
||||
.await;
|
||||
|
||||
if let Err(err) = &result
|
||||
&& is_connectivity_error(err)
|
||||
{
|
||||
self.clear_failed_delivery_state().await;
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
@@ -525,6 +557,22 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn failed_delivery_cleanup_is_bounded_when_the_producer_lock_is_busy() {
|
||||
let target = Arc::new(PulsarTarget::<String>::new("pulsar:test".to_string(), base_args()).expect("target should build"));
|
||||
target.connected.store(true, Ordering::SeqCst);
|
||||
let producer_guard = target.producer.lock().await;
|
||||
let cleanup = {
|
||||
let target = Arc::clone(&target);
|
||||
tokio::spawn(async move { target.clear_failed_delivery_state().await })
|
||||
};
|
||||
|
||||
cleanup.await.expect("cleanup task should not panic");
|
||||
|
||||
assert!(!target.connected.load(Ordering::SeqCst));
|
||||
drop(producer_guard);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_redacts_pulsar_secret_fields() {
|
||||
let args = PulsarArgs {
|
||||
|
||||
@@ -26,6 +26,7 @@ use crate::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, is_connectivity_error,
|
||||
mark_target_disconnected_on_connectivity_error, open_target_queue_store, persist_queued_payload_to_store,
|
||||
with_delivery_deadline,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -48,6 +49,19 @@ use tokio::sync::Mutex;
|
||||
use tracing::{debug, info, instrument, warn};
|
||||
use url::Url;
|
||||
|
||||
const REDIS_CONNECTION_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
|
||||
const REDIS_RESPONSE_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
|
||||
|
||||
fn redis_total_delivery_timeout(args: &RedisArgs) -> Duration {
|
||||
let attempts = u32::try_from(args.max_retry_attempts).unwrap_or(u32::MAX);
|
||||
let per_attempt = args
|
||||
.connection_timeout
|
||||
.unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT)
|
||||
.saturating_add(args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT))
|
||||
.saturating_add(args.max_retry_delay.unwrap_or(Duration::from_secs(2)));
|
||||
per_attempt.saturating_mul(attempts)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RedisTlsPolicy {
|
||||
SystemCa,
|
||||
@@ -214,6 +228,16 @@ impl RedisArgs {
|
||||
));
|
||||
}
|
||||
|
||||
if self.connection_timeout == Some(Duration::ZERO) {
|
||||
return Err(TargetError::Configuration(
|
||||
"Redis connection_timeout must be greater than zero".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if self.response_timeout == Some(Duration::ZERO) {
|
||||
return Err(TargetError::Configuration("Redis response_timeout must be greater than zero".to_string()));
|
||||
}
|
||||
|
||||
if self.pipeline_buffer_size == Some(0) {
|
||||
return Err(TargetError::Configuration(
|
||||
"Redis pipeline_buffer_size must be greater than zero".to_string(),
|
||||
@@ -464,71 +488,97 @@ where
|
||||
"Sending Redis payload"
|
||||
);
|
||||
|
||||
let mut attempt = 0usize;
|
||||
let mut last_error = None;
|
||||
while attempt < self.args.max_retry_attempts {
|
||||
attempt += 1;
|
||||
let result = with_delivery_deadline(redis_total_delivery_timeout(&self.args), "Redis delivery", async {
|
||||
let mut attempt = 0usize;
|
||||
let mut last_error = None;
|
||||
while attempt < self.args.max_retry_attempts {
|
||||
attempt += 1;
|
||||
|
||||
let mut publisher = self.get_or_create_publisher().await?;
|
||||
match publisher
|
||||
.publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
|
||||
let connection_timeout = self.args.connection_timeout.unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT);
|
||||
let mut publisher = match with_delivery_deadline(
|
||||
connection_timeout,
|
||||
"Redis connection",
|
||||
self.get_or_create_publisher(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(receiver_count) => {
|
||||
// PUBLISH returns the number of subscribers that received the
|
||||
// message. Redis pub/sub is best-effort: with zero subscribers
|
||||
// the event is delivered to no one, yet the durable copy is
|
||||
// deleted. Warn so operators relying on reliable delivery are
|
||||
// not silently losing events (backlog#982).
|
||||
if receiver_count == 0 {
|
||||
{
|
||||
Ok(publisher) => publisher,
|
||||
Err(err) => {
|
||||
invalidate_cache_on_connectivity_error(&err, || self.invalidate_cached_publisher()).await;
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let response_timeout = self.args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT);
|
||||
match with_delivery_deadline(response_timeout, "Redis publish response", async {
|
||||
publisher
|
||||
.publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
|
||||
.await
|
||||
.map_err(map_redis_error)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(receiver_count) => {
|
||||
// PUBLISH returns the number of subscribers that received the
|
||||
// message. Redis pub/sub is best-effort: with zero subscribers
|
||||
// the event is delivered to no one, yet the durable copy is
|
||||
// deleted. Warn so operators relying on reliable delivery are
|
||||
// not silently losing events (backlog#982).
|
||||
if receiver_count == 0 {
|
||||
warn!(
|
||||
target_id = %self.id,
|
||||
channel = %self.args.channel,
|
||||
"Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
|
||||
);
|
||||
}
|
||||
debug!(
|
||||
target_id = %self.id,
|
||||
channel = %self.args.channel,
|
||||
attempt,
|
||||
receiver_count,
|
||||
"Event published to Redis channel"
|
||||
);
|
||||
self.delivery_counters.record_success();
|
||||
return Ok(());
|
||||
}
|
||||
Err(mapped) => {
|
||||
invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
|
||||
|
||||
warn!(
|
||||
target_id = %self.id,
|
||||
channel = %self.args.channel,
|
||||
"Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
|
||||
attempt,
|
||||
max_attempts = self.args.max_retry_attempts,
|
||||
error = %mapped,
|
||||
"Redis publish attempt failed"
|
||||
);
|
||||
}
|
||||
debug!(
|
||||
target_id = %self.id,
|
||||
channel = %self.args.channel,
|
||||
attempt,
|
||||
receiver_count,
|
||||
"Event published to Redis channel"
|
||||
);
|
||||
self.delivery_counters.record_success();
|
||||
return Ok(());
|
||||
}
|
||||
Err(err) => {
|
||||
let mapped = map_redis_error(err);
|
||||
invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
|
||||
|
||||
warn!(
|
||||
target_id = %self.id,
|
||||
channel = %self.args.channel,
|
||||
attempt,
|
||||
max_attempts = self.args.max_retry_attempts,
|
||||
error = %mapped,
|
||||
"Redis publish attempt failed"
|
||||
);
|
||||
if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
|
||||
last_error = Some(mapped);
|
||||
break;
|
||||
}
|
||||
|
||||
if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
|
||||
last_error = Some(mapped);
|
||||
break;
|
||||
tokio::time::sleep(compute_retry_delay(
|
||||
attempt,
|
||||
self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
|
||||
self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
|
||||
))
|
||||
.await;
|
||||
}
|
||||
|
||||
last_error = Some(mapped);
|
||||
tokio::time::sleep(compute_retry_delay(
|
||||
attempt,
|
||||
self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
|
||||
self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
|
||||
))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
Err(last_error.unwrap_or(TargetError::Unknown(
|
||||
"Redis publish failed without a captured error".to_string(),
|
||||
)))
|
||||
})
|
||||
.await;
|
||||
|
||||
if let Err(err) = &result {
|
||||
invalidate_cache_on_connectivity_error(err, || self.invalidate_cached_publisher()).await;
|
||||
self.connected.store(false, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
self.connected.store(false, Ordering::SeqCst);
|
||||
|
||||
Err(last_error.unwrap_or(TargetError::Unknown("Redis publish failed without a captured error".to_string())))
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
@@ -551,7 +601,7 @@ where
|
||||
// thus a fresh TCP+TLS handshake — on every health check (backlog#982).
|
||||
// ensure_publisher_ready already invalidates the cached manager on a
|
||||
// connectivity error so the next attempt rebuilds it.
|
||||
match tokio::time::timeout(Duration::from_secs(5), self.ensure_publisher_ready()).await {
|
||||
match tokio::time::timeout(REDIS_CONNECTION_TIMEOUT_DEFAULT, self.ensure_publisher_ready()).await {
|
||||
Ok(Ok(())) => {
|
||||
self.connected.store(true, Ordering::SeqCst);
|
||||
Ok(true)
|
||||
@@ -917,6 +967,26 @@ mod tests {
|
||||
assert!(args.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_zero_connection_timeout() {
|
||||
let args = RedisArgs {
|
||||
connection_timeout: Some(Duration::ZERO),
|
||||
..base_args()
|
||||
};
|
||||
|
||||
assert!(args.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_zero_response_timeout() {
|
||||
let args = RedisArgs {
|
||||
response_timeout: Some(Duration::ZERO),
|
||||
..base_args()
|
||||
};
|
||||
|
||||
assert!(args.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_custom_ca_tls_policy() {
|
||||
let args = RedisArgs {
|
||||
@@ -1237,6 +1307,23 @@ mod tests {
|
||||
assert_eq!(target.delivery_snapshot().total_messages, 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn delivery_budget_respects_response_timeout_longer_than_sixty_seconds() {
|
||||
let mut args = base_args();
|
||||
args.max_retry_attempts = 1;
|
||||
args.response_timeout = Some(Duration::from_secs(90));
|
||||
let manager_config = build_redis_connection_manager_config(&args);
|
||||
|
||||
assert_eq!(manager_config.response_timeout(), Some(Duration::from_secs(90)));
|
||||
assert_eq!(redis_total_delivery_timeout(&args), Duration::from_secs(97));
|
||||
with_delivery_deadline(redis_total_delivery_timeout(&args), "Redis delivery", async {
|
||||
tokio::time::sleep(Duration::from_secs(70)).await;
|
||||
Ok::<_, TargetError>(())
|
||||
})
|
||||
.await
|
||||
.expect("the configured delivery budget must not impose a fixed sixty-second cap");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_body_sets_connected_false_after_retry_exhaustion() {
|
||||
let listener = match TcpListener::bind("127.0.0.1:0").await {
|
||||
|
||||
@@ -24,16 +24,17 @@ use crate::{
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, open_target_queue_store,
|
||||
persist_queued_payload_to_store, redacted_secret,
|
||||
TargetHealth, TargetHealthReason, TargetHealthState, TargetTlsState, TargetType, build_queued_payload,
|
||||
build_target_tls_fingerprint, open_target_queue_store, persist_queued_payload_to_store, redacted_secret,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use parking_lot::Mutex;
|
||||
use reqwest::{Client, StatusCode, Url};
|
||||
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::egress::OutboundPolicy;
|
||||
use std::{
|
||||
error::Error as StdError,
|
||||
fmt,
|
||||
marker::PhantomData,
|
||||
sync::{
|
||||
@@ -49,6 +50,69 @@ const LOG_COMPONENT_TARGETS: &str = "targets";
|
||||
const LOG_SUBSYSTEM_WEBHOOK: &str = "webhook";
|
||||
const EVENT_WEBHOOK_TARGET_STATE: &str = "webhook_target_state";
|
||||
const EVENT_WEBHOOK_DELIVERY_STATE: &str = "webhook_delivery_state";
|
||||
const WEBHOOK_HEALTH_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
fn classify_probe_error(err: &reqwest::Error) -> TargetHealthReason {
|
||||
if err.is_timeout() {
|
||||
return TargetHealthReason::TimedOut;
|
||||
}
|
||||
|
||||
let mut source = err.source();
|
||||
while let Some(cause) = source {
|
||||
if cause.downcast_ref::<rustls::Error>().is_some() {
|
||||
return TargetHealthReason::TlsFailure;
|
||||
}
|
||||
if let Some(io_error) = cause.downcast_ref::<std::io::Error>() {
|
||||
match io_error.kind() {
|
||||
std::io::ErrorKind::ConnectionRefused => return TargetHealthReason::ConnectionRefused,
|
||||
std::io::ErrorKind::NotFound | std::io::ErrorKind::AddrNotAvailable => {
|
||||
return TargetHealthReason::DnsFailure;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let label = cause.to_string().to_ascii_lowercase();
|
||||
if label.contains("dns error") || label.contains("failed to lookup") || label.contains("name or service not known") {
|
||||
return TargetHealthReason::DnsFailure;
|
||||
}
|
||||
if label.contains("certificate") || label.contains("tls") {
|
||||
return TargetHealthReason::TlsFailure;
|
||||
}
|
||||
source = cause.source();
|
||||
}
|
||||
|
||||
TargetHealthReason::Unreachable
|
||||
}
|
||||
|
||||
async fn probe_health_url(client: &Client, health_check_url: &Url) -> TargetHealth {
|
||||
match tokio::time::timeout(WEBHOOK_HEALTH_TIMEOUT, client.head(health_check_url.as_str()).send()).await {
|
||||
Ok(Ok(_)) => TargetHealth::online(TargetHealthReason::Reachable),
|
||||
Ok(Err(err)) => TargetHealth::error(classify_probe_error(&err)),
|
||||
Err(_) => TargetHealth::error(TargetHealthReason::TimedOut),
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_delivery_status(status: StatusCode) -> Result<(), TargetError> {
|
||||
if status.is_success() {
|
||||
Ok(())
|
||||
} else if status.is_redirection() {
|
||||
Err(TargetError::Request(format!(
|
||||
"Webhook endpoint returned redirect '{}'; redirects are not followed for webhook delivery",
|
||||
status
|
||||
)))
|
||||
} else if status == StatusCode::FORBIDDEN || status == StatusCode::UNAUTHORIZED {
|
||||
Err(TargetError::Authentication(format!(
|
||||
"Webhook endpoint returned '{}', please check if your auth token is correctly set",
|
||||
status
|
||||
)))
|
||||
} else {
|
||||
Err(TargetError::Request(format!(
|
||||
"Webhook endpoint returned '{}', please check your endpoint configuration",
|
||||
status
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Arguments for configuring a Webhook target
|
||||
#[derive(Clone)]
|
||||
@@ -79,7 +143,7 @@ impl fmt::Debug for WebhookArgs {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("WebhookArgs")
|
||||
.field("enable", &self.enable)
|
||||
.field("endpoint", &self.endpoint)
|
||||
.field("endpoint_origin", &self.endpoint.origin().ascii_serialization())
|
||||
.field("auth_token", &redacted_secret(&self.auth_token))
|
||||
.field("queue_dir", &self.queue_dir)
|
||||
.field("queue_limit", &self.queue_limit)
|
||||
@@ -102,7 +166,8 @@ impl WebhookArgs {
|
||||
if self.endpoint.as_str().is_empty() {
|
||||
return Err(TargetError::Configuration("endpoint empty".to_string()));
|
||||
}
|
||||
validate_outbound_url(&self.endpoint)
|
||||
outbound_policy()?
|
||||
.validate_url(&self.endpoint)
|
||||
.map_err(|err| TargetError::Configuration(format!("webhook endpoint is not allowed: {err}")))?;
|
||||
|
||||
if !self.queue_dir.is_empty() {
|
||||
@@ -183,8 +248,16 @@ where
|
||||
None
|
||||
};
|
||||
|
||||
// Build HTTP client using the helper function
|
||||
let http_client = Arc::new(Mutex::new(Self::build_http_client(&args)?));
|
||||
let http_client = if args.enable {
|
||||
Self::build_http_client(&args)?
|
||||
} else {
|
||||
Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to build disabled webhook HTTP client: {e}")))?
|
||||
};
|
||||
let http_client = Arc::new(Mutex::new(http_client));
|
||||
|
||||
let queue_store = open_target_queue_store(
|
||||
&args.queue_dir,
|
||||
@@ -221,7 +294,19 @@ where
|
||||
}
|
||||
|
||||
fn build_http_client(args: &WebhookArgs) -> Result<Client, TargetError> {
|
||||
let resolver = outbound_policy()?
|
||||
.resolver_for(&args.endpoint)
|
||||
.map_err(|err| TargetError::Configuration(format!("webhook endpoint is not allowed: {err}")))?;
|
||||
Self::build_http_client_with_resolver(args, resolver)
|
||||
}
|
||||
|
||||
fn build_http_client_with_resolver(
|
||||
args: &WebhookArgs,
|
||||
resolver: impl reqwest::dns::Resolve + 'static,
|
||||
) -> Result<Client, TargetError> {
|
||||
let mut client_builder = Client::builder()
|
||||
.no_proxy()
|
||||
.dns_resolver(resolver)
|
||||
.timeout(Duration::from_secs(30))
|
||||
// SSRF hardening (backlog#974): never follow HTTP redirects on webhook delivery.
|
||||
// reqwest follows up to 10 redirects by default, which lets a malicious or
|
||||
@@ -230,11 +315,6 @@ where
|
||||
// bypassing the outbound-endpoint validation performed on the configured URL.
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.user_agent(crate::get_user_agent(crate::ServiceType::Basis));
|
||||
#[cfg(test)]
|
||||
{
|
||||
client_builder = client_builder.no_proxy();
|
||||
}
|
||||
|
||||
// 1. Configure server certificate verification
|
||||
if args.skip_tls_verify {
|
||||
// DANGEROUS: For testing only, skip all certificate verification
|
||||
@@ -243,7 +323,7 @@ where
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
endpoint = %args.endpoint,
|
||||
endpoint_origin = %args.endpoint.origin().ascii_serialization(),
|
||||
state = "tls_verification_skipped",
|
||||
fallback = "danger_accept_invalid_certs",
|
||||
"webhook target state"
|
||||
@@ -308,8 +388,14 @@ where
|
||||
fn health_check_url(endpoint: &Url) -> Result<Url, TargetError> {
|
||||
endpoint
|
||||
.host()
|
||||
.ok_or_else(|| TargetError::Configuration(format!("Webhook endpoint '{}' is missing a host", endpoint)))?;
|
||||
.ok_or_else(|| TargetError::Configuration("Webhook endpoint is missing a host".to_string()))?;
|
||||
let mut health_check_url = endpoint.clone();
|
||||
health_check_url
|
||||
.set_username("")
|
||||
.map_err(|_| TargetError::Configuration("Webhook endpoint contains invalid user information".to_string()))?;
|
||||
health_check_url
|
||||
.set_password(None)
|
||||
.map_err(|_| TargetError::Configuration("Webhook endpoint contains invalid user information".to_string()))?;
|
||||
health_check_url.set_path("/");
|
||||
health_check_url.set_query(None);
|
||||
health_check_url.set_fragment(None);
|
||||
@@ -317,39 +403,34 @@ where
|
||||
Ok(health_check_url)
|
||||
}
|
||||
|
||||
async fn probe_reachability(&self) -> Result<bool, TargetError> {
|
||||
async fn probe_health(&self) -> TargetHealth {
|
||||
let Some(health_check_url) = self.health_check_url.as_ref() else {
|
||||
return Ok(false);
|
||||
return TargetHealth::offline(TargetHealthReason::Unreachable);
|
||||
};
|
||||
|
||||
let client = self.http_client.lock().clone();
|
||||
match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await {
|
||||
Ok(Ok(resp)) => {
|
||||
debug!(
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
status = %resp.status(),
|
||||
health_check_url = %health_check_url,
|
||||
state = "reachability_probe_succeeded",
|
||||
"webhook target state"
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
Ok(Err(err)) if err.is_timeout() => Err(TargetError::Timeout(format!(
|
||||
"Webhook health check request to {} timed out",
|
||||
health_check_url
|
||||
))),
|
||||
Ok(Err(err)) if err.is_connect() => Ok(false),
|
||||
Ok(Err(err)) => Err(TargetError::Network(format!(
|
||||
"Webhook health check request to {} failed: {}",
|
||||
health_check_url, err
|
||||
))),
|
||||
Err(_) => Err(TargetError::Timeout(format!(
|
||||
"Webhook health check request to {} timed out",
|
||||
health_check_url
|
||||
))),
|
||||
let health = probe_health_url(&client, health_check_url).await;
|
||||
if health.state == TargetHealthState::Online {
|
||||
debug!(
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
state = "reachability_probe_succeeded",
|
||||
"webhook target state"
|
||||
);
|
||||
}
|
||||
health
|
||||
}
|
||||
|
||||
async fn probe_reachability(&self) -> Result<bool, TargetError> {
|
||||
let health = self.probe_health().await;
|
||||
match health.state {
|
||||
TargetHealthState::Online => Ok(true),
|
||||
TargetHealthState::Offline | TargetHealthState::Disabled => Ok(false),
|
||||
TargetHealthState::Error => match health.reason {
|
||||
TargetHealthReason::TimedOut => Err(TargetError::Timeout("Webhook health check timed out".to_string())),
|
||||
_ => Err(TargetError::Network(format!("Webhook health check failed: {}", health.reason.as_str()))),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -371,7 +452,6 @@ where
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
health_check_url = ?self.health_check_url,
|
||||
state = "reachable",
|
||||
"webhook target state"
|
||||
);
|
||||
@@ -447,7 +527,7 @@ where
|
||||
if e.is_timeout() || e.is_connect() {
|
||||
TargetError::NotConnected
|
||||
} else {
|
||||
TargetError::Request(format!("Failed to send request: {e}"))
|
||||
TargetError::Request("Webhook delivery request failed".to_string())
|
||||
}
|
||||
})?;
|
||||
|
||||
@@ -456,7 +536,8 @@ where
|
||||
// pool and can be reused (keep-alive) instead of being closed mid-stream
|
||||
// (backlog#983). The body content is not needed for delivery accounting.
|
||||
let _ = resp.bytes().await;
|
||||
if status.is_success() {
|
||||
let result = classify_delivery_status(status);
|
||||
if result.is_ok() {
|
||||
debug!(
|
||||
event = EVENT_WEBHOOK_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
@@ -467,29 +548,15 @@ where
|
||||
"webhook delivery state"
|
||||
);
|
||||
self.delivery_counters.record_success();
|
||||
Ok(())
|
||||
} else if status.is_redirection() {
|
||||
// SSRF hardening (backlog#974): redirects are intentionally not followed
|
||||
// (see build_http_client). Treat a 3xx response as a delivery failure rather
|
||||
// than silently chasing the redirect to a potentially internal target.
|
||||
Err(TargetError::Request(format!(
|
||||
"{} returned redirect '{}'; redirects are not followed for webhook delivery",
|
||||
self.args.endpoint, status
|
||||
)))
|
||||
} else if status == StatusCode::FORBIDDEN {
|
||||
Err(TargetError::Authentication(format!(
|
||||
"{} returned '{}', please check if your auth token is correctly set",
|
||||
self.args.endpoint, status
|
||||
)))
|
||||
} else {
|
||||
Err(TargetError::Request(format!(
|
||||
"{} returned '{}', please check your endpoint configuration",
|
||||
self.args.endpoint, status
|
||||
)))
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
fn outbound_policy() -> Result<&'static OutboundPolicy, TargetError> {
|
||||
OutboundPolicy::from_env_cached().map_err(|err| TargetError::Configuration(format!("invalid outbound policy: {err}")))
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<E> Target<E> for WebhookTarget<E>
|
||||
where
|
||||
@@ -507,6 +574,13 @@ where
|
||||
self.probe_reachability().await
|
||||
}
|
||||
|
||||
async fn health(&self) -> TargetHealth {
|
||||
if !self.args.enable {
|
||||
return TargetHealth::disabled();
|
||||
}
|
||||
self.probe_health().await
|
||||
}
|
||||
|
||||
async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
|
||||
let queued = match self.build_queued_payload(&event) {
|
||||
Ok(queued) => queued,
|
||||
@@ -700,11 +774,63 @@ where
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{WebhookArgs, WebhookTarget};
|
||||
use crate::target::{REDACTED_SECRET, Target, TargetType, decode_object_name};
|
||||
use super::{WebhookArgs, WebhookTarget, classify_delivery_status, probe_health_url};
|
||||
use crate::target::{REDACTED_SECRET, Target, TargetHealthReason, TargetHealthState, TargetType, decode_object_name};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use url::Url;
|
||||
use url::form_urlencoded;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct StaticResolver(IpAddr);
|
||||
|
||||
impl reqwest::dns::Resolve for StaticResolver {
|
||||
fn resolve(&self, _name: reqwest::dns::Name) -> reqwest::dns::Resolving {
|
||||
let address = SocketAddr::new(self.0, 0);
|
||||
Box::pin(async move { Ok(Box::new(std::iter::once(address)) as reqwest::dns::Addrs) })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct FailingResolver;
|
||||
|
||||
impl reqwest::dns::Resolve for FailingResolver {
|
||||
fn resolve(&self, _name: reqwest::dns::Name) -> reqwest::dns::Resolving {
|
||||
Box::pin(async { Err(std::io::Error::new(std::io::ErrorKind::NotFound, "test DNS failure").into()) })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLog(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
|
||||
|
||||
struct CapturedLogWriter(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
|
||||
|
||||
impl std::io::Write for CapturedLogWriter {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
self.0.lock().expect("captured log lock").extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'writer> tracing_subscriber::fmt::MakeWriter<'writer> for CapturedLog {
|
||||
type Writer = CapturedLogWriter;
|
||||
|
||||
fn make_writer(&'writer self) -> Self::Writer {
|
||||
CapturedLogWriter(self.0.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl CapturedLog {
|
||||
fn contents(&self) -> String {
|
||||
String::from_utf8(self.0.lock().expect("captured log lock").clone()).expect("captured logs must be UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
fn base_args() -> WebhookArgs {
|
||||
WebhookArgs {
|
||||
enable: true,
|
||||
@@ -720,9 +846,25 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn http_status_url(status: u16) -> Url {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind test server");
|
||||
let address = listener.local_addr().expect("test server address");
|
||||
tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("accept health probe");
|
||||
let mut request = [0u8; 1024];
|
||||
let _ = stream.read(&mut request).await;
|
||||
stream
|
||||
.write_all(format!("HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").as_bytes())
|
||||
.await
|
||||
.expect("write health response");
|
||||
});
|
||||
Url::parse(&format!("http://{address}/")).expect("health probe URL")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_redacts_webhook_secret_fields() {
|
||||
let args = WebhookArgs {
|
||||
endpoint: Url::parse("https://user:password@example.com/private?token=query-secret").expect("debug URL"),
|
||||
auth_token: "webhook-token".to_string(),
|
||||
client_key: "/etc/rustfs/webhook.key".to_string(),
|
||||
..base_args()
|
||||
@@ -732,10 +874,111 @@ mod tests {
|
||||
|
||||
assert!(!rendered.contains("webhook-token"));
|
||||
assert!(!rendered.contains("/etc/rustfs/webhook.key"));
|
||||
assert!(!rendered.contains("password"));
|
||||
assert!(!rendered.contains("/private"));
|
||||
assert!(!rendered.contains("query-secret"));
|
||||
assert!(rendered.contains(REDACTED_SECRET));
|
||||
assert!(rendered.contains("https://example.com"));
|
||||
assert!(rendered.contains("WebhookArgs"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webhook_client_uses_the_supplied_connection_resolver() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind resolver test listener");
|
||||
let address = listener.local_addr().expect("resolver test listener address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("accept resolved webhook request");
|
||||
let mut request = [0_u8; 1024];
|
||||
let read = stream.read(&mut request).await.expect("read webhook request");
|
||||
assert!(String::from_utf8_lossy(&request[..read]).starts_with("GET /hook HTTP/1.1"));
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("write webhook response");
|
||||
});
|
||||
let args = WebhookArgs {
|
||||
endpoint: Url::parse(&format!("http://webhook.test:{}/hook", address.port())).expect("endpoint should parse"),
|
||||
..base_args()
|
||||
};
|
||||
let client = WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(&args, StaticResolver(address.ip()))
|
||||
.expect("webhook client should build");
|
||||
|
||||
let response = client
|
||||
.get(args.endpoint)
|
||||
.send()
|
||||
.await
|
||||
.expect("resolver should route request to test listener");
|
||||
assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT);
|
||||
server.await.expect("resolver test server should finish");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn webhook_client_ignores_environment_proxies_before_dns_filtering() {
|
||||
const CHILD_ENV: &str = "RUSTFS_TEST_WEBHOOK_PROXY_CHILD";
|
||||
const TARGET_URL_ENV: &str = "RUSTFS_TEST_WEBHOOK_PROXY_TARGET_URL";
|
||||
const TARGET_ADDR_ENV: &str = "RUSTFS_TEST_WEBHOOK_PROXY_TARGET_ADDR";
|
||||
|
||||
if std::env::var_os(CHILD_ENV).is_some() {
|
||||
let endpoint = Url::parse(&std::env::var(TARGET_URL_ENV).expect("child target URL")).expect("target URL");
|
||||
let address = std::env::var(TARGET_ADDR_ENV)
|
||||
.expect("child target address")
|
||||
.parse::<SocketAddr>()
|
||||
.expect("target address");
|
||||
let args = WebhookArgs { endpoint, ..base_args() };
|
||||
let client = WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(&args, StaticResolver(address.ip()))
|
||||
.expect("webhook client should build");
|
||||
tokio::runtime::Runtime::new().expect("child runtime").block_on(async {
|
||||
let response = client
|
||||
.get(args.endpoint)
|
||||
.send()
|
||||
.await
|
||||
.expect("webhook request should bypass environment proxy");
|
||||
assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener as StdTcpListener;
|
||||
|
||||
let target_listener = StdTcpListener::bind("127.0.0.1:0").expect("bind webhook target listener");
|
||||
let target_address = target_listener.local_addr().expect("webhook target address");
|
||||
let target = std::thread::spawn(move || {
|
||||
let (mut stream, _) = target_listener.accept().expect("accept direct webhook request");
|
||||
let mut request = [0_u8; 1024];
|
||||
let read = stream.read(&mut request).expect("read direct webhook request");
|
||||
assert!(String::from_utf8_lossy(&request[..read]).starts_with("GET /hook HTTP/1.1"));
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.expect("write direct webhook response");
|
||||
});
|
||||
let proxy_listener = StdTcpListener::bind("127.0.0.1:0").expect("reserve refused proxy address");
|
||||
let proxy_address = proxy_listener.local_addr().expect("proxy address");
|
||||
drop(proxy_listener);
|
||||
let target_url = format!("http://webhook.test:{}/hook", target_address.port());
|
||||
let proxy_url = format!("http://{proxy_address}");
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("resolve current test executable"))
|
||||
.arg("webhook_client_ignores_environment_proxies_before_dns_filtering")
|
||||
.arg("--nocapture")
|
||||
.env(CHILD_ENV, "1")
|
||||
.env(TARGET_URL_ENV, target_url)
|
||||
.env(TARGET_ADDR_ENV, target_address.to_string())
|
||||
.env("HTTP_PROXY", &proxy_url)
|
||||
.env("HTTPS_PROXY", &proxy_url)
|
||||
.env("ALL_PROXY", &proxy_url)
|
||||
.env("NO_PROXY", "")
|
||||
.output()
|
||||
.expect("run isolated proxy test child");
|
||||
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"proxy test child failed: stdout={} stderr={}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
target.join().expect("direct webhook target should finish");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_skip_tls_verify_and_client_ca_mutually_exclusive() {
|
||||
let args = WebhookArgs {
|
||||
@@ -770,6 +1013,36 @@ mod tests {
|
||||
assert!(args.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn webhook_tls_warning_redacts_endpoint_details() {
|
||||
let captured = CapturedLog::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
.with_writer(captured.clone())
|
||||
.finish();
|
||||
let args = WebhookArgs {
|
||||
endpoint: Url::parse("https://webhook.test/private?token=secret").expect("webhook endpoint"),
|
||||
skip_tls_verify: true,
|
||||
..base_args()
|
||||
};
|
||||
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(
|
||||
&args,
|
||||
StaticResolver(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)),
|
||||
)
|
||||
.expect("webhook client should build");
|
||||
});
|
||||
|
||||
let logs = captured.contents();
|
||||
assert!(logs.contains("https://webhook.test"));
|
||||
for secret in ["/private", "token=secret"] {
|
||||
assert!(!logs.contains(secret), "TLS warning leaked {secret}: {logs}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rejects_loopback_endpoint() {
|
||||
let args = WebhookArgs {
|
||||
@@ -824,12 +1097,133 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_health_check_url_ignores_endpoint_path() {
|
||||
let endpoint = Url::parse("https://example.com:9443/hook/path").unwrap();
|
||||
let health_check_url = WebhookTarget::<serde_json::Value>::health_check_url(&endpoint).unwrap();
|
||||
let endpoint = Url::parse("https://user:password@example.com:9443/hook/path?token=secret").expect("webhook endpoint URL");
|
||||
let health_check_url = WebhookTarget::<serde_json::Value>::health_check_url(&endpoint).expect("webhook health-check URL");
|
||||
|
||||
assert_eq!(health_check_url.as_str(), "https://example.com:9443/");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn head_http_responses_only_measure_reachability() {
|
||||
let client = WebhookTarget::<serde_json::Value>::build_http_client(&base_args()).expect("build client");
|
||||
for status in [401, 404, 500] {
|
||||
let health = probe_health_url(&client, &http_status_url(status).await).await;
|
||||
assert_eq!(health.state, TargetHealthState::Online, "HEAD {status} is reachable");
|
||||
assert_eq!(health.reason, TargetHealthReason::Reachable);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refused_connection_has_stable_health_reason() {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve refused port");
|
||||
let address = listener.local_addr().expect("refused address");
|
||||
drop(listener);
|
||||
let url = Url::parse(&format!("http://{address}/")).expect("refused URL");
|
||||
let client = WebhookTarget::<serde_json::Value>::build_http_client(&base_args()).expect("build client");
|
||||
|
||||
let health = probe_health_url(&client, &url).await;
|
||||
|
||||
assert_eq!(health.state, TargetHealthState::Error);
|
||||
assert_eq!(health.reason, TargetHealthReason::ConnectionRefused);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dns_failure_has_stable_health_reason() {
|
||||
let url = Url::parse("http://unresolvable.test/").expect("invalid test domain URL");
|
||||
let client = WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(&base_args(), FailingResolver)
|
||||
.expect("build client");
|
||||
|
||||
let health = probe_health_url(&client, &url).await;
|
||||
|
||||
assert_eq!(health.state, TargetHealthState::Error);
|
||||
// On systems with DNS interception (common on macOS), `.invalid` may resolve
|
||||
// to an interception address, producing `Unreachable` instead of `DnsFailure`.
|
||||
assert!(
|
||||
matches!(health.reason, TargetHealthReason::DnsFailure | TargetHealthReason::Unreachable),
|
||||
"expected DnsFailure or Unreachable, got {:?}",
|
||||
health.reason
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn webhook_health_probe_has_a_five_second_total_budget() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind stalled server");
|
||||
let address = listener.local_addr().expect("stalled server address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (_stream, _) = listener.accept().await.expect("accept stalled probe");
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
let url = Url::parse(&format!("http://{address}/")).expect("stalled URL");
|
||||
let client = WebhookTarget::<serde_json::Value>::build_http_client(&base_args()).expect("build client");
|
||||
let started = tokio::time::Instant::now();
|
||||
|
||||
let health = probe_health_url(&client, &url).await;
|
||||
|
||||
assert_eq!(started.elapsed(), super::WEBHOOK_HEALTH_TIMEOUT);
|
||||
assert_eq!(health.state, TargetHealthState::Error);
|
||||
assert_eq!(health.reason, TargetHealthReason::TimedOut);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tls_failure_has_stable_health_reason() {
|
||||
use rustls::{
|
||||
ServerConfig, ServerConnection, StreamOwned,
|
||||
pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
|
||||
};
|
||||
use std::io::Read;
|
||||
use std::sync::{Arc, Once};
|
||||
|
||||
static INSTALL_CRYPTO_PROVIDER: Once = Once::new();
|
||||
INSTALL_CRYPTO_PROVIDER.call_once(|| {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
});
|
||||
let rcgen::CertifiedKey { cert, signing_key } =
|
||||
rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).expect("cert should generate");
|
||||
let server_config = Arc::new(
|
||||
ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_single_cert(
|
||||
vec![cert.der().clone()],
|
||||
PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(signing_key.serialize_der())),
|
||||
)
|
||||
.expect("server cert should be valid"),
|
||||
);
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind TLS server");
|
||||
let address = listener.local_addr().expect("TLS server address");
|
||||
let server = std::thread::spawn(move || {
|
||||
let (stream, _) = listener.accept().expect("accept TLS client");
|
||||
let connection = ServerConnection::new(server_config).expect("server connection");
|
||||
let mut tls_stream = StreamOwned::new(connection, stream);
|
||||
let mut request = [0u8; 1024];
|
||||
let _ = tls_stream.read(&mut request);
|
||||
});
|
||||
let url = Url::parse(&format!("https://localhost:{}/", address.port())).expect("TLS health URL");
|
||||
let client =
|
||||
WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(&base_args(), StaticResolver(address.ip()))
|
||||
.expect("build client");
|
||||
|
||||
let health = probe_health_url(&client, &url).await;
|
||||
|
||||
assert_eq!(health.state, TargetHealthState::Error);
|
||||
assert_eq!(health.reason, TargetHealthReason::TlsFailure);
|
||||
server.join().expect("TLS server thread");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn post_status_classification_requires_success() {
|
||||
assert!(classify_delivery_status(reqwest::StatusCode::NO_CONTENT).is_ok());
|
||||
for status in [
|
||||
reqwest::StatusCode::MOVED_PERMANENTLY,
|
||||
reqwest::StatusCode::UNAUTHORIZED,
|
||||
reqwest::StatusCode::INTERNAL_SERVER_ERROR,
|
||||
] {
|
||||
assert!(classify_delivery_status(status).is_err(), "POST {status} must fail");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_disabled_target_can_be_constructed_without_origin_probe() {
|
||||
let args = WebhookArgs {
|
||||
@@ -941,18 +1335,20 @@ mod tests {
|
||||
});
|
||||
|
||||
let args = WebhookArgs {
|
||||
endpoint: Url::parse(&format!("https://localhost:{}/hook", addr.port())).expect("endpoint should parse"),
|
||||
client_ca: ca_path.to_string_lossy().into_owned(),
|
||||
..base_args()
|
||||
};
|
||||
let client = WebhookTarget::<serde_json::Value>::build_http_client(&args).expect("build https client");
|
||||
let client = WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(&args, StaticResolver(addr.ip()))
|
||||
.expect("build https client");
|
||||
let resp = client
|
||||
.head(format!("https://localhost:{}/hook", addr.port()))
|
||||
.head(args.endpoint)
|
||||
.send()
|
||||
.await
|
||||
.expect("https webhook probe should trust configured ca");
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
assert_eq!(resp.text_with_charset("utf-8").await.expect("read response body"), "");
|
||||
assert!(resp.bytes().await.expect("read response body").is_empty());
|
||||
handle.join().expect("tls server thread");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,6 +42,7 @@ lz4 = { workspace = true, optional = true }
|
||||
md-5 = { workspace = true, optional = true }
|
||||
netif = { workspace = true, optional = true }
|
||||
regex = { workspace = true, optional = true }
|
||||
reqwest = { workspace = true, optional = true }
|
||||
rustix = { workspace = true, optional = true, features = ["fs"] }
|
||||
serde = { workspace = true, optional = true, features = ["derive"] }
|
||||
sha1 = { workspace = true, optional = true }
|
||||
@@ -50,7 +51,7 @@ convert_case = { workspace = true, optional = true }
|
||||
siphasher = { workspace = true, optional = true }
|
||||
snap = { workspace = true, optional = true }
|
||||
tempfile = { workspace = true, optional = true }
|
||||
tokio = { workspace = true, optional = true, features = ["io-util", "time"] }
|
||||
tokio = { workspace = true, optional = true, features = ["io-util", "net", "time"] }
|
||||
tracing = { workspace = true }
|
||||
transform-stream = { workspace = true, optional = true }
|
||||
url = { workspace = true, optional = true }
|
||||
@@ -72,7 +73,7 @@ workspace = true
|
||||
default = ["ip"] # features that are enabled by default
|
||||
ip = ["dep:local-ip-address"] # ip characteristics and their dependencies
|
||||
net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:hyper", "dep:tokio"] # network features with DNS resolver
|
||||
egress = ["ip", "dep:url"]
|
||||
egress = ["ip", "dep:reqwest", "dep:tokio", "dep:url"]
|
||||
io = ["dep:tokio"]
|
||||
path = [] # path manipulation features
|
||||
compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
|
||||
|
||||
+704
-34
@@ -12,10 +12,24 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashSet;
|
||||
#[cfg(test)]
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::fmt;
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
|
||||
use std::sync::OnceLock;
|
||||
#[cfg(test)]
|
||||
use std::sync::{Arc, Mutex};
|
||||
use url::Url;
|
||||
|
||||
/// Comma-separated exact HTTP(S) origins that may resolve to otherwise restricted addresses.
|
||||
///
|
||||
/// This is an operator-owned process setting. Target configuration cannot extend it. Link-local,
|
||||
/// metadata, and unspecified addresses remain forbidden even when their origin appears in the list.
|
||||
pub const ENV_OUTBOUND_ALLOW_ORIGINS: &str = "RUSTFS_OUTBOUND_ALLOW_ORIGINS";
|
||||
|
||||
static OUTBOUND_POLICY: OnceLock<Result<OutboundPolicy, OutboundPolicyError>> = OnceLock::new();
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum OutboundUrlError {
|
||||
MissingHost,
|
||||
@@ -35,6 +49,199 @@ impl fmt::Display for OutboundUrlError {
|
||||
|
||||
impl std::error::Error for OutboundUrlError {}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum OutboundPolicyError {
|
||||
NonUnicodeEnvironment,
|
||||
InvalidAllowedOrigin { position: usize },
|
||||
MissingHost,
|
||||
UnsupportedScheme { scheme: String },
|
||||
UserInfoNotAllowed,
|
||||
ForbiddenHost { host: String, reason: &'static str },
|
||||
}
|
||||
|
||||
impl fmt::Display for OutboundPolicyError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
OutboundPolicyError::NonUnicodeEnvironment => {
|
||||
write!(f, "{ENV_OUTBOUND_ALLOW_ORIGINS} must contain valid Unicode")
|
||||
}
|
||||
OutboundPolicyError::InvalidAllowedOrigin { position } => {
|
||||
write!(f, "invalid origin at position {position} in {ENV_OUTBOUND_ALLOW_ORIGINS}")
|
||||
}
|
||||
OutboundPolicyError::MissingHost => write!(f, "outbound URL is missing a host"),
|
||||
OutboundPolicyError::UnsupportedScheme { scheme } => {
|
||||
write!(f, "outbound URL scheme '{scheme}' is not allowed")
|
||||
}
|
||||
OutboundPolicyError::UserInfoNotAllowed => write!(f, "outbound URL userinfo is not allowed"),
|
||||
OutboundPolicyError::ForbiddenHost { host, reason } => {
|
||||
write!(f, "outbound URL host '{host}' is not allowed: {reason}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for OutboundPolicyError {}
|
||||
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
pub struct OutboundPolicy {
|
||||
allowed_restricted_origins: HashSet<String>,
|
||||
}
|
||||
|
||||
impl OutboundPolicy {
|
||||
pub fn from_env() -> Result<Self, OutboundPolicyError> {
|
||||
match std::env::var(ENV_OUTBOUND_ALLOW_ORIGINS) {
|
||||
Ok(value) => Self::from_allowed_origins(&value),
|
||||
Err(std::env::VarError::NotPresent) => Ok(Self::default()),
|
||||
Err(std::env::VarError::NotUnicode(_)) => Err(OutboundPolicyError::NonUnicodeEnvironment),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_env_cached() -> Result<&'static Self, OutboundPolicyError> {
|
||||
match OUTBOUND_POLICY.get_or_init(Self::from_env) {
|
||||
Ok(policy) => Ok(policy),
|
||||
Err(err) => Err(err.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_allowed_origins(value: &str) -> Result<Self, OutboundPolicyError> {
|
||||
let mut allowed_restricted_origins = HashSet::new();
|
||||
if value.trim().is_empty() {
|
||||
return Ok(Self::default());
|
||||
}
|
||||
for (position, entry) in value.split(',').map(str::trim).enumerate() {
|
||||
let position = position + 1;
|
||||
if entry.is_empty() {
|
||||
return Err(OutboundPolicyError::InvalidAllowedOrigin { position });
|
||||
}
|
||||
let url = Url::parse(entry).map_err(|_| OutboundPolicyError::InvalidAllowedOrigin { position })?;
|
||||
let origin = normalized_origin(&url).ok_or(OutboundPolicyError::InvalidAllowedOrigin { position })?;
|
||||
if url.path() != "/" || url.query().is_some() || url.fragment().is_some() {
|
||||
return Err(OutboundPolicyError::InvalidAllowedOrigin { position });
|
||||
}
|
||||
allowed_restricted_origins.insert(origin);
|
||||
}
|
||||
Ok(Self {
|
||||
allowed_restricted_origins,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn validate_url(&self, url: &Url) -> Result<(), OutboundPolicyError> {
|
||||
validate_http_url_shape(url)?;
|
||||
let raw_host = url.host_str().ok_or(OutboundPolicyError::MissingHost)?;
|
||||
let normalized_host = raw_host.trim_end_matches('.').trim_matches(['[', ']']);
|
||||
let result = if normalized_host.eq_ignore_ascii_case("localhost") {
|
||||
Err("loopback host")
|
||||
} else if let Ok(ip) = normalized_host.parse::<IpAddr>() {
|
||||
validate_policy_ip(ip)
|
||||
} else {
|
||||
Ok(())
|
||||
};
|
||||
match result {
|
||||
Ok(()) => Ok(()),
|
||||
Err(reason) if self.allows_restricted_origin(url) && restricted_reason_can_be_overridden(reason) => Ok(()),
|
||||
Err(reason) => Err(OutboundPolicyError::ForbiddenHost {
|
||||
host: raw_host.to_string(),
|
||||
reason,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resolver_for(&self, url: &Url) -> Result<OutboundDnsResolver, OutboundPolicyError> {
|
||||
self.validate_url(url)?;
|
||||
let host = normalized_host(url).ok_or(OutboundPolicyError::MissingHost)?;
|
||||
Ok(OutboundDnsResolver::new(host, self.allows_restricted_origin(url)))
|
||||
}
|
||||
|
||||
fn allows_restricted_origin(&self, url: &Url) -> bool {
|
||||
normalized_origin(url).is_some_and(|origin| self.allowed_restricted_origins.contains(&origin))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct OutboundDnsResolver {
|
||||
allowed_restricted_host: Option<String>,
|
||||
#[cfg(test)]
|
||||
overrides: Option<DnsOverrideAnswers>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type DnsOverrideAnswers = Arc<Mutex<HashMap<String, VecDeque<Vec<SocketAddr>>>>>;
|
||||
|
||||
impl OutboundDnsResolver {
|
||||
fn new(host: String, allow_restricted: bool) -> Self {
|
||||
Self {
|
||||
allowed_restricted_host: allow_restricted.then_some(host),
|
||||
#[cfg(test)]
|
||||
overrides: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn with_overrides(mut self, overrides: HashMap<String, Vec<IpAddr>>) -> Self {
|
||||
let overrides = overrides
|
||||
.into_iter()
|
||||
.map(|(host, ips)| {
|
||||
let addresses = ips.into_iter().map(|ip| SocketAddr::new(ip, 0)).collect();
|
||||
(host, VecDeque::from([addresses]))
|
||||
})
|
||||
.collect();
|
||||
self.overrides = Some(Arc::new(Mutex::new(overrides)));
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn with_override_sequence(mut self, host: &str, answers: Vec<Vec<SocketAddr>>) -> Self {
|
||||
self.overrides = Some(Arc::new(Mutex::new(HashMap::from([(host.to_string(), VecDeque::from(answers))]))));
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl reqwest::dns::Resolve for OutboundDnsResolver {
|
||||
fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
|
||||
let host = name.as_str().trim_end_matches('.').to_ascii_lowercase();
|
||||
let allow_restricted = self.allowed_restricted_host.as_deref() == Some(host.as_str());
|
||||
#[cfg(test)]
|
||||
let overrides = self.overrides.clone();
|
||||
Box::pin(async move {
|
||||
#[cfg(test)]
|
||||
let overridden = overrides.as_ref().and_then(|entries| {
|
||||
let mut entries = entries.lock().expect("test DNS overrides must not be poisoned");
|
||||
let answers = entries.get_mut(&host)?;
|
||||
if answers.len() > 1 {
|
||||
answers.pop_front()
|
||||
} else {
|
||||
answers.front().cloned()
|
||||
}
|
||||
});
|
||||
#[cfg(not(test))]
|
||||
let overridden: Option<Vec<SocketAddr>> = None;
|
||||
|
||||
let addresses = if let Some(addresses) = overridden {
|
||||
addresses
|
||||
} else {
|
||||
// Do not use the cached resolver in `crate::net`: every new connection must
|
||||
// classify the addresses returned at that boundary so DNS rebinding fails closed.
|
||||
tokio::net::lookup_host((host.as_str(), 0))
|
||||
.await
|
||||
.map_err(|err| std::io::Error::new(std::io::ErrorKind::NotFound, err))?
|
||||
.collect()
|
||||
};
|
||||
let addrs = addresses
|
||||
.into_iter()
|
||||
.filter(|address| resolved_ip_allowed(address.ip(), allow_restricted))
|
||||
.collect::<Vec<_>>();
|
||||
if addrs.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::PermissionDenied,
|
||||
format!("outbound DNS resolution for '{host}' returned no allowed addresses"),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate_outbound_url(url: &Url) -> Result<(), OutboundUrlError> {
|
||||
let Some(raw_host) = url.host_str() else {
|
||||
return Err(OutboundUrlError::MissingHost);
|
||||
@@ -58,10 +265,168 @@ pub fn validate_outbound_url(url: &Url) -> Result<(), OutboundUrlError> {
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_http_url_shape(url: &Url) -> Result<(), OutboundPolicyError> {
|
||||
if !matches!(url.scheme(), "http" | "https") {
|
||||
return Err(OutboundPolicyError::UnsupportedScheme {
|
||||
scheme: url.scheme().to_string(),
|
||||
});
|
||||
}
|
||||
if !url.username().is_empty() || url.password().is_some() {
|
||||
return Err(OutboundPolicyError::UserInfoNotAllowed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalized_host(url: &Url) -> Option<String> {
|
||||
url.host_str()
|
||||
.map(|host| host.trim_end_matches('.').trim_matches(&['[', ']'][..]).to_ascii_lowercase())
|
||||
}
|
||||
|
||||
fn normalized_origin(url: &Url) -> Option<String> {
|
||||
validate_http_url_shape(url).ok()?;
|
||||
let host = normalized_host(url)?;
|
||||
let port = url.port_or_known_default()?;
|
||||
// `Url::origin()` preserves a trailing DNS dot, but the resolver treats dotted and undotted
|
||||
// names as the same host. Normalize it here so policy matching follows connection semantics.
|
||||
if host.parse::<Ipv6Addr>().is_ok() {
|
||||
Some(format!("{}://[{host}]:{port}", url.scheme()))
|
||||
} else {
|
||||
Some(format!("{}://{host}:{port}", url.scheme()))
|
||||
}
|
||||
}
|
||||
|
||||
fn restricted_reason_can_be_overridden(reason: &str) -> bool {
|
||||
matches!(
|
||||
reason,
|
||||
"loopback host" | "loopback address" | "private address" | "shared address" | "reserved address"
|
||||
)
|
||||
}
|
||||
|
||||
fn resolved_ip_allowed(ip: IpAddr, allow_restricted: bool) -> bool {
|
||||
match validate_policy_ip(ip) {
|
||||
Ok(()) => true,
|
||||
Err(reason) => allow_restricted && restricted_reason_can_be_overridden(reason),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_policy_ip(ip: IpAddr) -> Result<(), &'static str> {
|
||||
if is_metadata_endpoint(ip) {
|
||||
return Err("metadata endpoint");
|
||||
}
|
||||
|
||||
let original_v6 = match ip {
|
||||
IpAddr::V6(v6) => Some(v6),
|
||||
IpAddr::V4(_) => None,
|
||||
};
|
||||
let ip = match original_v6 {
|
||||
Some(v6) => match policy_embedded_ipv4(v6) {
|
||||
Some(v4) => IpAddr::V4(v4),
|
||||
None => IpAddr::V6(v6),
|
||||
},
|
||||
None => ip,
|
||||
};
|
||||
|
||||
if is_metadata_endpoint(ip) {
|
||||
return Err("metadata endpoint");
|
||||
}
|
||||
validate_outbound_ip(ip)?;
|
||||
if original_v6.is_some_and(|v6| v6.segments()[0] == 0x2002) {
|
||||
return Err("reserved address");
|
||||
}
|
||||
|
||||
match ip {
|
||||
IpAddr::V4(ipv4) => {
|
||||
let octets = ipv4.octets();
|
||||
if octets[0] == 100 && (64..=127).contains(&octets[1]) {
|
||||
return Err("shared address");
|
||||
}
|
||||
if octets[0] == 0
|
||||
|| octets[0] >= 224
|
||||
|| (octets[0] == 192 && ((octets[1] == 0 && matches!(octets[2], 0 | 2)) || (octets[1] == 88 && octets[2] == 99)))
|
||||
|| (octets[0] == 198 && (octets[1] == 18 || octets[1] == 19))
|
||||
|| (octets[0] == 198 && octets[1] == 51 && octets[2] == 100)
|
||||
|| (octets[0] == 203 && octets[1] == 0 && octets[2] == 113)
|
||||
{
|
||||
return Err("reserved address");
|
||||
}
|
||||
}
|
||||
IpAddr::V6(ipv6) => {
|
||||
let segments = ipv6.segments();
|
||||
if segments[0] == 0x0064 && segments[1] == 0xff9b && segments[2] == 1 {
|
||||
return Err("translation address");
|
||||
}
|
||||
if is_special_use_ipv6(ipv6) {
|
||||
return Err("reserved address");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_special_use_ipv6(ip: Ipv6Addr) -> bool {
|
||||
let segments = ip.segments();
|
||||
let bits = u128::from_be_bytes(ip.octets());
|
||||
let ietf_protocol_assignment = segments[0] == 0x2001
|
||||
&& segments[1] < 0x0200
|
||||
&& bits != 0x2001_0001_0000_0000_0000_0000_0000_0001
|
||||
&& bits != 0x2001_0001_0000_0000_0000_0000_0000_0002
|
||||
&& segments[1] != 0x0003
|
||||
&& !(segments[1] == 0x0004 && segments[2] == 0x0112)
|
||||
&& !(0x0020..=0x003f).contains(&segments[1]);
|
||||
|
||||
(segments[0] & 0xff00) == 0xff00
|
||||
|| (segments[0] & 0xffc0) == 0xfec0
|
||||
|| (segments[0] == 0x0100 && segments[1..4] == [0, 0, 0])
|
||||
|| ietf_protocol_assignment
|
||||
|| segments[0] == 0x2002
|
||||
|| (segments[0] == 0x2001 && segments[1] == 0x0db8)
|
||||
|| (segments[0] == 0x3fff && segments[1] <= 0x0fff)
|
||||
|| segments[0] == 0x5f00
|
||||
}
|
||||
|
||||
fn is_metadata_endpoint(ip: IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(ipv4) => matches!(
|
||||
ipv4.octets(),
|
||||
[169, 254, 169, 254]
|
||||
| [169, 254, 170, 2]
|
||||
| [169, 254, 170, 23]
|
||||
| [169, 254, 0, 23]
|
||||
| [100, 100, 100, 200]
|
||||
| [168, 63, 129, 16]
|
||||
),
|
||||
IpAddr::V6(ipv6) => matches!(
|
||||
ipv6.segments(),
|
||||
[0xfd00, 0x0ec2, 0, 0, 0, 0, 0, 0x0254]
|
||||
| [0xfd00, 0x0ec2, 0, 0, 0, 0, 0, 0x0023]
|
||||
| [0xfd20, 0x00ce, 0, 0, 0, 0, 0, 0x0254]
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn policy_embedded_ipv4(v6: Ipv6Addr) -> Option<Ipv4Addr> {
|
||||
if let Some(v4) = embedded_ipv4(v6) {
|
||||
return Some(v4);
|
||||
}
|
||||
let segments = v6.segments();
|
||||
if segments[..6] == [0, 0, 0, 0, 0xffff, 0] {
|
||||
let hi = segments[6].to_be_bytes();
|
||||
let lo = segments[7].to_be_bytes();
|
||||
return Some(Ipv4Addr::new(hi[0], hi[1], lo[0], lo[1]));
|
||||
}
|
||||
let octets = v6.octets();
|
||||
if octets[..12] == [0x00, 0x64, 0xff, 0x9b, 0, 0, 0, 0, 0, 0, 0, 0] {
|
||||
return Some(Ipv4Addr::new(octets[12], octets[13], octets[14], octets[15]));
|
||||
}
|
||||
if octets[0..2] == [0x20, 0x02] {
|
||||
return Some(Ipv4Addr::new(octets[2], octets[3], octets[4], octets[5]));
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn validate_outbound_ip(ip: IpAddr) -> Result<(), &'static str> {
|
||||
// Reject pure-IPv6 special forms first, before any IPv4 normalization. ::1 (loopback) and ::
|
||||
// (unspecified) are technically IPv4-compatible forms too, so normalizing first would map
|
||||
// them to a harmless-looking 0.0.0.1 / 0.0.0.0 and let them through.
|
||||
if let IpAddr::V6(v6) = ip {
|
||||
if v6.is_loopback() {
|
||||
return Err("loopback address");
|
||||
@@ -77,10 +442,6 @@ fn validate_outbound_ip(ip: IpAddr) -> Result<(), &'static str> {
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize IPv4-mapped (::ffff:a.b.c.d) AND IPv4-compatible (::a.b.c.d) IPv6 addresses to
|
||||
// their embedded IPv4 so the IPv4 rules below apply. The std is_* checks on the IPv6 variant
|
||||
// never inspect the embedded IPv4, so without this an attacker bypasses the guard with e.g.
|
||||
// ::ffff:127.0.0.1, ::127.0.0.1 (loopback) or ::169.254.169.254 (cloud metadata service).
|
||||
let ip = match ip {
|
||||
IpAddr::V6(v6) => match embedded_ipv4(v6) {
|
||||
Some(v4) => IpAddr::V4(v4),
|
||||
@@ -92,44 +453,32 @@ fn validate_outbound_ip(ip: IpAddr) -> Result<(), &'static str> {
|
||||
if ip.is_unspecified() {
|
||||
return Err("unspecified address");
|
||||
}
|
||||
|
||||
if ip == IpAddr::V4(Ipv4Addr::new(169, 254, 169, 254)) {
|
||||
return Err("metadata endpoint");
|
||||
}
|
||||
|
||||
match ip {
|
||||
IpAddr::V4(ipv4) => {
|
||||
if ipv4.is_loopback() {
|
||||
return Err("loopback address");
|
||||
}
|
||||
if ipv4.is_link_local() {
|
||||
return Err("link-local address");
|
||||
}
|
||||
if ipv4.is_private() {
|
||||
return Err("private address");
|
||||
}
|
||||
if let IpAddr::V4(ipv4) = ip {
|
||||
if ipv4.is_loopback() {
|
||||
return Err("loopback address");
|
||||
}
|
||||
if ipv4.is_link_local() {
|
||||
return Err("link-local address");
|
||||
}
|
||||
if ipv4.is_private() {
|
||||
return Err("private address");
|
||||
}
|
||||
// Genuine IPv6 (no embedded IPv4) was already classified above.
|
||||
IpAddr::V6(_) => {}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extract the embedded IPv4 from an IPv4-mapped (`::ffff:a.b.c.d`) or IPv4-compatible
|
||||
/// (`::a.b.c.d`) IPv6 address. The pure-IPv6 specials `::` and `::1` are rejected by the caller
|
||||
/// before this runs, so returning `None` here means a genuine IPv6 host.
|
||||
fn embedded_ipv4(v6: Ipv6Addr) -> Option<Ipv4Addr> {
|
||||
if let Some(v4) = v6.to_ipv4_mapped() {
|
||||
return Some(v4);
|
||||
}
|
||||
// IPv4-compatible: the top 96 bits are zero and the low 32 bits carry the IPv4.
|
||||
let segs = v6.segments();
|
||||
if segs[0..6] == [0, 0, 0, 0, 0, 0] {
|
||||
let hi = segs[6].to_be_bytes();
|
||||
let lo = segs[7].to_be_bytes();
|
||||
let segments = v6.segments();
|
||||
if segments[0..6] == [0, 0, 0, 0, 0, 0] {
|
||||
let hi = segments[6].to_be_bytes();
|
||||
let lo = segments[7].to_be_bytes();
|
||||
let v4 = Ipv4Addr::new(hi[0], hi[1], lo[0], lo[1]);
|
||||
// `::` and `::1` are already handled by the caller; anything else is a real embedded v4.
|
||||
if !v4.is_unspecified() && v4 != Ipv4Addr::new(0, 0, 0, 1) {
|
||||
return Some(v4);
|
||||
}
|
||||
@@ -139,7 +488,9 @@ fn embedded_ipv4(v6: Ipv6Addr) -> Option<Ipv4Addr> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{OutboundUrlError, validate_outbound_url};
|
||||
use super::{OutboundPolicy, OutboundUrlError, validate_outbound_url};
|
||||
use std::collections::HashMap;
|
||||
use std::net::SocketAddr;
|
||||
use url::Url;
|
||||
|
||||
#[test]
|
||||
@@ -174,6 +525,17 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_outbound_url_rejects_alternate_ipv4_loopback_syntax() {
|
||||
for endpoint in ["http://2130706433/hook", "http://0177.0.0.1/hook", "http://0x7f000001/hook"] {
|
||||
let url = Url::parse(endpoint).expect("alternate IPv4 URL should parse");
|
||||
assert!(
|
||||
validate_outbound_url(&url).is_err(),
|
||||
"alternate loopback syntax must be rejected: {endpoint}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_outbound_url_rejects_private_ip() {
|
||||
let url = Url::parse("https://10.0.0.5/webhook").expect("private URL should parse");
|
||||
@@ -187,6 +549,35 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outbound_policy_rejects_special_use_addresses() {
|
||||
let policy = OutboundPolicy::default();
|
||||
for endpoint in [
|
||||
"http://0.0.0.1/hook",
|
||||
"http://192.0.2.1/hook",
|
||||
"http://198.18.0.1/hook",
|
||||
"http://198.51.100.1/hook",
|
||||
"http://203.0.113.1/hook",
|
||||
"http://224.0.0.1/hook",
|
||||
"http://[100::1]/hook",
|
||||
"http://[2001:100::1]/hook",
|
||||
"http://[2001:db8::1]/hook",
|
||||
"http://[3fff::1]/hook",
|
||||
"http://[5f00::1]/hook",
|
||||
"http://[ff02::1]/hook",
|
||||
] {
|
||||
let url = Url::parse(endpoint).expect("special-use URL should parse");
|
||||
assert!(policy.validate_url(&url).is_err(), "special-use address must be rejected: {endpoint}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_outbound_url_preserves_legacy_shared_address_behavior() {
|
||||
let url = Url::parse("http://100.64.0.5/hook").expect("shared URL should parse");
|
||||
assert!(validate_outbound_url(&url).is_ok());
|
||||
assert!(OutboundPolicy::default().validate_url(&url).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_outbound_url_rejects_metadata_endpoint() {
|
||||
let url = Url::parse("http://169.254.169.254/latest/meta-data").expect("metadata URL should parse");
|
||||
@@ -286,6 +677,26 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outbound_policy_rejects_embedded_private_destinations() {
|
||||
let policy = OutboundPolicy::default();
|
||||
for endpoint in [
|
||||
"http://[64:ff9b::7f00:1]/hook",
|
||||
"http://[64:ff9b::a00:1]/hook",
|
||||
"http://[64:ff9b::a9fe:a9fe]/latest/meta-data",
|
||||
"http://[2002:7f00:1::]/hook",
|
||||
"http://[2002:a00:1::]/hook",
|
||||
"http://[::ffff:0:7f00:1]/hook",
|
||||
"http://[::ffff:0:a9fe:a9fe]/latest/meta-data",
|
||||
] {
|
||||
let url = Url::parse(endpoint).expect("embedded IPv4 URL should parse");
|
||||
assert!(
|
||||
policy.validate_url(&url).is_err(),
|
||||
"embedded private destination must be rejected: {endpoint}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_outbound_url_rejects_ipv6_loopback_and_unspecified() {
|
||||
// ::1 / :: must stay rejected even though they look like IPv4-compatible forms.
|
||||
@@ -306,4 +717,263 @@ mod tests {
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outbound_policy_allows_only_the_exact_operator_origin() {
|
||||
let policy = OutboundPolicy::from_allowed_origins("http://127.0.0.1:9443").expect("operator origin should parse");
|
||||
|
||||
assert!(
|
||||
policy
|
||||
.validate_url(&Url::parse("http://127.0.0.1:9443/hook").expect("allowed URL"))
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
policy
|
||||
.validate_url(&Url::parse("http://127.0.0.1:9444/hook").expect("wrong-port URL"))
|
||||
.is_err(),
|
||||
"an allowlisted origin must not authorize another port"
|
||||
);
|
||||
assert!(
|
||||
policy
|
||||
.validate_url(&Url::parse("http://[::1]:9443/hook").expect("different-host URL"))
|
||||
.is_err(),
|
||||
"an allowlisted origin must not authorize a different host"
|
||||
);
|
||||
|
||||
let shared_policy =
|
||||
OutboundPolicy::from_allowed_origins("http://100.64.0.5:9443").expect("shared operator origin should parse");
|
||||
assert!(
|
||||
shared_policy
|
||||
.validate_url(&Url::parse("http://100.64.0.5:9443/hook").expect("shared URL"))
|
||||
.is_ok(),
|
||||
"an exact operator origin may opt into an internal shared address"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outbound_policy_never_allows_metadata_or_unspecified_addresses() {
|
||||
let policy = OutboundPolicy::from_allowed_origins(
|
||||
"http://169.254.1.10,http://169.254.169.254,http://169.254.170.2,http://169.254.170.23,http://169.254.0.23,http://100.100.100.200,http://168.63.129.16,http://[fd00:ec2::254],http://[fd00:ec2::23],http://[fd20:ce::254],http://[64:ff9b:1::1],http://0.0.0.0",
|
||||
)
|
||||
.expect("syntactically valid origins should parse");
|
||||
|
||||
for endpoint in [
|
||||
"http://169.254.1.10/hook",
|
||||
"http://169.254.169.254/latest",
|
||||
"http://169.254.170.2/credentials",
|
||||
"http://169.254.170.23/v1/credentials",
|
||||
"http://169.254.0.23/latest/meta-data",
|
||||
"http://100.100.100.200/latest/meta-data",
|
||||
"http://168.63.129.16/machine?comp=goalstate",
|
||||
"http://[fd00:ec2::254]/latest",
|
||||
"http://[fd00:ec2::23]/v1/credentials",
|
||||
"http://[fd20:ce::254]/computeMetadata/v1",
|
||||
"http://[64:ff9b:1::1]/hook",
|
||||
"http://0.0.0.0/hook",
|
||||
] {
|
||||
assert!(
|
||||
policy
|
||||
.validate_url(&Url::parse(endpoint).expect("test endpoint should parse"))
|
||||
.is_err(),
|
||||
"endpoint must remain forbidden: {endpoint}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outbound_policy_rejects_non_http_and_userinfo_origins() {
|
||||
for origin in [
|
||||
"ftp://webhook.internal",
|
||||
"https://user@webhook.internal",
|
||||
"https://webhook.internal/path",
|
||||
] {
|
||||
assert!(
|
||||
OutboundPolicy::from_allowed_origins(origin).is_err(),
|
||||
"invalid operator origin must fail closed: {origin}"
|
||||
);
|
||||
}
|
||||
assert!(OutboundPolicy::from_allowed_origins("https://one.example,,https://two.example").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outbound_policy_rejects_non_http_and_userinfo_targets() {
|
||||
let policy = OutboundPolicy::default();
|
||||
for endpoint in ["ftp://webhook.test/hook", "https://user:secret@webhook.test/hook"] {
|
||||
assert!(
|
||||
policy
|
||||
.validate_url(&Url::parse(endpoint).expect("target URL should parse"))
|
||||
.is_err(),
|
||||
"target must be rejected: {endpoint}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn outbound_dns_resolver_filters_rebinding_and_mixed_answers() {
|
||||
let policy = OutboundPolicy::default();
|
||||
let endpoint = Url::parse("https://webhook.test/hook").expect("endpoint should parse");
|
||||
let resolver = policy
|
||||
.resolver_for(&endpoint)
|
||||
.expect("public hostname should be accepted")
|
||||
.with_overrides(HashMap::from([
|
||||
(
|
||||
"webhook.test".to_string(),
|
||||
vec![
|
||||
"10.0.0.5".parse().expect("private IP"),
|
||||
"100.64.0.5".parse().expect("shared IP"),
|
||||
"100.100.100.200".parse().expect("metadata IP"),
|
||||
"8.8.8.8".parse().expect("public IP"),
|
||||
"::ffff:127.0.0.1".parse().expect("mapped loopback IP"),
|
||||
],
|
||||
),
|
||||
("rebound.test".to_string(), vec!["127.0.0.1".parse().expect("loopback IP")]),
|
||||
]));
|
||||
|
||||
let addrs = reqwest::dns::Resolve::resolve(&resolver, "webhook.test".parse().expect("resolver hostname"))
|
||||
.await
|
||||
.expect("one public address should remain")
|
||||
.map(|addr| addr.ip())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(addrs, vec!["8.8.8.8".parse::<std::net::IpAddr>().expect("public IP")]);
|
||||
assert!(
|
||||
reqwest::dns::Resolve::resolve(&resolver, "rebound.test".parse().expect("resolver hostname"))
|
||||
.await
|
||||
.is_err(),
|
||||
"a rebound answer containing only restricted addresses must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_operator_origin_allows_private_dns_answers_for_that_host_only() {
|
||||
let policy = OutboundPolicy::from_allowed_origins("https://webhook.test:9443").expect("operator origin should parse");
|
||||
let resolver = policy
|
||||
.resolver_for(&Url::parse("https://webhook.test:9443/hook").expect("endpoint should parse"))
|
||||
.expect("allowlisted endpoint should be accepted")
|
||||
.with_overrides(HashMap::from([
|
||||
("webhook.test".to_string(), vec!["10.0.0.5".parse().expect("private IP")]),
|
||||
("other.test".to_string(), vec!["10.0.0.6".parse().expect("private IP")]),
|
||||
]));
|
||||
|
||||
let allowed = reqwest::dns::Resolve::resolve(&resolver, "webhook.test".parse().expect("resolver hostname"))
|
||||
.await
|
||||
.expect("allowlisted host should resolve")
|
||||
.count();
|
||||
assert_eq!(allowed, 1);
|
||||
assert!(
|
||||
reqwest::dns::Resolve::resolve(&resolver, "other.test".parse().expect("resolver hostname"))
|
||||
.await
|
||||
.is_err(),
|
||||
"the allowlist must not authorize another private hostname"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_operator_origin_never_allows_metadata_dns_answers() {
|
||||
let policy = OutboundPolicy::from_allowed_origins("https://webhook.test:9443").expect("operator origin should parse");
|
||||
let resolver = policy
|
||||
.resolver_for(&Url::parse("https://webhook.test:9443/hook").expect("endpoint should parse"))
|
||||
.expect("allowlisted endpoint should be accepted")
|
||||
.with_overrides(HashMap::from([(
|
||||
"webhook.test".to_string(),
|
||||
vec![
|
||||
"169.254.170.23".parse().expect("EKS credential IP"),
|
||||
"169.254.0.23".parse().expect("Tencent metadata IP"),
|
||||
"fd00:ec2::23".parse().expect("EKS credential IPv6"),
|
||||
"fd20:ce::254".parse().expect("GCP metadata IPv6"),
|
||||
"0.0.0.0".parse().expect("unspecified IP"),
|
||||
],
|
||||
)]));
|
||||
|
||||
assert!(
|
||||
reqwest::dns::Resolve::resolve(&resolver, "webhook.test".parse().expect("resolver hostname"))
|
||||
.await
|
||||
.is_err(),
|
||||
"metadata and unspecified answers must remain forbidden for an allowlisted origin"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn outbound_dns_resolver_preserves_ipv6_scope_id() {
|
||||
use std::net::SocketAddrV6;
|
||||
|
||||
let policy = OutboundPolicy::default();
|
||||
let scoped = SocketAddr::V6(SocketAddrV6::new("2001:4860:4860::8888".parse().expect("public IPv6"), 0, 0, 7));
|
||||
let resolver = policy
|
||||
.resolver_for(&Url::parse("https://webhook.test:9443/hook").expect("endpoint should parse"))
|
||||
.expect("public endpoint should be accepted")
|
||||
.with_override_sequence("webhook.test", vec![vec![scoped]]);
|
||||
|
||||
let resolved = reqwest::dns::Resolve::resolve(&resolver, "webhook.test".parse().expect("resolver hostname"))
|
||||
.await
|
||||
.expect("public scoped address should resolve")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(resolved, vec![scoped]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reqwest_rechecks_dns_policy_for_each_new_connection() {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind first connection listener");
|
||||
let address = listener.local_addr().expect("first connection address");
|
||||
let endpoint = Url::parse(&format!("http://rebind.test:{}/hook", address.port())).expect("endpoint should parse");
|
||||
let policy = OutboundPolicy::from_allowed_origins(&endpoint.origin().ascii_serialization())
|
||||
.expect("loopback test origin should be explicitly allowed");
|
||||
let resolver = policy
|
||||
.resolver_for(&endpoint)
|
||||
.expect("allowlisted endpoint should be accepted")
|
||||
.with_override_sequence(
|
||||
"rebind.test",
|
||||
vec![
|
||||
vec![SocketAddr::new(address.ip(), 0)],
|
||||
vec![SocketAddr::new("169.254.169.254".parse().expect("metadata IP"), 0)],
|
||||
],
|
||||
);
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("accept first connection");
|
||||
let mut request = [0_u8; 1024];
|
||||
let _ = stream.read(&mut request).await.expect("read first request");
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("write first response");
|
||||
});
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.dns_resolver(resolver)
|
||||
.timeout(std::time::Duration::from_secs(2))
|
||||
.build()
|
||||
.expect("build test client");
|
||||
|
||||
assert_eq!(
|
||||
client
|
||||
.get(endpoint.clone())
|
||||
.send()
|
||||
.await
|
||||
.expect("first request should connect")
|
||||
.status(),
|
||||
reqwest::StatusCode::NO_CONTENT
|
||||
);
|
||||
server.await.expect("first connection server should finish");
|
||||
let error = client
|
||||
.get(endpoint)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("the second connection must reject a metadata DNS answer");
|
||||
let mut error_chain = vec![error.to_string()];
|
||||
let mut source = std::error::Error::source(&error);
|
||||
while let Some(error) = source {
|
||||
error_chain.push(error.to_string());
|
||||
source = error.source();
|
||||
}
|
||||
assert!(
|
||||
error_chain
|
||||
.iter()
|
||||
.any(|message| message.contains("returned no allowed addresses")),
|
||||
"request must fail in the DNS policy layer: {error_chain:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,19 +19,24 @@
|
||||
|---|---|---|
|
||||
| admin_auth_test | 3 | ✅ |
|
||||
| admin_iam_crud_test | 2 | ✅ |
|
||||
| admin_pools_test | 1 | ✅ |
|
||||
| admin_timeout_regression_test | 1 | |
|
||||
| anonymous_access_test | 3 | ✅ |
|
||||
| api_rate_limit_test | 3 | |
|
||||
| archive_download_integrity_test | 13 | |
|
||||
| bucket_logging_test | 3 | |
|
||||
| bucket_policy_check_test | 1 | ✅ |
|
||||
| checksum_upload_test | 7 | |
|
||||
| cluster_concurrency_test | 2 | |
|
||||
| common | 3 | |
|
||||
| cluster_multidrive_pool_test | 2 | |
|
||||
| common | 10 | |
|
||||
| compression_test | 1 | |
|
||||
| connection_cap_test | 2 | |
|
||||
| console_smoke_test | 1 | ✅ |
|
||||
| content_encoding_test | 3 | ✅ |
|
||||
| copy_object_checksum_test | 3 | |
|
||||
| copy_object_metadata_test | 1 | ✅ |
|
||||
| copy_object_version_restore_test | 1 | |
|
||||
| copy_object_version_restore_test | 2 | |
|
||||
| copy_source_invalid_date_test | 1 | ✅ |
|
||||
| create_bucket_region_test | 2 | ✅ |
|
||||
| degraded_read_eof_regression_test | 3 | |
|
||||
@@ -40,6 +45,7 @@
|
||||
| delete_objects_versioning_test | 2 | ✅ |
|
||||
| existing_object_tag_policy_test | 4 | |
|
||||
| fake_s3_target | 4 | ✅ |
|
||||
| fault_proxy | 7 | |
|
||||
| get_codec_streaming_compat_test | 1 | |
|
||||
| head_object_consistency_test | 1 | ✅ |
|
||||
| head_object_range_test | 1 | ✅ |
|
||||
@@ -56,13 +62,13 @@
|
||||
| multipart_auth_test | 109 | |
|
||||
| namespace_lock_quorum_test | 2 | |
|
||||
| negative_sigv4_test | 6 | ✅ |
|
||||
| notification_webhook_test | 1 | ✅ |
|
||||
| notification_webhook_test | 2 | ✅ |
|
||||
| object_lambda_test | 16 | |
|
||||
| object_lock | 33 | |
|
||||
| overwrite_cleanup_regression_test | 1 | |
|
||||
| presigned_negative_test | 7 | ✅ |
|
||||
| protocols | 16 | |
|
||||
| quota_test | 13 | |
|
||||
| quota_test | 14 | |
|
||||
| reliability_disk_fault_test | 3 | |
|
||||
| reliant | 10 | 4 ✅ |
|
||||
| replication_extension_test | 47 | 20 ✅ +27 🌙 |
|
||||
@@ -75,4 +81,6 @@
|
||||
| tls_hot_reload_test | 1 | ✅ |
|
||||
| version_id_regression_test | 10 | ✅ |
|
||||
|
||||
**Total listed: 439 tests across 57 modules · PR smoke subset: 112 tests / 27 modules** (25 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-16.
|
||||
`notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above.
|
||||
|
||||
**Total listed: 467 tests across 63 modules · PR smoke subset: 114 tests / 28 modules** (26 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-21.
|
||||
|
||||
@@ -21,6 +21,8 @@ forced to update its pinned test (red -> green).
|
||||
| [GHSA-3p3x-734c-h5vx](https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx) | Constant-time secret comparison on WebDAV/FTPS password login | rustfs/rustfs#4403 | `assert_ftps_ghsa_3p3x_wrong_credentials_rejected` (`crates/e2e_test/src/protocols/ftps_core.rs`); `GHSA-3p3x` auth-failure block in `test_webdav_core_operations` (`crates/e2e_test/src/protocols/webdav_core.rs`) | e2e (protocols suite) |
|
||||
| [GHSA-r5qv-rc46-hv8q](https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q) | Internode RPC authentication must fail closed | rustfs/rustfs#4402 | `ghsa_r5qv_resolve_shared_secret_rejects_default_fallback`, `ghsa_r5qv_verify_rpc_signature_fails_closed_on_missing_or_invalid_auth` (`crates/ecstore/src/cluster/rpc/http_auth.rs`) | unit |
|
||||
| [GHSA-m77q-r63m-pj89](https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89) | STS JWTs signed with shared root secret (intentionally unfixed) | n/a | `test_ghsa_m77q_sts_session_token_signed_with_root_secret` (flow-level pin: signing key == root secret, root-only decode, authorizes) and `test_created_sts_credentials_authorize_with_session_token_claims` (`crates/iam/src/sys.rs`); `token_signing_key` doc (`crates/iam/src/root_credentials.rs`) — pin current by-design behavior; fixing m77q must update red -> green | unit |
|
||||
| [GHSA-5354-r3w2-34m8](https://github.com/rustfs/rustfs/security/advisories/GHSA-5354-r3w2-34m8) | Service-account parent must stay within caller scope — a non-owner holding `CreateServiceAccountAdminAction` could parent a service account to the root credential and authenticate as owner | rustfs/rustfs#5141 | `ghsa_5354_non_owner_service_account_parent_confined_to_scope` and the `add_service_account_parent_within_scope` invariant it pins (`rustfs/src/admin/handlers/service_account.rs`) | unit |
|
||||
| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned object reads must be authorized against `s3:GetObjectVersion`, not `s3:GetObject` (`get_object`, CopyObject source, UploadPartCopy source) | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` and the `versioned_read_action` helper it pins (`rustfs/src/storage/access.rs`) | unit |
|
||||
|
||||
## Where these run (CI-execution map)
|
||||
|
||||
@@ -28,11 +30,13 @@ Every security regression must land where CI actually runs it — a named test i
|
||||
an unexecuted suite is theater. The suites split across three execution paths by
|
||||
topology:
|
||||
|
||||
- **Unit tests** — `ghsa_r5qv_*` (`crates/ecstore`) and the GHSA-m77q STS
|
||||
pinning (`crates/iam`) run automatically in the default CI pass
|
||||
- **Unit tests** — `ghsa_r5qv_*` (`crates/ecstore`), the GHSA-m77q STS
|
||||
pinning (`crates/iam`), and `ghsa_5354_*` / `ghsa_3ppv_*` (`rustfs` lib) run
|
||||
automatically in the default CI pass
|
||||
(`cargo nextest run --profile ci --all --exclude e2e_test`) — no special
|
||||
wiring. This is the CI-executed regression for the RPC fail-closed (r5qv) and
|
||||
STS-signing (m77q) advisories.
|
||||
wiring. This is the CI-executed regression for the RPC fail-closed (r5qv),
|
||||
STS-signing (m77q), service-account parent-scope (5354), and versioned-read
|
||||
authorization (3ppv) advisories.
|
||||
- **S3-API negative-auth e2e (e2e-smoke, PR-gated)** — the attacker-facing S3
|
||||
auth-rejection suites run on every PR via the `e2e-smoke` nextest profile
|
||||
(`.config/nextest.toml`), which each spawns its own server on a random port
|
||||
|
||||
@@ -60,7 +60,7 @@
|
||||
{
|
||||
default = rustPlatform.buildRustPackage {
|
||||
pname = "rustfs";
|
||||
version = "1.0.0-beta.10";
|
||||
version = "1.0.0-beta.11";
|
||||
|
||||
src = ./.;
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ apiVersion: v2
|
||||
name: rustfs
|
||||
description: RustFS helm chart to deploy RustFS on kubernetes cluster.
|
||||
type: application
|
||||
version: "0.10.0"
|
||||
appVersion: "1.0.0-beta.10"
|
||||
version: "0.11.0"
|
||||
appVersion: "1.0.0-beta.11"
|
||||
home: https://rustfs.com
|
||||
icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg
|
||||
maintainers:
|
||||
|
||||
+5
-2
@@ -1,9 +1,9 @@
|
||||
%global _enable_debug_packages 0
|
||||
%global _empty_manifest_terminate_build 0
|
||||
%global prerelease beta.10
|
||||
%global prerelease beta.11
|
||||
Name: rustfs
|
||||
Version: 1.0.0
|
||||
Release: beta.10
|
||||
Release: beta.11
|
||||
Summary: High-performance distributed object storage for MinIO alternative
|
||||
|
||||
License: Apache-2.0
|
||||
@@ -58,6 +58,9 @@ install %_builddir/%{name}-%{version}-%{prerelease}/target/%_arch/%_arch-unknown
|
||||
%_bindir/rustfs
|
||||
|
||||
%changelog
|
||||
* Thu Jul 23 2026 overtrue <anzhengchao@gmail.com>
|
||||
- Update RPM package to RustFS 1.0.0-beta.11
|
||||
|
||||
* Fri Jul 17 2026 overtrue <anzhengchao@gmail.com>
|
||||
- Update RPM package to RustFS 1.0.0-beta.10
|
||||
|
||||
|
||||
@@ -16,8 +16,8 @@ use crate::admin::{
|
||||
auth::validate_admin_request,
|
||||
handlers::audit_runtime_config::{load_server_config_from_store, update_audit_config_and_reload},
|
||||
handlers::target_descriptor::{
|
||||
AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
|
||||
build_json_response, collect_runtime_statuses, extract_supported_target_params,
|
||||
AdminTargetSpec, EndpointKey, RuntimeHealthStatus, TargetEndpointSource, admin_target_spec_from_builtin,
|
||||
build_enabled_target_kvs, build_json_response, collect_runtime_statuses, extract_supported_target_params,
|
||||
merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
|
||||
target_mutation_block_reason as shared_target_mutation_block_reason,
|
||||
},
|
||||
@@ -192,6 +192,8 @@ struct AuditEndpoint {
|
||||
account_id: String,
|
||||
service: String,
|
||||
status: String,
|
||||
health_state: String,
|
||||
health_reason: String,
|
||||
source: TargetEndpointSource,
|
||||
}
|
||||
|
||||
@@ -221,7 +223,7 @@ async fn authorize_audit_admin_request(req: &S3Request<Body>, action: AdminActio
|
||||
validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
|
||||
}
|
||||
|
||||
fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
|
||||
fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> S3Result<Option<String>> {
|
||||
shared_target_mutation_block_reason(
|
||||
audit_target_specs(),
|
||||
AUDIT_ROUTE_PREFIX,
|
||||
@@ -248,16 +250,23 @@ async fn audit_target_operation_block_reason(action: &str) -> Option<String> {
|
||||
target_module_disabled_reason("audit", rustfs_config::ENV_AUDIT_ENABLE, is_audit_module_enabled(), action)
|
||||
}
|
||||
|
||||
fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<AuditEndpoint> {
|
||||
shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)
|
||||
.into_iter()
|
||||
.map(|endpoint| AuditEndpoint {
|
||||
account_id: endpoint.account_id,
|
||||
service: endpoint.service,
|
||||
status: endpoint.status,
|
||||
source: endpoint.source,
|
||||
})
|
||||
.collect()
|
||||
fn merge_audit_endpoints(
|
||||
config: &Config,
|
||||
runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
|
||||
) -> S3Result<Vec<AuditEndpoint>> {
|
||||
Ok(
|
||||
shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)?
|
||||
.into_iter()
|
||||
.map(|endpoint| AuditEndpoint {
|
||||
account_id: endpoint.account_id,
|
||||
service: endpoint.service,
|
||||
status: endpoint.status,
|
||||
health_state: endpoint.health_state,
|
||||
health_reason: endpoint.health_reason,
|
||||
source: endpoint.source,
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
|
||||
@@ -279,7 +288,7 @@ impl Operation for AuditTargetConfig {
|
||||
return Err(s3_error!(InvalidRequest, "{reason}"));
|
||||
}
|
||||
let config_snapshot = load_server_config_from_store().await?;
|
||||
if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
|
||||
if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
|
||||
log_audit_target_operation_blocked!("set_audit_target_config", Some(target_type), Some(target_name), &reason);
|
||||
return Err(s3_error!(InvalidRequest, "{reason}"));
|
||||
}
|
||||
@@ -305,12 +314,14 @@ impl Operation for AuditTargetConfig {
|
||||
)
|
||||
.await?;
|
||||
|
||||
update_audit_config_and_reload(audit_target_specs(), |config| {
|
||||
let mutation_target_type = target_type.to_lowercase();
|
||||
let mutation_target_name = target_name.to_lowercase();
|
||||
update_audit_config_and_reload(audit_target_specs(), move |config| {
|
||||
config
|
||||
.0
|
||||
.entry(target_type.to_lowercase())
|
||||
.entry(mutation_target_type.clone())
|
||||
.or_default()
|
||||
.insert(target_name.to_lowercase(), kvs.clone());
|
||||
.insert(mutation_target_name.clone(), kvs.clone());
|
||||
true
|
||||
})
|
||||
.await
|
||||
@@ -344,7 +355,7 @@ impl Operation for ListAuditTargets {
|
||||
}
|
||||
|
||||
let config = load_server_config_from_store().await?;
|
||||
let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses);
|
||||
let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses)?;
|
||||
let data = serde_json::to_vec(&AuditEndpointsResponse { audit_endpoints }).map_err(|e| {
|
||||
log_audit_target_request_failed!("list_audit_targets", "serialize_audit_targets_failed", None, None, e);
|
||||
s3_error!(InternalError, "failed to serialize audit targets: {}", e)
|
||||
@@ -369,19 +380,21 @@ impl Operation for RemoveAuditTarget {
|
||||
return Err(s3_error!(InvalidRequest, "{reason}"));
|
||||
}
|
||||
let config_snapshot = load_server_config_from_store().await?;
|
||||
if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
|
||||
if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
|
||||
log_audit_target_operation_blocked!("remove_audit_target_config", Some(target_type), Some(target_name), &reason);
|
||||
return Err(s3_error!(InvalidRequest, "{reason}"));
|
||||
}
|
||||
|
||||
update_audit_config_and_reload(audit_target_specs(), |config| {
|
||||
let mutation_target_type = target_type.to_lowercase();
|
||||
let mutation_target_name = target_name.to_lowercase();
|
||||
update_audit_config_and_reload(audit_target_specs(), move |config| {
|
||||
let mut changed = false;
|
||||
if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) {
|
||||
if targets.remove(&target_name.to_lowercase()).is_some() {
|
||||
if let Some(targets) = config.0.get_mut(&mutation_target_type) {
|
||||
if targets.remove(&mutation_target_name).is_some() {
|
||||
changed = true;
|
||||
}
|
||||
if targets.is_empty() {
|
||||
config.0.remove(&target_type.to_lowercase());
|
||||
config.0.remove(&mutation_target_type);
|
||||
}
|
||||
}
|
||||
changed
|
||||
@@ -422,6 +435,14 @@ mod tests {
|
||||
}])
|
||||
}
|
||||
|
||||
fn online_health() -> RuntimeHealthStatus {
|
||||
RuntimeHealthStatus {
|
||||
status: "online".to_string(),
|
||||
state: "online".to_string(),
|
||||
reason: "reachable".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_audit_webhook_target_env_cleared<F>(target_name: &str, f: F)
|
||||
where
|
||||
F: FnOnce(),
|
||||
@@ -466,10 +487,10 @@ mod tests {
|
||||
],
|
||||
|| {
|
||||
let runtime = HashMap::from([
|
||||
(("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
|
||||
(("env-only".to_string(), "webhook".to_string()), "online".to_string()),
|
||||
(("mixed-target".to_string(), "webhook".to_string()), online_health()),
|
||||
(("env-only".to_string(), "webhook".to_string()), online_health()),
|
||||
]);
|
||||
let merged = merge_audit_endpoints(&config, runtime);
|
||||
let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
|
||||
|
||||
let mixed = merged
|
||||
.iter()
|
||||
@@ -509,10 +530,10 @@ mod tests {
|
||||
],
|
||||
|| {
|
||||
let runtime = HashMap::from([
|
||||
(("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
|
||||
(("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
|
||||
(("mixed-kafka".to_string(), "kafka".to_string()), online_health()),
|
||||
(("env-kafka".to_string(), "kafka".to_string()), online_health()),
|
||||
]);
|
||||
let merged = merge_audit_endpoints(&config, runtime);
|
||||
let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
|
||||
|
||||
let mixed = merged
|
||||
.iter()
|
||||
@@ -546,10 +567,10 @@ mod tests {
|
||||
],
|
||||
|| {
|
||||
let runtime = HashMap::from([
|
||||
(("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
|
||||
(("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
|
||||
(("mixed-amqp".to_string(), "amqp".to_string()), online_health()),
|
||||
(("env-amqp".to_string(), "amqp".to_string()), online_health()),
|
||||
]);
|
||||
let merged = merge_audit_endpoints(&config, runtime);
|
||||
let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
|
||||
|
||||
let mixed = merged
|
||||
.iter()
|
||||
@@ -576,7 +597,8 @@ mod tests {
|
||||
],
|
||||
|| {
|
||||
let config = Config(HashMap::new());
|
||||
let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
|
||||
let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
|
||||
.expect("audit target mutation block reason");
|
||||
assert!(reason.is_some());
|
||||
assert!(reason.unwrap().contains("managed by environment variables"));
|
||||
},
|
||||
@@ -613,7 +635,8 @@ mod tests {
|
||||
AUDIT_WEBHOOK_SUB_SYS.to_string(),
|
||||
HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
|
||||
)]));
|
||||
let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
|
||||
let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
|
||||
.expect("audit target mutation block reason");
|
||||
assert!(reason.is_some());
|
||||
assert!(reason.unwrap().contains("both persisted config and environment variables"));
|
||||
});
|
||||
@@ -633,7 +656,7 @@ mod tests {
|
||||
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
|
||||
],
|
||||
|| {
|
||||
let merged = merge_audit_endpoints(&config, HashMap::new());
|
||||
let merged = merge_audit_endpoints(&config, HashMap::new()).expect("merge audit endpoints");
|
||||
let mixed = merged
|
||||
.iter()
|
||||
.find(|entry| entry.account_id == "mixed-disabled")
|
||||
@@ -655,7 +678,7 @@ mod tests {
|
||||
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
|
||||
],
|
||||
|| {
|
||||
let merged = merge_audit_endpoints(&config, HashMap::new());
|
||||
let merged = merge_audit_endpoints(&config, HashMap::new()).expect("merge audit endpoints");
|
||||
let env_only = merged
|
||||
.iter()
|
||||
.find(|entry| entry.account_id == "env-only")
|
||||
@@ -767,8 +790,8 @@ mod tests {
|
||||
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")),
|
||||
],
|
||||
|| {
|
||||
let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
|
||||
let merged = merge_audit_endpoints(&config, runtime);
|
||||
let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), online_health())]);
|
||||
let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
|
||||
let mixed = merged
|
||||
.iter()
|
||||
.find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
|
||||
@@ -787,7 +810,11 @@ mod tests {
|
||||
)]));
|
||||
|
||||
with_audit_webhook_target_env_cleared("primarycase", || {
|
||||
assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase").is_none());
|
||||
assert!(
|
||||
audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase")
|
||||
.expect("audit target mutation block reason")
|
||||
.is_none()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -795,7 +822,11 @@ mod tests {
|
||||
fn audit_target_mutation_block_reason_allows_runtime_only_target() {
|
||||
with_audit_webhook_target_env_cleared("primary", || {
|
||||
let config = Config(HashMap::new());
|
||||
assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary").is_none());
|
||||
assert!(
|
||||
audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
|
||||
.expect("audit target mutation block reason")
|
||||
.is_none()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -14,11 +14,15 @@
|
||||
|
||||
use crate::admin::handlers::target_descriptor::AdminTargetSpec;
|
||||
use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context};
|
||||
use crate::admin::storage_api::config::{read_admin_config_without_migrate, save_admin_server_config};
|
||||
use crate::admin::storage_api::config::{
|
||||
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
|
||||
with_admin_server_config_write_lock,
|
||||
};
|
||||
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
|
||||
use rustfs_config::DEFAULT_DELIMITER;
|
||||
use rustfs_config::server_config::Config;
|
||||
use s3s::{S3Result, s3_error};
|
||||
use tracing::warn;
|
||||
|
||||
pub(crate) async fn load_server_config_from_store_for_context(context: Option<&AppContext>) -> S3Result<Config> {
|
||||
let Some(store) = current_object_store_handle_for_context(context) else {
|
||||
@@ -51,30 +55,31 @@ pub(crate) async fn apply_audit_runtime_config(specs: &[AdminTargetSpec], config
|
||||
match system.get_state().await {
|
||||
AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
|
||||
if has_targets {
|
||||
system
|
||||
.reload_config(config)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?;
|
||||
system.reload_config(config).await.map_err(|_| {
|
||||
warn!(reason = "reload_failed", "Failed to reload local audit runtime");
|
||||
s3_error!(InternalError, "failed to reload audit config")
|
||||
})?;
|
||||
} else {
|
||||
system
|
||||
.close()
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?;
|
||||
system.close().await.map_err(|_| {
|
||||
warn!(reason = "stop_failed", "Failed to stop local audit runtime");
|
||||
s3_error!(InternalError, "failed to stop audit system")
|
||||
})?;
|
||||
}
|
||||
}
|
||||
AuditSystemState::Stopped | AuditSystemState::Stopping => {
|
||||
if has_targets {
|
||||
system
|
||||
.start(config)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
|
||||
system.start(config).await.map_err(|_| {
|
||||
warn!(reason = "start_failed", "Failed to start local audit runtime");
|
||||
s3_error!(InternalError, "failed to start audit system")
|
||||
})?;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if has_targets {
|
||||
start_global_audit_system(config)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
|
||||
start_global_audit_system(config).await.map_err(|_| {
|
||||
warn!(reason = "start_failed", "Failed to start global audit runtime");
|
||||
s3_error!(InternalError, "failed to start audit system")
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -86,30 +91,40 @@ async fn update_audit_config_and_reload_for_context<F>(
|
||||
mut modifier: F,
|
||||
) -> S3Result<()>
|
||||
where
|
||||
F: FnMut(&mut Config) -> bool,
|
||||
F: FnMut(&mut Config) -> bool + Send + 'static,
|
||||
{
|
||||
let Some(store) = current_object_store_handle_for_context(context) else {
|
||||
return Err(s3_error!(InternalError, "server storage not initialized"));
|
||||
};
|
||||
|
||||
let mut config = read_admin_config_without_migrate(store.clone())
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
|
||||
let specs = specs.to_vec();
|
||||
let lock_store = store.clone();
|
||||
with_admin_server_config_write_lock(lock_store, move || async move {
|
||||
let mut config = read_admin_config_without_migrate_no_lock(store.clone())
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
|
||||
|
||||
if !modifier(&mut config) {
|
||||
return Ok(());
|
||||
}
|
||||
if !modifier(&mut config) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
save_admin_server_config(store, &config)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
|
||||
save_admin_server_config_no_lock(store, &config)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
|
||||
|
||||
apply_audit_runtime_config(specs, config).await
|
||||
// Keep persistence and runtime publication in one detached, serialized
|
||||
// mutation. Otherwise a cancelled caller or two concurrent updates can
|
||||
// leave the persisted config and active audit generation disagreeing.
|
||||
apply_audit_runtime_config(&specs, config).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn update_audit_config_and_reload<F>(specs: &[AdminTargetSpec], modifier: F) -> S3Result<()>
|
||||
where
|
||||
F: FnMut(&mut Config) -> bool,
|
||||
F: FnMut(&mut Config) -> bool + Send + 'static,
|
||||
{
|
||||
let context = current_app_context();
|
||||
update_audit_config_and_reload_for_context(context.as_deref(), specs, modifier).await
|
||||
@@ -121,26 +136,30 @@ pub(crate) async fn set_audit_target_config(
|
||||
target_name: &str,
|
||||
kvs: rustfs_config::server_config::KVS,
|
||||
) -> S3Result<()> {
|
||||
update_audit_config_and_reload(specs, |config| {
|
||||
let subsystem = subsystem.to_lowercase();
|
||||
let target_name = target_name.to_lowercase();
|
||||
update_audit_config_and_reload(specs, move |config| {
|
||||
config
|
||||
.0
|
||||
.entry(subsystem.to_lowercase())
|
||||
.entry(subsystem.clone())
|
||||
.or_default()
|
||||
.insert(target_name.to_lowercase(), kvs.clone());
|
||||
.insert(target_name.clone(), kvs.clone());
|
||||
true
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_audit_target_config(specs: &[AdminTargetSpec], subsystem: &str, target_name: &str) -> S3Result<()> {
|
||||
update_audit_config_and_reload(specs, |config| {
|
||||
let subsystem = subsystem.to_lowercase();
|
||||
let target_name = target_name.to_lowercase();
|
||||
update_audit_config_and_reload(specs, move |config| {
|
||||
let mut changed = false;
|
||||
if let Some(targets) = config.0.get_mut(&subsystem.to_lowercase()) {
|
||||
if targets.remove(&target_name.to_lowercase()).is_some() {
|
||||
if let Some(targets) = config.0.get_mut(&subsystem) {
|
||||
if targets.remove(&target_name).is_some() {
|
||||
changed = true;
|
||||
}
|
||||
if targets.is_empty() {
|
||||
config.0.remove(&subsystem.to_lowercase());
|
||||
config.0.remove(&subsystem);
|
||||
}
|
||||
}
|
||||
changed
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::handlers::supervise_admin_mutation;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::{
|
||||
current_app_context, current_object_store_handle_for_context, current_server_config_for_context, publish_server_config,
|
||||
@@ -20,12 +21,14 @@ use crate::admin::runtime_sources::{
|
||||
use crate::admin::service::config::{
|
||||
CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN,
|
||||
LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem,
|
||||
prepare_server_config, signal_config_snapshot_reload, signal_dynamic_config_reload,
|
||||
preflight_dynamic_config_reload, prepare_server_config, signal_config_snapshot_reload_checked,
|
||||
signal_dynamic_config_reload_checked,
|
||||
};
|
||||
use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
|
||||
use crate::admin::storage_api::config::{
|
||||
RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate,
|
||||
save_admin_config, save_admin_server_config,
|
||||
read_admin_config_without_migrate_no_lock, save_admin_config, save_admin_server_config_no_lock,
|
||||
with_admin_server_config_write_lock,
|
||||
};
|
||||
use crate::admin::storage_api::contract::list::ListOperations as _;
|
||||
use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
|
||||
@@ -47,7 +50,7 @@ use rustfs_config::notify::{
|
||||
ENV_NOTIFY_MQTT_QOS, ENV_NOTIFY_MQTT_QUEUE_DIR, ENV_NOTIFY_MQTT_QUEUE_LIMIT, ENV_NOTIFY_MQTT_RECONNECT_INTERVAL,
|
||||
ENV_NOTIFY_MQTT_TOPIC, ENV_NOTIFY_MQTT_USERNAME, ENV_NOTIFY_WEBHOOK_AUTH_TOKEN, ENV_NOTIFY_WEBHOOK_CLIENT_CERT,
|
||||
ENV_NOTIFY_WEBHOOK_CLIENT_KEY, ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, ENV_NOTIFY_WEBHOOK_QUEUE_DIR,
|
||||
ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
|
||||
ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT, NOTIFY_MQTT_SUB_SYS, NOTIFY_SUB_SYSTEMS, NOTIFY_WEBHOOK_SUB_SYS,
|
||||
};
|
||||
use rustfs_config::oidc::{
|
||||
ENV_IDENTITY_OPENID_CLAIM_NAME, ENV_IDENTITY_OPENID_CLAIM_PREFIX, ENV_IDENTITY_OPENID_CLIENT_ID,
|
||||
@@ -726,6 +729,14 @@ async fn load_server_config_from_store() -> S3Result<ServerConfig> {
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn load_server_config_from_store_locked() -> S3Result<ServerConfig> {
|
||||
let store = object_store()?;
|
||||
read_admin_config_without_migrate_no_lock(store)
|
||||
.await
|
||||
.map_err(ApiError::from)
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn load_active_server_config() -> S3Result<ServerConfig> {
|
||||
if let Ok(config) = load_server_config_from_store().await {
|
||||
return Ok(config);
|
||||
@@ -736,9 +747,9 @@ async fn load_active_server_config() -> S3Result<ServerConfig> {
|
||||
.ok_or_else(|| s3_error!(InternalError, "server config is not initialized"))
|
||||
}
|
||||
|
||||
async fn save_server_config_to_store(config: &ServerConfig) -> S3Result<()> {
|
||||
async fn save_server_config_to_store_locked(config: &ServerConfig) -> S3Result<()> {
|
||||
let store = object_store()?;
|
||||
save_admin_server_config(store, config)
|
||||
save_admin_server_config_no_lock(store, config)
|
||||
.await
|
||||
.map_err(ApiError::from)
|
||||
.map_err(Into::into)
|
||||
@@ -1563,20 +1574,144 @@ async fn commit_prepared_config(
|
||||
/// Re-apply local mutable worker families after a full-config replacement.
|
||||
/// Peers receive one full-snapshot signal after this returns; signaling each
|
||||
/// family here as well would recreate audit/scanner targets twice per peer.
|
||||
async fn apply_dynamic_subsystems(config: &ServerConfig) {
|
||||
fn publish_notify_config_intent(
|
||||
config: &ServerConfig,
|
||||
sub_system: Option<&str>,
|
||||
) -> Option<rustfs_notify::NotificationLifecycleTransition> {
|
||||
(sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)))
|
||||
.then(|| rustfs_notify::ensure_live_events().publish_config(config.clone()))
|
||||
}
|
||||
|
||||
async fn preflight_notify_config_intent(sub_system: Option<&str>) -> S3Result<()> {
|
||||
if sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)) {
|
||||
preflight_dynamic_config_reload(sub_system.unwrap_or(NOTIFY_WEBHOOK_SUB_SYS)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_notify_config_intent(transition: Option<rustfs_notify::NotificationLifecycleTransition>) -> S3Result<bool> {
|
||||
let Some(transition) = transition else {
|
||||
return Ok(false);
|
||||
};
|
||||
transition.wait().await.map_err(|err| {
|
||||
warn!(error = %err, "Failed to apply local notification config");
|
||||
s3_error!(InternalError, "failed to apply notification config")
|
||||
})?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn apply_non_notify_dynamic_subsystems(config: &ServerConfig) -> Vec<String> {
|
||||
let mut failures = Vec::new();
|
||||
for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
|
||||
if let Err(err) = apply_dynamic_config_for_subsystem(config, sub_system).await {
|
||||
if NOTIFY_SUB_SYSTEMS.contains(&sub_system) {
|
||||
continue;
|
||||
}
|
||||
if apply_dynamic_config_for_subsystem(config, sub_system).await.is_err() {
|
||||
failures.push(format!("local {sub_system}"));
|
||||
warn!(
|
||||
event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
config_subsystem = sub_system,
|
||||
state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
|
||||
error = %err,
|
||||
reason = "apply_failed",
|
||||
"Published server config but failed to reload a local worker subsystem"
|
||||
);
|
||||
}
|
||||
}
|
||||
failures
|
||||
}
|
||||
|
||||
fn finish_config_reconciliation(errors: Vec<String>) -> S3Result<()> {
|
||||
if errors.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(s3_error!(
|
||||
InternalError,
|
||||
"server config persisted but runtime convergence failed: {}",
|
||||
errors.join("; ")
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn reconcile_targeted_config(
|
||||
config: ServerConfig,
|
||||
sub_system: Option<String>,
|
||||
storage_class_applied: bool,
|
||||
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
|
||||
) -> S3Result<bool> {
|
||||
let mut errors = Vec::new();
|
||||
let notify_applied = notify_transition.is_some();
|
||||
if let Err(err) = wait_notify_config_intent(notify_transition).await {
|
||||
warn!(error = %err, "Local notification config failed to converge");
|
||||
errors.push("local notify".to_string());
|
||||
}
|
||||
|
||||
let config_applied = if notify_applied {
|
||||
if let Some(sub_system) = sub_system.as_deref()
|
||||
&& let Err(err) = signal_dynamic_config_reload_checked(sub_system).await
|
||||
{
|
||||
warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed");
|
||||
errors.push(format!("peer {sub_system}"));
|
||||
}
|
||||
true
|
||||
} else if storage_class_applied {
|
||||
if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await {
|
||||
warn!(error = %err, "Peer storage-class reload failed");
|
||||
errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
|
||||
}
|
||||
true
|
||||
} else if let Some(sub_system) = sub_system.as_deref()
|
||||
&& is_dynamic_config_subsystem(sub_system)
|
||||
{
|
||||
let config_applied = match apply_dynamic_config_for_subsystem(&config, sub_system).await {
|
||||
Ok(applied) => applied,
|
||||
Err(_) => {
|
||||
warn!(config_subsystem = sub_system, reason = "apply_failed", "Local config reload failed");
|
||||
errors.push(format!("local {sub_system}"));
|
||||
false
|
||||
}
|
||||
};
|
||||
if let Err(err) = signal_dynamic_config_reload_checked(sub_system).await {
|
||||
warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed");
|
||||
errors.push(format!("peer {sub_system}"));
|
||||
}
|
||||
config_applied
|
||||
} else {
|
||||
if let Err(err) = signal_config_snapshot_reload_checked().await {
|
||||
warn!(error = %err, "Peer config snapshot reload failed");
|
||||
errors.push("peer config snapshot".to_string());
|
||||
}
|
||||
false
|
||||
};
|
||||
|
||||
finish_config_reconciliation(errors)?;
|
||||
Ok(config_applied)
|
||||
}
|
||||
|
||||
async fn reconcile_full_config(
|
||||
config: ServerConfig,
|
||||
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
|
||||
) -> S3Result<()> {
|
||||
let mut errors = Vec::new();
|
||||
if let Err(err) = wait_notify_config_intent(notify_transition).await {
|
||||
warn!(error = %err, "Local notification config failed to converge");
|
||||
errors.push("local notify".to_string());
|
||||
}
|
||||
if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await {
|
||||
warn!(error = %err, "Peer storage-class reload failed");
|
||||
errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
|
||||
}
|
||||
errors.extend(apply_non_notify_dynamic_subsystems(&config).await);
|
||||
if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await {
|
||||
warn!(error = %err, "Peer notification config reload failed");
|
||||
errors.push("peer notify".to_string());
|
||||
}
|
||||
if let Err(err) = signal_config_snapshot_reload_checked().await {
|
||||
warn!(error = %err, "Peer config snapshot reload failed");
|
||||
errors.push("peer config snapshot".to_string());
|
||||
}
|
||||
finish_config_reconciliation(errors)
|
||||
}
|
||||
|
||||
pub struct GetConfigKVHandler {}
|
||||
@@ -1611,37 +1746,39 @@ impl Operation for SetConfigKVHandler {
|
||||
}
|
||||
validate_config_directives(&directives)?;
|
||||
|
||||
let sub_system = config_update_sub_system(&directives)?;
|
||||
let mut config = load_server_config_from_store().await?;
|
||||
apply_set_directives(&mut config, &directives)?;
|
||||
let prepared = prepare_server_config(&config, sub_system).await?;
|
||||
save_server_config_history(&body).await?;
|
||||
let config_applied = if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
|
||||
commit_prepared_config(
|
||||
config.clone(),
|
||||
prepared,
|
||||
save_server_config_to_store(&config),
|
||||
publish_prepared_config_snapshots,
|
||||
)
|
||||
.await?;
|
||||
signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
|
||||
true
|
||||
} else {
|
||||
save_server_config_to_store(&config).await?;
|
||||
publish_server_config(config.clone());
|
||||
if let Some(sub_system) = sub_system
|
||||
&& is_dynamic_config_subsystem(sub_system)
|
||||
{
|
||||
let config_applied = apply_dynamic_config_for_subsystem(&config, sub_system).await?;
|
||||
if config_applied {
|
||||
signal_dynamic_config_reload(sub_system).await;
|
||||
}
|
||||
config_applied
|
||||
} else {
|
||||
signal_config_snapshot_reload().await;
|
||||
false
|
||||
}
|
||||
};
|
||||
let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
|
||||
let transaction_sub_system = sub_system.clone();
|
||||
let config_store = object_store()?;
|
||||
let config_applied = supervise_admin_mutation("config mutation", async move {
|
||||
preflight_notify_config_intent(sub_system.as_deref()).await?;
|
||||
let (config, storage_class_applied, notify_transition) =
|
||||
with_admin_server_config_write_lock(config_store, move || async move {
|
||||
let sub_system = transaction_sub_system.as_deref();
|
||||
let mut config = load_server_config_from_store_locked().await?;
|
||||
apply_set_directives(&mut config, &directives)?;
|
||||
let prepared = prepare_server_config(&config, sub_system).await?;
|
||||
save_server_config_history(&body).await?;
|
||||
if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
|
||||
commit_prepared_config(
|
||||
config.clone(),
|
||||
prepared,
|
||||
save_server_config_to_store_locked(&config),
|
||||
publish_prepared_config_snapshots,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
save_server_config_to_store_locked(&config).await?;
|
||||
publish_server_config(config.clone());
|
||||
}
|
||||
let notify_transition = publish_notify_config_intent(&config, sub_system);
|
||||
Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
|
||||
|
||||
reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
|
||||
})
|
||||
.await?;
|
||||
|
||||
success_response(config_applied)
|
||||
}
|
||||
@@ -1660,37 +1797,39 @@ impl Operation for DelConfigKVHandler {
|
||||
}
|
||||
validate_config_directives(&directives)?;
|
||||
|
||||
let sub_system = config_update_sub_system(&directives)?;
|
||||
let mut config = load_server_config_from_store().await?;
|
||||
apply_delete_directives(&mut config, &directives);
|
||||
let prepared = prepare_server_config(&config, sub_system).await?;
|
||||
save_server_config_history(&body).await?;
|
||||
let config_applied = if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
|
||||
commit_prepared_config(
|
||||
config.clone(),
|
||||
prepared,
|
||||
save_server_config_to_store(&config),
|
||||
publish_prepared_config_snapshots,
|
||||
)
|
||||
.await?;
|
||||
signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
|
||||
true
|
||||
} else {
|
||||
save_server_config_to_store(&config).await?;
|
||||
publish_server_config(config.clone());
|
||||
if let Some(sub_system) = sub_system
|
||||
&& is_dynamic_config_subsystem(sub_system)
|
||||
{
|
||||
let config_applied = apply_dynamic_config_for_subsystem(&config, sub_system).await?;
|
||||
if config_applied {
|
||||
signal_dynamic_config_reload(sub_system).await;
|
||||
}
|
||||
config_applied
|
||||
} else {
|
||||
signal_config_snapshot_reload().await;
|
||||
false
|
||||
}
|
||||
};
|
||||
let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
|
||||
let transaction_sub_system = sub_system.clone();
|
||||
let config_store = object_store()?;
|
||||
let config_applied = supervise_admin_mutation("config mutation", async move {
|
||||
preflight_notify_config_intent(sub_system.as_deref()).await?;
|
||||
let (config, storage_class_applied, notify_transition) =
|
||||
with_admin_server_config_write_lock(config_store, move || async move {
|
||||
let sub_system = transaction_sub_system.as_deref();
|
||||
let mut config = load_server_config_from_store_locked().await?;
|
||||
apply_delete_directives(&mut config, &directives);
|
||||
let prepared = prepare_server_config(&config, sub_system).await?;
|
||||
save_server_config_history(&body).await?;
|
||||
if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
|
||||
commit_prepared_config(
|
||||
config.clone(),
|
||||
prepared,
|
||||
save_server_config_to_store_locked(&config),
|
||||
publish_prepared_config_snapshots,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
save_server_config_to_store_locked(&config).await?;
|
||||
publish_server_config(config.clone());
|
||||
}
|
||||
let notify_transition = publish_notify_config_intent(&config, sub_system);
|
||||
Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
|
||||
|
||||
reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
|
||||
})
|
||||
.await?;
|
||||
|
||||
success_response(config_applied)
|
||||
}
|
||||
@@ -1776,16 +1915,26 @@ impl Operation for RestoreConfigHistoryKVHandler {
|
||||
let mut config = ServerConfig::new();
|
||||
apply_set_directives(&mut config, &directives)?;
|
||||
let prepared = prepare_server_config(&config, None).await?;
|
||||
commit_prepared_config(
|
||||
config.clone(),
|
||||
prepared,
|
||||
save_server_config_to_store(&config),
|
||||
publish_prepared_config_snapshots,
|
||||
)
|
||||
let config_store = object_store()?;
|
||||
supervise_admin_mutation("config mutation", async move {
|
||||
preflight_notify_config_intent(None).await?;
|
||||
let persisted_config = config.clone();
|
||||
let notify_transition = with_admin_server_config_write_lock(config_store, move || async move {
|
||||
commit_prepared_config(
|
||||
persisted_config.clone(),
|
||||
prepared,
|
||||
save_server_config_to_store_locked(&persisted_config),
|
||||
publish_prepared_config_snapshots,
|
||||
)
|
||||
.await?;
|
||||
Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config restore: {}", err))??;
|
||||
|
||||
reconcile_full_config(config, notify_transition).await
|
||||
})
|
||||
.await?;
|
||||
signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
|
||||
apply_dynamic_subsystems(&config).await;
|
||||
signal_config_snapshot_reload().await;
|
||||
|
||||
success_response(false)
|
||||
}
|
||||
@@ -1820,17 +1969,27 @@ impl Operation for SetConfigHandler {
|
||||
let mut config = ServerConfig::new();
|
||||
apply_set_directives(&mut config, &directives)?;
|
||||
let prepared = prepare_server_config(&config, None).await?;
|
||||
save_server_config_history(&body).await?;
|
||||
commit_prepared_config(
|
||||
config.clone(),
|
||||
prepared,
|
||||
save_server_config_to_store(&config),
|
||||
publish_prepared_config_snapshots,
|
||||
)
|
||||
let config_store = object_store()?;
|
||||
supervise_admin_mutation("config mutation", async move {
|
||||
preflight_notify_config_intent(None).await?;
|
||||
save_server_config_history(&body).await?;
|
||||
let persisted_config = config.clone();
|
||||
let notify_transition = with_admin_server_config_write_lock(config_store, move || async move {
|
||||
commit_prepared_config(
|
||||
persisted_config.clone(),
|
||||
prepared,
|
||||
save_server_config_to_store_locked(&persisted_config),
|
||||
publish_prepared_config_snapshots,
|
||||
)
|
||||
.await?;
|
||||
Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock full server config update: {}", err))??;
|
||||
|
||||
reconcile_full_config(config, notify_transition).await
|
||||
})
|
||||
.await?;
|
||||
signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
|
||||
apply_dynamic_subsystems(&config).await;
|
||||
signal_config_snapshot_reload().await;
|
||||
|
||||
success_response(false)
|
||||
}
|
||||
@@ -1890,92 +2049,6 @@ mod tests {
|
||||
assert_eq!(*events.lock().expect("result events lock"), ["persist"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn storage_config_write_handlers_persist_before_publishing() {
|
||||
const SOURCE: &str = include_str!("config_admin.rs");
|
||||
|
||||
for (handler, next_handler, follow_up) in [
|
||||
(
|
||||
"SetConfigKVHandler",
|
||||
"DelConfigKVHandler",
|
||||
"signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
|
||||
),
|
||||
(
|
||||
"DelConfigKVHandler",
|
||||
"HelpConfigKVHandler",
|
||||
"signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
|
||||
),
|
||||
(
|
||||
"RestoreConfigHistoryKVHandler",
|
||||
"GetConfigHandler",
|
||||
"signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
|
||||
),
|
||||
(
|
||||
"SetConfigHandler",
|
||||
"#[cfg(test)]",
|
||||
"signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
|
||||
),
|
||||
] {
|
||||
let start_marker = format!("impl Operation for {handler}");
|
||||
let start = SOURCE
|
||||
.find(&start_marker)
|
||||
.unwrap_or_else(|| panic!("missing {handler} implementation"));
|
||||
let tail = &SOURCE[start..];
|
||||
let end = tail
|
||||
.find(next_handler)
|
||||
.unwrap_or_else(|| panic!("missing {next_handler} after {handler}"));
|
||||
let implementation = &tail[..end];
|
||||
let prepared_commit_path = if matches!(handler, "SetConfigKVHandler" | "DelConfigKVHandler") {
|
||||
let branch_start = implementation
|
||||
.find("if sub_system == Some(STORAGE_CLASS_SUB_SYS)")
|
||||
.unwrap_or_else(|| panic!("missing storage-class branch in {handler}"));
|
||||
let branch = &implementation[branch_start..];
|
||||
let branch_end = branch
|
||||
.find("} else {")
|
||||
.unwrap_or_else(|| panic!("missing non-storage branch in {handler}"));
|
||||
&branch[..branch_end]
|
||||
} else {
|
||||
implementation
|
||||
};
|
||||
|
||||
assert_eq!(prepared_commit_path.matches("commit_prepared_config(").count(), 1, "{handler}");
|
||||
let commit_start = prepared_commit_path.find("commit_prepared_config(").expect("commit call");
|
||||
let commit_end = prepared_commit_path[commit_start..].find(';').expect("commit terminator") + commit_start;
|
||||
let commit_statement = &prepared_commit_path[commit_start..=commit_end];
|
||||
assert!(commit_statement.contains(".await?;"), "{handler} must propagate commit failure");
|
||||
|
||||
let follow_up_start = prepared_commit_path
|
||||
.find(follow_up)
|
||||
.unwrap_or_else(|| panic!("missing follow-up in {handler}"));
|
||||
assert!(follow_up_start > commit_end, "{handler} must run follow-up only after commit");
|
||||
assert!(
|
||||
!prepared_commit_path.contains("publish_server_config("),
|
||||
"{handler} must not publish directly"
|
||||
);
|
||||
assert!(
|
||||
!prepared_commit_path.contains(".publish_storage_class("),
|
||||
"{handler} must not publish directly"
|
||||
);
|
||||
|
||||
if matches!(handler, "RestoreConfigHistoryKVHandler" | "SetConfigHandler") {
|
||||
let worker_start = prepared_commit_path
|
||||
.find("apply_dynamic_subsystems(&config).await")
|
||||
.unwrap_or_else(|| panic!("missing local worker apply in {handler}"));
|
||||
let signal_start = prepared_commit_path
|
||||
.find("signal_config_snapshot_reload().await")
|
||||
.unwrap_or_else(|| panic!("missing snapshot signal in {handler}"));
|
||||
assert!(
|
||||
worker_start > follow_up_start,
|
||||
"{handler} must converge peer parity before local worker apply"
|
||||
);
|
||||
assert!(
|
||||
signal_start > worker_start,
|
||||
"{handler} must signal the full snapshot after local worker apply"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenize_config_line_handles_quotes_and_escapes() {
|
||||
let tokens = tokenize_config_line(r#"identity_openid client_id="console app" client_secret="s3cr\"et" enable=on"#)
|
||||
|
||||
@@ -15,13 +15,16 @@
|
||||
use crate::admin::{
|
||||
auth::validate_admin_request,
|
||||
handlers::notify_runtime_access::{get_notification_system, load_notification_config_snapshot},
|
||||
handlers::supervise_admin_mutation,
|
||||
handlers::target_descriptor::{
|
||||
AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
|
||||
build_json_response, collect_runtime_statuses, extract_supported_target_params,
|
||||
AdminTargetSpec, EndpointKey, RuntimeHealthStatus, TargetEndpointSource, admin_target_spec_from_builtin,
|
||||
build_enabled_target_kvs, build_json_response, collect_runtime_statuses, extract_supported_target_params,
|
||||
merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
|
||||
target_mutation_block_reason as shared_target_mutation_block_reason,
|
||||
},
|
||||
router::{AdminOperation, Operation, S3Router},
|
||||
runtime_sources::{AppContext, app_context_from_req},
|
||||
service::config::{preflight_dynamic_config_reload_for_context, signal_dynamic_config_reload_checked_for_context},
|
||||
};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::server::{
|
||||
@@ -43,6 +46,45 @@ use std::sync::LazyLock;
|
||||
use tracing::{Span, error, info, warn};
|
||||
|
||||
const LOG_COMPONENT_ADMIN_API: &str = "admin_api";
|
||||
|
||||
async fn converge_target_mutation_on_cluster(
|
||||
context: Option<&AppContext>,
|
||||
target_type: &str,
|
||||
local_result: Result<(), rustfs_notify::NotificationError>,
|
||||
) -> S3Result<()> {
|
||||
let subsystem = notification_target_subsystem(target_type)?;
|
||||
let peer_result = signal_dynamic_config_reload_checked_for_context(context, subsystem).await;
|
||||
|
||||
match (local_result, peer_result) {
|
||||
(Ok(()), Ok(())) => Ok(()),
|
||||
(Err(local), Ok(())) => {
|
||||
warn!(target_type, error = %local, "Local notification runtime failed to converge");
|
||||
Err(s3_error!(InternalError, "local notification runtime failed to converge"))
|
||||
}
|
||||
(Ok(()), Err(peer)) => Err(peer),
|
||||
(Err(local), Err(peer)) => {
|
||||
warn!(target_type, error = %local, "Local notification runtime failed while peer convergence also failed");
|
||||
Err(s3_error!(
|
||||
InternalError,
|
||||
"local notification runtime and peer convergence failed: {}",
|
||||
peer
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn notification_target_subsystem(target_type: &str) -> S3Result<&'static str> {
|
||||
notification_target_specs()
|
||||
.iter()
|
||||
.find(|spec| spec.subsystem == target_type)
|
||||
.map(|spec| spec.subsystem)
|
||||
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported notification target type: {}", target_type))
|
||||
}
|
||||
|
||||
async fn preflight_target_mutation_on_cluster(context: Option<&AppContext>, target_type: &str) -> S3Result<()> {
|
||||
preflight_dynamic_config_reload_for_context(context, notification_target_subsystem(target_type)?).await
|
||||
}
|
||||
|
||||
const LOG_SUBSYSTEM_NOTIFICATION_TARGET: &str = "notification_target";
|
||||
const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected";
|
||||
const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed";
|
||||
@@ -198,11 +240,14 @@ struct NotificationEndpoint {
|
||||
account_id: String,
|
||||
service: String,
|
||||
status: String,
|
||||
health_state: String,
|
||||
health_reason: String,
|
||||
source: TargetEndpointSource,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
struct NotificationEndpointsResponse {
|
||||
notify_enabled: bool,
|
||||
notification_endpoints: Vec<NotificationEndpoint>,
|
||||
}
|
||||
|
||||
@@ -229,7 +274,7 @@ async fn authorize_notification_admin_request(req: &S3Request<Body>, action: Adm
|
||||
validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
|
||||
}
|
||||
|
||||
fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
|
||||
fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> S3Result<Option<String>> {
|
||||
shared_target_mutation_block_reason(
|
||||
notification_target_specs(),
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
@@ -256,16 +301,23 @@ async fn notification_target_operation_block_reason(action: &str) -> Option<Stri
|
||||
target_module_disabled_reason("notify", rustfs_config::ENV_NOTIFY_ENABLE, is_notify_module_enabled(), action)
|
||||
}
|
||||
|
||||
fn merge_notification_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<NotificationEndpoint> {
|
||||
shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)
|
||||
.into_iter()
|
||||
.map(|endpoint| NotificationEndpoint {
|
||||
account_id: endpoint.account_id,
|
||||
service: endpoint.service,
|
||||
status: endpoint.status,
|
||||
source: endpoint.source,
|
||||
})
|
||||
.collect()
|
||||
fn merge_notification_endpoints(
|
||||
config: &Config,
|
||||
runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
|
||||
) -> S3Result<Vec<NotificationEndpoint>> {
|
||||
Ok(
|
||||
shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)?
|
||||
.into_iter()
|
||||
.map(|endpoint| NotificationEndpoint {
|
||||
account_id: endpoint.account_id,
|
||||
service: endpoint.service,
|
||||
status: endpoint.status,
|
||||
health_state: endpoint.health_state,
|
||||
health_reason: endpoint.health_reason,
|
||||
source: endpoint.source,
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
fn collect_online_target_arns(region: &str, target_statuses: Vec<(rustfs_targets::arn::TargetID, String)>) -> Vec<String> {
|
||||
@@ -284,6 +336,7 @@ impl Operation for NotificationTarget {
|
||||
let span = Span::current();
|
||||
let _enter = span.enter();
|
||||
let (target_type, target_name) = extract_target_params(¶ms)?;
|
||||
let context = app_context_from_req(&req);
|
||||
|
||||
authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
|
||||
if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
|
||||
@@ -291,7 +344,7 @@ impl Operation for NotificationTarget {
|
||||
return Err(s3_error!(InvalidRequest, "{reason}"));
|
||||
}
|
||||
let (ns, config_snapshot) = load_notification_config_snapshot().await?;
|
||||
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
|
||||
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
|
||||
log_notification_target_operation_blocked!("set_target_config", Some(target_type), Some(target_name), &reason);
|
||||
return Err(s3_error!(InvalidRequest, "{reason}"));
|
||||
}
|
||||
@@ -320,7 +373,15 @@ impl Operation for NotificationTarget {
|
||||
)
|
||||
.await?;
|
||||
|
||||
ns.set_target_config(target_type, target_name, kvs).await.map_err(|e| {
|
||||
let mutation_target_type = target_type.to_owned();
|
||||
let mutation_target_name = target_name.to_owned();
|
||||
supervise_admin_mutation("notification target mutation", async move {
|
||||
preflight_target_mutation_on_cluster(context.as_deref(), &mutation_target_type).await?;
|
||||
let local_result = ns.set_target_config(&mutation_target_type, &mutation_target_name, kvs).await;
|
||||
converge_target_mutation_on_cluster(context.as_deref(), &mutation_target_type, local_result).await
|
||||
})
|
||||
.await
|
||||
.map_err(|e| {
|
||||
log_notification_target_request_failed!(
|
||||
"set_target_config",
|
||||
"set_target_config_failed",
|
||||
@@ -328,7 +389,7 @@ impl Operation for NotificationTarget {
|
||||
Some(target_name),
|
||||
e
|
||||
);
|
||||
s3_error!(InternalError, "failed to set target config: {}", e)
|
||||
e
|
||||
})?;
|
||||
log_notification_target_config_updated!("set_target_config", target_type, target_name);
|
||||
|
||||
@@ -343,11 +404,29 @@ impl Operation for ListNotificationTargets {
|
||||
let span = Span::current();
|
||||
let _enter = span.enter();
|
||||
authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;
|
||||
refresh_persisted_module_switches_from_store().await.map_err(|err| {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_REQUEST_FAILED,
|
||||
component = LOG_COMPONENT_ADMIN_API,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION_TARGET,
|
||||
operation = "list_targets",
|
||||
result = "failed",
|
||||
reason = "module_switch_refresh_failed",
|
||||
error = %err,
|
||||
"admin request failed"
|
||||
);
|
||||
s3_error!(InternalError, "failed to refresh notification module state")
|
||||
})?;
|
||||
let notify_enabled = refresh_notify_module_enabled();
|
||||
let (ns, config) = load_notification_config_snapshot().await?;
|
||||
let runtime_statuses = collect_runtime_statuses(ns.get_target_values().await).await;
|
||||
let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses);
|
||||
let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses)?;
|
||||
|
||||
let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints }).map_err(|e| {
|
||||
let data = serde_json::to_vec(&NotificationEndpointsResponse {
|
||||
notify_enabled,
|
||||
notification_endpoints,
|
||||
})
|
||||
.map_err(|e| {
|
||||
log_notification_target_request_failed!("list_targets", "serialize_targets_failed", None, None, e);
|
||||
s3_error!(InternalError, "failed to serialize targets: {}", e)
|
||||
})?;
|
||||
@@ -388,7 +467,7 @@ impl Operation for ListTargetsArns {
|
||||
let target_statuses = collect_runtime_statuses(ns.get_target_values().await)
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|((account_id, service), status)| (rustfs_targets::arn::TargetID::new(account_id, service), status))
|
||||
.map(|((account_id, service), health)| (rustfs_targets::arn::TargetID::new(account_id, service), health.status))
|
||||
.collect();
|
||||
|
||||
let data_target_arn_list = collect_online_target_arns(region.as_str(), target_statuses);
|
||||
@@ -409,6 +488,7 @@ impl Operation for RemoveNotificationTarget {
|
||||
let span = Span::current();
|
||||
let _enter = span.enter();
|
||||
let (target_type, target_name) = extract_target_params(¶ms)?;
|
||||
let context = app_context_from_req(&req);
|
||||
|
||||
authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
|
||||
if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
|
||||
@@ -416,12 +496,20 @@ impl Operation for RemoveNotificationTarget {
|
||||
return Err(s3_error!(InvalidRequest, "{reason}"));
|
||||
}
|
||||
let (ns, config_snapshot) = load_notification_config_snapshot().await?;
|
||||
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
|
||||
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
|
||||
log_notification_target_operation_blocked!("remove_target_config", Some(target_type), Some(target_name), &reason);
|
||||
return Err(s3_error!(InvalidRequest, "{reason}"));
|
||||
}
|
||||
|
||||
ns.remove_target_config(target_type, target_name).await.map_err(|e| {
|
||||
let mutation_target_type = target_type.to_owned();
|
||||
let mutation_target_name = target_name.to_owned();
|
||||
supervise_admin_mutation("notification target mutation", async move {
|
||||
preflight_target_mutation_on_cluster(context.as_deref(), &mutation_target_type).await?;
|
||||
let local_result = ns.remove_target_config(&mutation_target_type, &mutation_target_name).await;
|
||||
converge_target_mutation_on_cluster(context.as_deref(), &mutation_target_type, local_result).await
|
||||
})
|
||||
.await
|
||||
.map_err(|e| {
|
||||
log_notification_target_request_failed!(
|
||||
"remove_target_config",
|
||||
"remove_target_config_failed",
|
||||
@@ -429,7 +517,7 @@ impl Operation for RemoveNotificationTarget {
|
||||
Some(target_name),
|
||||
e
|
||||
);
|
||||
s3_error!(InternalError, "failed to remove target config: {}", e)
|
||||
e
|
||||
})?;
|
||||
log_notification_target_config_updated!("remove_target_config", target_type, target_name);
|
||||
|
||||
@@ -464,6 +552,54 @@ mod tests {
|
||||
}])
|
||||
}
|
||||
|
||||
fn runtime_health(status: &str) -> RuntimeHealthStatus {
|
||||
let online = status == "online";
|
||||
RuntimeHealthStatus {
|
||||
status: status.to_string(),
|
||||
state: if online { "online" } else { "offline" }.to_string(),
|
||||
reason: if online { "reachable" } else { "unreachable" }.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_target_subsystem_resolves_admin_route_type() {
|
||||
assert_eq!(
|
||||
notification_target_subsystem(NOTIFY_WEBHOOK_SUB_SYS).expect("webhook subsystem should resolve"),
|
||||
NOTIFY_WEBHOOK_SUB_SYS
|
||||
);
|
||||
assert!(notification_target_subsystem("webhook").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_endpoints_response_includes_required_module_state() {
|
||||
let response = NotificationEndpointsResponse {
|
||||
notify_enabled: true,
|
||||
notification_endpoints: vec![NotificationEndpoint {
|
||||
account_id: "primary".to_string(),
|
||||
service: "webhook".to_string(),
|
||||
status: "online".to_string(),
|
||||
health_state: "online".to_string(),
|
||||
health_reason: "reachable".to_string(),
|
||||
source: TargetEndpointSource::Config,
|
||||
}],
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
serde_json::to_value(response).expect("notification target response should serialize"),
|
||||
serde_json::json!({
|
||||
"notify_enabled": true,
|
||||
"notification_endpoints": [{
|
||||
"account_id": "primary",
|
||||
"service": "webhook",
|
||||
"status": "online",
|
||||
"health_state": "online",
|
||||
"health_reason": "reachable",
|
||||
"source": "config"
|
||||
}]
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_notification_endpoints_keeps_configured_targets_after_runtime_loss() {
|
||||
let mut cfg_map = HashMap::new();
|
||||
@@ -477,8 +613,15 @@ mod tests {
|
||||
);
|
||||
let config = Config(cfg_map);
|
||||
|
||||
let runtime = HashMap::from([(("webhook-a".to_string(), "webhook".to_string()), "online".to_string())]);
|
||||
let merged = merge_notification_endpoints(&config, runtime);
|
||||
let runtime = HashMap::from([(
|
||||
("webhook-a".to_string(), "webhook".to_string()),
|
||||
RuntimeHealthStatus {
|
||||
status: "offline".to_string(),
|
||||
state: "error".to_string(),
|
||||
reason: "timed_out".to_string(),
|
||||
},
|
||||
)]);
|
||||
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
|
||||
|
||||
let mqtt = merged
|
||||
.iter()
|
||||
@@ -491,7 +634,9 @@ mod tests {
|
||||
.iter()
|
||||
.find(|entry| entry.account_id == "webhook-a" && entry.service == "webhook")
|
||||
.expect("webhook-a should be present");
|
||||
assert_eq!(webhook.status, "online");
|
||||
assert_eq!(webhook.status, "offline");
|
||||
assert_eq!(webhook.health_state, "error");
|
||||
assert_eq!(webhook.health_reason, "timed_out");
|
||||
assert_eq!(webhook.source, TargetEndpointSource::Config);
|
||||
}
|
||||
|
||||
@@ -504,10 +649,10 @@ mod tests {
|
||||
let config = Config(HashMap::from([(NOTIFY_WEBHOOK_SUB_SYS.to_string(), webhook_targets)]));
|
||||
|
||||
let runtime = HashMap::from([
|
||||
(("webhook-enabled".to_string(), "webhook".to_string()), "online".to_string()),
|
||||
(("env-only".to_string(), "mqtt".to_string()), "offline".to_string()),
|
||||
(("webhook-enabled".to_string(), "webhook".to_string()), runtime_health("online")),
|
||||
(("env-only".to_string(), "mqtt".to_string()), runtime_health("offline")),
|
||||
]);
|
||||
let merged = merge_notification_endpoints(&config, runtime);
|
||||
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
|
||||
|
||||
let env_only = merged
|
||||
.iter()
|
||||
@@ -546,10 +691,10 @@ mod tests {
|
||||
],
|
||||
|| {
|
||||
let runtime = HashMap::from([
|
||||
(("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
|
||||
(("env-only".to_string(), "webhook".to_string()), "online".to_string()),
|
||||
(("mixed-target".to_string(), "webhook".to_string()), runtime_health("online")),
|
||||
(("env-only".to_string(), "webhook".to_string()), runtime_health("online")),
|
||||
]);
|
||||
let merged = merge_notification_endpoints(&config, runtime);
|
||||
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
|
||||
|
||||
let mixed = merged
|
||||
.iter()
|
||||
@@ -589,10 +734,10 @@ mod tests {
|
||||
],
|
||||
|| {
|
||||
let runtime = HashMap::from([
|
||||
(("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
|
||||
(("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
|
||||
(("mixed-kafka".to_string(), "kafka".to_string()), runtime_health("online")),
|
||||
(("env-kafka".to_string(), "kafka".to_string()), runtime_health("online")),
|
||||
]);
|
||||
let merged = merge_notification_endpoints(&config, runtime);
|
||||
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
|
||||
|
||||
let mixed = merged
|
||||
.iter()
|
||||
@@ -626,10 +771,10 @@ mod tests {
|
||||
],
|
||||
|| {
|
||||
let runtime = HashMap::from([
|
||||
(("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
|
||||
(("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
|
||||
(("mixed-amqp".to_string(), "amqp".to_string()), runtime_health("online")),
|
||||
(("env-amqp".to_string(), "amqp".to_string()), runtime_health("online")),
|
||||
]);
|
||||
let merged = merge_notification_endpoints(&config, runtime);
|
||||
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
|
||||
|
||||
let mixed = merged
|
||||
.iter()
|
||||
@@ -656,7 +801,8 @@ mod tests {
|
||||
],
|
||||
|| {
|
||||
let config = Config(HashMap::new());
|
||||
let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
|
||||
let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary")
|
||||
.expect("target mutation block reason");
|
||||
assert!(reason.is_some());
|
||||
assert!(reason.unwrap().contains("managed by environment variables"));
|
||||
},
|
||||
@@ -696,7 +842,8 @@ mod tests {
|
||||
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
|
||||
HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
|
||||
)]));
|
||||
let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
|
||||
let reason =
|
||||
target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary").expect("target mutation block reason");
|
||||
assert!(reason.is_some());
|
||||
assert!(reason.unwrap().contains("both persisted config and environment variables"));
|
||||
});
|
||||
@@ -709,7 +856,11 @@ mod tests {
|
||||
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
|
||||
HashMap::from([(target_name.to_string(), enabled_kvs("on"))]),
|
||||
)]));
|
||||
assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name).is_none());
|
||||
assert!(
|
||||
target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name)
|
||||
.expect("target mutation block reason")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -726,7 +877,7 @@ mod tests {
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
|
||||
],
|
||||
|| {
|
||||
let merged = merge_notification_endpoints(&config, HashMap::new());
|
||||
let merged = merge_notification_endpoints(&config, HashMap::new()).expect("merge notification endpoints");
|
||||
let mixed = merged
|
||||
.iter()
|
||||
.find(|entry| entry.account_id == "mixed-disabled")
|
||||
@@ -748,7 +899,7 @@ mod tests {
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
|
||||
],
|
||||
|| {
|
||||
let merged = merge_notification_endpoints(&config, HashMap::new());
|
||||
let merged = merge_notification_endpoints(&config, HashMap::new()).expect("merge notification endpoints");
|
||||
let env_only = merged
|
||||
.iter()
|
||||
.find(|entry| entry.account_id == "env-only")
|
||||
@@ -797,8 +948,8 @@ mod tests {
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")),
|
||||
],
|
||||
|| {
|
||||
let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
|
||||
let merged = merge_notification_endpoints(&config, runtime);
|
||||
let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), runtime_health("online"))]);
|
||||
let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
|
||||
let mixed = merged
|
||||
.iter()
|
||||
.find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
|
||||
@@ -835,7 +986,11 @@ mod tests {
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", None::<&str>),
|
||||
],
|
||||
|| {
|
||||
assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase").is_none());
|
||||
assert!(
|
||||
target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase")
|
||||
.expect("target mutation block reason")
|
||||
.is_none()
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -864,6 +1019,22 @@ mod tests {
|
||||
list_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"),
|
||||
"notification target list should require GetBucketTargetAction"
|
||||
);
|
||||
let authorize_index = list_block
|
||||
.find("authorize_notification_admin_request")
|
||||
.expect("target list should authorize the request");
|
||||
let refresh_index = list_block
|
||||
.find("refresh_persisted_module_switches_from_store().await.map_err")
|
||||
.expect("target list should fail when persisted module state cannot be refreshed");
|
||||
let effective_state_index = list_block
|
||||
.find("let notify_enabled = refresh_notify_module_enabled();")
|
||||
.expect("target list should resolve the effective env and persisted module state");
|
||||
let load_index = list_block
|
||||
.find("load_notification_config_snapshot().await?")
|
||||
.expect("target list should load notification config");
|
||||
assert!(
|
||||
authorize_index < refresh_index && refresh_index < effective_state_index && effective_state_index < load_index,
|
||||
"target list should authorize, refresh persisted state, resolve effective state, then load targets"
|
||||
);
|
||||
assert!(
|
||||
arns_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"),
|
||||
"notification target arn listing should require GetBucketTargetAction"
|
||||
|
||||
@@ -486,6 +486,8 @@ mod tests {
|
||||
account_id: "primary".to_string(),
|
||||
service: "webhook".to_string(),
|
||||
status: "offline".to_string(),
|
||||
health_state: "offline".to_string(),
|
||||
health_reason: "not_loaded_in_runtime".to_string(),
|
||||
source: PluginInstanceSource::Config,
|
||||
enabled: true,
|
||||
config: HashMap::from([("endpoint".to_string(), "https://example.test/webhook".to_string())]),
|
||||
|
||||
@@ -63,6 +63,19 @@ pub mod user_iam;
|
||||
pub mod user_lifecycle;
|
||||
pub mod user_policy_binding;
|
||||
|
||||
pub(crate) async fn supervise_admin_mutation<T>(
|
||||
operation: &'static str,
|
||||
mutation: impl std::future::Future<Output = s3s::S3Result<T>> + Send + 'static,
|
||||
) -> s3s::S3Result<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
{
|
||||
tokio::spawn(mutation).await.map_err(|err| {
|
||||
let outcome = if err.is_cancelled() { "cancelled" } else { "panicked" };
|
||||
s3s::s3_error!(InternalError, "{} task {}", operation, outcome)
|
||||
})?
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -114,6 +127,44 @@ mod tests {
|
||||
// Test passes if we reach this point without panicking
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn supervised_admin_mutation_survives_waiter_cancellation() {
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let (completed_tx, completed_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let waiter = tokio::spawn(async move {
|
||||
supervise_admin_mutation("test mutation", async move {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.await;
|
||||
let _ = completed_tx.send(());
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
|
||||
started_rx.await.expect("mutation started");
|
||||
waiter.abort();
|
||||
release_tx.send(()).expect("release mutation");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), completed_rx)
|
||||
.await
|
||||
.expect("detached mutation should complete")
|
||||
.expect("completion signal");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn supervised_admin_mutation_does_not_expose_panic_payload() {
|
||||
let error = supervise_admin_mutation::<()>("test mutation", async {
|
||||
panic!("do-not-expose-payload");
|
||||
})
|
||||
.await
|
||||
.expect_err("panicking mutation should fail");
|
||||
|
||||
let rendered = error.to_string();
|
||||
assert!(rendered.contains("panicked"));
|
||||
assert!(!rendered.contains("do-not-expose-payload"));
|
||||
}
|
||||
|
||||
// Note: Testing the actual async handler implementations requires:
|
||||
// 1. S3Request setup with proper headers, URI, and credentials
|
||||
// 2. Global object store initialization
|
||||
|
||||
@@ -12,26 +12,31 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::runtime_sources::default_admin_usecase;
|
||||
use crate::admin::runtime_sources::{AppContext, app_context_from_req, default_admin_usecase};
|
||||
use crate::admin::service::config::{
|
||||
preflight_dynamic_config_reload_for_context, signal_dynamic_config_reload_checked_for_context,
|
||||
};
|
||||
use crate::admin::{
|
||||
auth::validate_admin_request,
|
||||
handlers::supervise_admin_mutation,
|
||||
router::{AdminOperation, Operation, S3Router},
|
||||
};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::server::{
|
||||
ADMIN_PREFIX, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, RemoteAddr, current_module_switch_snapshot,
|
||||
init_event_notifier, refresh_audit_module_enabled, refresh_notify_module_enabled,
|
||||
refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, shutdown_event_notifier,
|
||||
start_audit_system, stop_audit_system, validate_module_switch_update,
|
||||
ADMIN_PREFIX, MODULE_SWITCHES_SIGNAL_SUBSYSTEM, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches,
|
||||
RemoteAddr, apply_audit_module_switch_for_context, current_module_switch_snapshot, mark_event_notifier_reconciled,
|
||||
mark_event_notifier_unreconciled, refresh_audit_module_enabled, refresh_notify_module_enabled,
|
||||
refresh_persisted_module_switches_from, refresh_persisted_module_switches_from_store, save_persisted_module_switches_to,
|
||||
validate_module_switch_update,
|
||||
};
|
||||
use http::{HeaderMap, StatusCode};
|
||||
use hyper::Method;
|
||||
use matchit::Params;
|
||||
use rustfs_audit::AuditError;
|
||||
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub fn register_module_switch_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
|
||||
r.insert(
|
||||
@@ -139,6 +144,74 @@ async fn refresh_module_switch_snapshot() -> S3Result<ModuleSwitchSnapshot> {
|
||||
Ok(current_module_switch_snapshot())
|
||||
}
|
||||
|
||||
async fn apply_module_switch_update(context: Arc<AppContext>, switches: PersistedModuleSwitches) -> S3Result<()> {
|
||||
preflight_dynamic_config_reload_for_context(Some(context.as_ref()), MODULE_SWITCHES_SIGNAL_SUBSYSTEM).await?;
|
||||
let store = context.object_store();
|
||||
mark_event_notifier_unreconciled();
|
||||
let notification_system = rustfs_notify::ensure_live_events();
|
||||
if switches.notify_enabled {
|
||||
notification_system
|
||||
.reload_persisted_config_from_store(store.clone())
|
||||
.await
|
||||
.map_err(|err| {
|
||||
tracing::warn!(error = %err, "Failed to load notification config for module switch update");
|
||||
s3_error!(InternalError, "failed to load notification config")
|
||||
})?;
|
||||
}
|
||||
|
||||
let transition_system = notification_system.clone();
|
||||
let notify_transition = save_persisted_module_switches_to(store.clone(), switches, move || {
|
||||
let enabled = refresh_notify_module_enabled();
|
||||
transition_system.publish_targets_enabled(enabled, None)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| {
|
||||
tracing::warn!(error = %err, "Failed to save module switches");
|
||||
s3_error!(InternalError, "failed to save module switches")
|
||||
})?;
|
||||
|
||||
let mut failures = Vec::new();
|
||||
let mut notify_converged = true;
|
||||
if let Err(err) = notify_transition.wait().await {
|
||||
tracing::warn!(error = %err, "Local notification runtime failed to apply module switch update");
|
||||
notify_converged = false;
|
||||
failures.push("local notify");
|
||||
}
|
||||
if !switches.notify_enabled
|
||||
&& let Err(err) = notification_system.reload_persisted_config_from_store(store).await
|
||||
{
|
||||
tracing::warn!(error = %err, "Local notification config cache failed to reload after module disable");
|
||||
notify_converged = false;
|
||||
failures.push("local notify config cache");
|
||||
}
|
||||
if notify_converged && notification_system.runtime_lifecycle_is_converged() {
|
||||
mark_event_notifier_reconciled();
|
||||
} else if notify_converged {
|
||||
failures.push("local notify convergence");
|
||||
}
|
||||
|
||||
if apply_audit_module_switch_for_context(Some(context.as_ref())).await.is_err() {
|
||||
tracing::warn!(reason = "apply_failed", "Local audit runtime failed to apply module switch update");
|
||||
failures.push("local audit");
|
||||
}
|
||||
if let Err(err) =
|
||||
signal_dynamic_config_reload_checked_for_context(Some(context.as_ref()), MODULE_SWITCHES_SIGNAL_SUBSYSTEM).await
|
||||
{
|
||||
tracing::warn!(error = %err, "Peer nodes failed to apply module switch update");
|
||||
failures.push("peer module switches");
|
||||
}
|
||||
|
||||
if failures.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(s3_error!(
|
||||
InternalError,
|
||||
"module switches persisted but runtime convergence failed: {}",
|
||||
failures.join("; ")
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct GetModuleSwitchesHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -156,7 +229,9 @@ pub struct UpdateModuleSwitchesHandler {}
|
||||
impl Operation for UpdateModuleSwitchesHandler {
|
||||
async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
authorize_module_switch_request(&req, AdminAction::ConfigUpdateAdminAction).await?;
|
||||
refresh_persisted_module_switches_from_store()
|
||||
let context = app_context_from_req(&req).ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))?;
|
||||
let store = context.object_store();
|
||||
refresh_persisted_module_switches_from(store.clone())
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to reload persisted module switches: {}", e))?;
|
||||
|
||||
@@ -183,28 +258,7 @@ impl Operation for UpdateModuleSwitchesHandler {
|
||||
return Err(s3_error!(InvalidRequest, "{err}"));
|
||||
}
|
||||
|
||||
save_persisted_module_switches_to_store(switches)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to save module switches: {}", e))?;
|
||||
|
||||
// Apply the new effective values immediately on this node so the console
|
||||
// response reflects the runtime state after to write completes.
|
||||
if refresh_notify_module_enabled() {
|
||||
init_event_notifier().await;
|
||||
} else {
|
||||
shutdown_event_notifier().await;
|
||||
}
|
||||
|
||||
if refresh_audit_module_enabled() {
|
||||
match start_audit_system().await {
|
||||
Ok(()) | Err(AuditError::AlreadyInitialized) => {}
|
||||
Err(e) => return Err(s3_error!(InternalError, "failed to apply audit module switch: {}", e)),
|
||||
}
|
||||
} else {
|
||||
stop_audit_system()
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to stop audit module after switch update: {}", e))?;
|
||||
}
|
||||
supervise_admin_mutation("module switch update", apply_module_switch_update(context, switches)).await?;
|
||||
|
||||
let snapshot = current_module_switch_snapshot();
|
||||
build_response(StatusCode::OK, &ModuleSwitchesResponse::from(snapshot), req.headers.get("x-request-id"))
|
||||
@@ -233,6 +287,12 @@ mod tests {
|
||||
put_block.contains("authorize_module_switch_request(&req, AdminAction::ConfigUpdateAdminAction).await?;"),
|
||||
"module switch PUT should require ConfigUpdateAdminAction"
|
||||
);
|
||||
assert!(
|
||||
put_block.contains(
|
||||
"supervise_admin_mutation(\"module switch update\", apply_module_switch_update(context, switches)).await?;"
|
||||
),
|
||||
"module switch PUT must delegate the complete mutation to its supervisor"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -12,31 +12,33 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::server::init_event_notifier;
|
||||
use crate::server::{init_event_notifier, is_event_notifier_reconciled};
|
||||
use rustfs_config::server_config::Config;
|
||||
use s3s::{S3Result, s3_error};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
static NOTIFICATION_SYSTEM_INIT_LOCK: Mutex<()> = Mutex::const_new(());
|
||||
|
||||
pub(crate) async fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>> {
|
||||
if let Some(system) = rustfs_notify::notification_system() {
|
||||
if is_event_notifier_reconciled()
|
||||
&& let Some(system) = rustfs_notify::notification_system()
|
||||
&& system.runtime_lifecycle_is_converged()
|
||||
{
|
||||
return Ok(system);
|
||||
}
|
||||
|
||||
let _guard = NOTIFICATION_SYSTEM_INIT_LOCK.lock().await;
|
||||
if let Some(system) = rustfs_notify::notification_system() {
|
||||
return Ok(system);
|
||||
init_event_notifier()
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to reconcile notification runtime: {}", err))?;
|
||||
let system =
|
||||
rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))?;
|
||||
if !system.runtime_lifecycle_is_converged() {
|
||||
return Err(s3_error!(InternalError, "latest notification lifecycle generation has not converged"));
|
||||
}
|
||||
|
||||
init_event_notifier().await;
|
||||
rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))
|
||||
Ok(system)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_notification_config_snapshot() -> S3Result<(Arc<rustfs_notify::NotificationSystem>, Config)> {
|
||||
let system = get_notification_system().await?;
|
||||
let config = system.config.read().await.clone();
|
||||
let config = system.config_snapshot().await;
|
||||
Ok((system, config))
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,10 @@ use crate::admin::runtime_sources::{
|
||||
current_server_config_for_context,
|
||||
};
|
||||
use crate::admin::service::federated_identity::DefaultFederatedSessionBinding;
|
||||
use crate::admin::storage_api::config::{read_admin_config_without_migrate, save_admin_server_config};
|
||||
use crate::admin::storage_api::config::{
|
||||
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
|
||||
with_admin_server_config_write_lock,
|
||||
};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
|
||||
use http::StatusCode;
|
||||
@@ -368,13 +371,20 @@ impl Operation for PutOidcConfigHandler {
|
||||
if is_env_managed_provider(provider_id) {
|
||||
return Err(s3_error!(AccessDenied, "provider is managed by environment variables"));
|
||||
}
|
||||
let provider_id = provider_id.to_owned();
|
||||
|
||||
let request: OidcConfigUpsertRequest = parse_json_body(&mut req).await?;
|
||||
let mut config = load_server_config_from_store().await?;
|
||||
let existing_secret = persisted_provider_secret(&config, provider_id);
|
||||
let provider_config = build_provider_config_from_upsert(provider_id, request, existing_secret)?;
|
||||
upsert_persisted_provider_config(&mut config, &provider_config);
|
||||
save_server_config_to_store(&config).await?;
|
||||
let store = oidc_config_store()?;
|
||||
let lock_store = store.clone();
|
||||
with_admin_server_config_write_lock(lock_store, move || async move {
|
||||
let mut config = load_server_config_from_store_locked(store.clone()).await?;
|
||||
let existing_secret = persisted_provider_secret(&config, &provider_id);
|
||||
let provider_config = build_provider_config_from_upsert(&provider_id, request, existing_secret)?;
|
||||
upsert_persisted_provider_config(&mut config, &provider_config);
|
||||
save_server_config_to_store_locked(store, &config).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
|
||||
|
||||
json_response(
|
||||
StatusCode::OK,
|
||||
@@ -403,10 +413,17 @@ impl Operation for DeleteOidcConfigHandler {
|
||||
if is_env_managed_provider(provider_id) {
|
||||
return Err(s3_error!(AccessDenied, "provider is managed by environment variables"));
|
||||
}
|
||||
let provider_id = provider_id.to_owned();
|
||||
|
||||
let mut config = load_server_config_from_store().await?;
|
||||
delete_persisted_provider_config(&mut config, provider_id)?;
|
||||
save_server_config_to_store(&config).await?;
|
||||
let store = oidc_config_store()?;
|
||||
let lock_store = store.clone();
|
||||
with_admin_server_config_write_lock(lock_store, move || async move {
|
||||
let mut config = load_server_config_from_store_locked(store.clone()).await?;
|
||||
delete_persisted_provider_config(&mut config, &provider_id)?;
|
||||
save_server_config_to_store_locked(store, &config).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
|
||||
|
||||
json_response(
|
||||
StatusCode::OK,
|
||||
@@ -576,8 +593,6 @@ impl Operation for OidcCallbackHandler {
|
||||
result = "code_exchange_failed",
|
||||
requested_provider_id = %provider_id,
|
||||
redirect_uri = %redirect_uri,
|
||||
code = %code,
|
||||
state = %state,
|
||||
code_len = code.len(),
|
||||
state_len = state.len(),
|
||||
error = %message,
|
||||
@@ -881,23 +896,32 @@ fn json_response<T: Serialize>(status: StatusCode, payload: &T) -> S3Result<S3Re
|
||||
}
|
||||
|
||||
async fn load_server_config_from_store() -> S3Result<ServerConfig> {
|
||||
let context = current_app_context();
|
||||
let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
|
||||
return Err(s3_error!(InternalError, "storage layer not initialized"));
|
||||
};
|
||||
let store = oidc_config_store()?;
|
||||
|
||||
read_admin_config_without_migrate(store)
|
||||
.await
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))
|
||||
}
|
||||
|
||||
async fn save_server_config_to_store(config: &ServerConfig) -> S3Result<()> {
|
||||
fn oidc_config_store() -> S3Result<std::sync::Arc<crate::admin::storage_api::runtime::ECStore>> {
|
||||
let context = current_app_context();
|
||||
let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
|
||||
return Err(s3_error!(InternalError, "storage layer not initialized"));
|
||||
};
|
||||
current_object_store_handle_for_context(context.as_deref())
|
||||
.ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))
|
||||
}
|
||||
|
||||
save_admin_server_config(store, config)
|
||||
async fn load_server_config_from_store_locked(
|
||||
store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
|
||||
) -> S3Result<ServerConfig> {
|
||||
read_admin_config_without_migrate_no_lock(store)
|
||||
.await
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))
|
||||
}
|
||||
|
||||
async fn save_server_config_to_store_locked(
|
||||
store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
|
||||
config: &ServerConfig,
|
||||
) -> S3Result<()> {
|
||||
save_admin_server_config_no_lock(store, config)
|
||||
.await
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}")))
|
||||
}
|
||||
|
||||
@@ -19,8 +19,8 @@ use crate::admin::{
|
||||
load_notification_config_snapshot, remove_notification_target_config, set_notification_target_config,
|
||||
},
|
||||
handlers::target_descriptor::{
|
||||
AdminTargetSpec, TargetEndpointSource, TargetInstanceReadModel, admin_target_spec_from_builtin, build_enabled_target_kvs,
|
||||
build_json_response, collect_runtime_statuses, collect_target_instances, find_target_instance,
|
||||
AdminTargetSpec, RuntimeHealthStatus, TargetEndpointSource, TargetInstanceReadModel, admin_target_spec_from_builtin,
|
||||
build_enabled_target_kvs, build_json_response, collect_runtime_statuses, collect_target_instances, find_target_instance,
|
||||
target_module_disabled_reason, target_mutation_block_reason as shared_target_mutation_block_reason,
|
||||
},
|
||||
plugin_contract::{
|
||||
@@ -205,6 +205,8 @@ fn map_instance(instance: TargetInstanceReadModel) -> PluginInstanceEntry {
|
||||
account_id: instance.account_id,
|
||||
service: instance.service,
|
||||
status: instance.status,
|
||||
health_state: instance.health_state,
|
||||
health_reason: instance.health_reason,
|
||||
source: map_instance_source(instance.source),
|
||||
enabled: instance.enabled,
|
||||
config,
|
||||
@@ -605,7 +607,7 @@ fn plugin_instance_mutation_block_reason(
|
||||
target_type: &str,
|
||||
target_name: &str,
|
||||
target_label: &str,
|
||||
) -> Option<String> {
|
||||
) -> S3Result<Option<String>> {
|
||||
shared_target_mutation_block_reason(context.specs, context.route_prefix, config, target_type, target_name, target_label)
|
||||
}
|
||||
|
||||
@@ -639,7 +641,9 @@ async fn plugin_instance_operation_block_reason(context: PluginInstanceDomainCon
|
||||
module_disabled_block_reason(context.domain, action)
|
||||
}
|
||||
|
||||
async fn plugin_instance_runtime_statuses(context: PluginInstanceDomainContext) -> S3Result<HashMap<(String, String), String>> {
|
||||
async fn plugin_instance_runtime_statuses(
|
||||
context: PluginInstanceDomainContext,
|
||||
) -> S3Result<HashMap<(String, String), RuntimeHealthStatus>> {
|
||||
match context.domain {
|
||||
PluginContractDomain::Notify => {
|
||||
let (ns, _) = load_notification_config_snapshot().await?;
|
||||
@@ -667,7 +671,7 @@ async fn collect_domain_instances(context: PluginInstanceDomainContext) -> S3Res
|
||||
let config = plugin_instance_config_snapshot(context).await?;
|
||||
let module_disabled_reason = module_disabled_block_reason(context.domain, "listing plugin instances");
|
||||
let mut entries = Vec::new();
|
||||
for instance in collect_target_instances(context.specs, context.route_prefix, &config, runtime_statuses) {
|
||||
for instance in collect_target_instances(context.specs, context.route_prefix, &config, runtime_statuses)? {
|
||||
entries.push(plugin_instance_list_entry(instance, module_disabled_reason.clone()));
|
||||
}
|
||||
Ok(entries)
|
||||
@@ -688,13 +692,7 @@ async fn find_plugin_instance(instance_id: &str) -> S3Result<Option<TargetInstan
|
||||
let context = plugin_instance_domain_context(parse_plugin_instance_id(instance_id)?.1);
|
||||
let runtime_statuses = plugin_instance_runtime_statuses(context).await?;
|
||||
let config = plugin_instance_config_snapshot(context).await?;
|
||||
Ok(find_target_instance(
|
||||
context.specs,
|
||||
context.route_prefix,
|
||||
&config,
|
||||
runtime_statuses,
|
||||
instance_id,
|
||||
))
|
||||
find_target_instance(context.specs, context.route_prefix, &config, runtime_statuses, instance_id)
|
||||
}
|
||||
|
||||
async fn set_plugin_instance_config(
|
||||
@@ -800,7 +798,7 @@ impl Operation for PutPluginInstanceHandler {
|
||||
resolved.target_spec.subsystem,
|
||||
&resolved.target_name,
|
||||
"plugin instance",
|
||||
) {
|
||||
)? {
|
||||
return Err(s3_error!(InvalidRequest, "{reason}"));
|
||||
}
|
||||
|
||||
@@ -852,7 +850,7 @@ impl Operation for DeletePluginInstanceHandler {
|
||||
resolved.target_spec.subsystem,
|
||||
&resolved.target_name,
|
||||
"plugin instance",
|
||||
) {
|
||||
)? {
|
||||
return Err(s3_error!(InvalidRequest, "{reason}"));
|
||||
}
|
||||
|
||||
@@ -871,7 +869,8 @@ mod tests {
|
||||
parse_plugin_instance_id, parse_plugin_instance_source, resolve_plugin_instance_target,
|
||||
};
|
||||
use crate::admin::handlers::target_descriptor::{
|
||||
TargetEndpointSource, TargetInstanceReadModel, canonical_target_instance_id, collect_target_instances,
|
||||
RuntimeHealthStatus, TargetEndpointSource, TargetInstanceReadModel, canonical_target_instance_id,
|
||||
collect_target_instances,
|
||||
};
|
||||
use crate::admin::plugin_contract::{
|
||||
PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry, PluginInstanceSource,
|
||||
@@ -965,16 +964,54 @@ mod tests {
|
||||
)]));
|
||||
|
||||
let instances =
|
||||
collect_target_instances(super::notification_target_specs(), NOTIFY_ROUTE_PREFIX, &config, HashMap::new());
|
||||
collect_target_instances(super::notification_target_specs(), NOTIFY_ROUTE_PREFIX, &config, HashMap::new())
|
||||
.expect("collect target instances");
|
||||
let primary = instances
|
||||
.into_iter()
|
||||
.find(|instance| instance.account_id == "primary" && instance.service == "webhook")
|
||||
.expect("configured instance should be present");
|
||||
|
||||
assert_eq!(primary.status, "offline");
|
||||
assert_eq!(primary.health_state, "offline");
|
||||
assert_eq!(primary.health_reason, "not_loaded_in_runtime");
|
||||
assert_eq!(primary.source, TargetEndpointSource::Config);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_instance_without_runtime_reports_disabled_health() {
|
||||
let config = Config(HashMap::from([(
|
||||
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
|
||||
HashMap::from([(
|
||||
"primary".to_string(),
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_string(),
|
||||
value: "off".to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: WEBHOOK_ENDPOINT.to_string(),
|
||||
value: "https://example.com/webhook".to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
]),
|
||||
)]),
|
||||
)]));
|
||||
|
||||
let instances =
|
||||
collect_target_instances(super::notification_target_specs(), NOTIFY_ROUTE_PREFIX, &config, HashMap::new())
|
||||
.expect("collect target instances");
|
||||
let primary = instances
|
||||
.into_iter()
|
||||
.find(|instance| instance.account_id == "primary" && instance.service == "webhook")
|
||||
.expect("disabled instance should be present");
|
||||
|
||||
assert_eq!(primary.status, "offline");
|
||||
assert_eq!(primary.health_state, "disabled");
|
||||
assert_eq!(primary.health_reason, "disabled");
|
||||
assert!(!primary.runtime_present);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn env_only_instance_appears_with_env_source() {
|
||||
temp_env::with_vars(
|
||||
@@ -988,7 +1025,8 @@ mod tests {
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
&Config(HashMap::new()),
|
||||
HashMap::new(),
|
||||
);
|
||||
)
|
||||
.expect("collect target instances");
|
||||
let env_only = instances
|
||||
.into_iter()
|
||||
.find(|instance| instance.account_id == "env-only")
|
||||
@@ -1002,13 +1040,21 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn runtime_only_instance_appears_with_runtime_source() {
|
||||
let runtime_statuses = HashMap::from([(("runtime-only".to_string(), "webhook".to_string()), "online".to_string())]);
|
||||
let runtime_statuses = HashMap::from([(
|
||||
("runtime-only".to_string(), "webhook".to_string()),
|
||||
RuntimeHealthStatus {
|
||||
status: "online".to_string(),
|
||||
state: "online".to_string(),
|
||||
reason: "reachable".to_string(),
|
||||
},
|
||||
)]);
|
||||
let instances = collect_target_instances(
|
||||
super::notification_target_specs(),
|
||||
NOTIFY_ROUTE_PREFIX,
|
||||
&Config(HashMap::new()),
|
||||
runtime_statuses,
|
||||
);
|
||||
)
|
||||
.expect("collect target instances");
|
||||
|
||||
let runtime_only = instances
|
||||
.into_iter()
|
||||
@@ -1017,6 +1063,8 @@ mod tests {
|
||||
|
||||
assert_eq!(runtime_only.source, TargetEndpointSource::Runtime);
|
||||
assert_eq!(runtime_only.status, "online");
|
||||
assert_eq!(runtime_only.health_state, "online");
|
||||
assert_eq!(runtime_only.health_reason, "reachable");
|
||||
assert_eq!(runtime_only.plugin_id, "builtin:webhook");
|
||||
assert_eq!(runtime_only.subsystem, NOTIFY_WEBHOOK_SUB_SYS);
|
||||
}
|
||||
@@ -1031,6 +1079,8 @@ mod tests {
|
||||
account_id: "Primary".to_string(),
|
||||
service: "webhook".to_string(),
|
||||
status: "offline".to_string(),
|
||||
health_state: "offline".to_string(),
|
||||
health_reason: "not_loaded_in_runtime".to_string(),
|
||||
runtime_present: false,
|
||||
source: TargetEndpointSource::Config,
|
||||
enabled: true,
|
||||
@@ -1053,6 +1103,8 @@ mod tests {
|
||||
account_id: "primary".to_string(),
|
||||
service: "webhook".to_string(),
|
||||
status: "online".to_string(),
|
||||
health_state: "online".to_string(),
|
||||
health_reason: "reachable".to_string(),
|
||||
runtime_present: true,
|
||||
source: TargetEndpointSource::Config,
|
||||
enabled: true,
|
||||
@@ -1484,6 +1536,8 @@ mod tests {
|
||||
account_id: "primary".to_string(),
|
||||
service: "webhook".to_string(),
|
||||
status: "offline".to_string(),
|
||||
health_state: "offline".to_string(),
|
||||
health_reason: "not_loaded_in_runtime".to_string(),
|
||||
runtime_present: false,
|
||||
source: TargetEndpointSource::Config,
|
||||
enabled: true,
|
||||
@@ -1508,6 +1562,8 @@ mod tests {
|
||||
account_id: "primary".to_string(),
|
||||
service: "webhook".to_string(),
|
||||
status: "offline".to_string(),
|
||||
health_state: "offline".to_string(),
|
||||
health_reason: "unreachable".to_string(),
|
||||
runtime_present: true,
|
||||
source: TargetEndpointSource::Runtime,
|
||||
enabled: true,
|
||||
@@ -1532,6 +1588,8 @@ mod tests {
|
||||
account_id: "primary".to_string(),
|
||||
service: "webhook".to_string(),
|
||||
status: "offline".to_string(),
|
||||
health_state: "disabled".to_string(),
|
||||
health_reason: "disabled".to_string(),
|
||||
runtime_present: false,
|
||||
source: TargetEndpointSource::Mixed,
|
||||
enabled: false,
|
||||
@@ -1573,6 +1631,8 @@ mod tests {
|
||||
account_id: "primary".to_string(),
|
||||
service: "webhook".to_string(),
|
||||
status: "offline".to_string(),
|
||||
health_state: "offline".to_string(),
|
||||
health_reason: "not_loaded_in_runtime".to_string(),
|
||||
runtime_present: false,
|
||||
source: TargetEndpointSource::Config,
|
||||
enabled: true,
|
||||
@@ -1686,6 +1746,13 @@ mod tests {
|
||||
account_id: input.account_id.to_string(),
|
||||
service: input.service.to_string(),
|
||||
status: input.status.to_string(),
|
||||
health_state: input.status.to_string(),
|
||||
health_reason: if input.status == "online" {
|
||||
"reachable"
|
||||
} else {
|
||||
"unreachable"
|
||||
}
|
||||
.to_string(),
|
||||
source: input.source,
|
||||
enabled: input.enabled,
|
||||
config: HashMap::new(),
|
||||
|
||||
@@ -108,6 +108,23 @@ fn is_service_account_owner_of(caller: &StoredCredentials, target_parent_user: &
|
||||
caller_parent == target_parent_user
|
||||
}
|
||||
|
||||
/// GHSA-5354: whether a caller resolving to `owner` may create a service account
|
||||
/// parented to `target_user`, given the caller's own key (`req_user`) and its
|
||||
/// effective parent (`req_parent_user`, which equals `req_user` for a top-level
|
||||
/// user or the parent for a derived credential).
|
||||
///
|
||||
/// The `CreateServiceAccountAdminAction` permission gates *whether* a caller may
|
||||
/// create service accounts, not *for whom*. A non-owner is therefore confined to
|
||||
/// its own scope; only an owner may target another user — including the root
|
||||
/// credential, whose existence check is deliberately lenient. Without this a
|
||||
/// caller holding only that action could pass `target_user = <root access key>`
|
||||
/// and mint a root-parented service account that authenticates as owner via
|
||||
/// `prepare_service_account_auth`, a full-takeover privilege escalation. Mirrors
|
||||
/// `imported_service_account_parent_allowed` on the ImportIam path (GHSA-566f).
|
||||
fn add_service_account_parent_within_scope(target_user: &str, req_user: &str, req_parent_user: &str, owner: bool) -> bool {
|
||||
owner || target_user == req_user || target_user == req_parent_user
|
||||
}
|
||||
|
||||
/// Fail-closed ownership check used by DeleteServiceAccount for a caller that
|
||||
/// lacks the RemoveServiceAccount admin action.
|
||||
///
|
||||
@@ -342,6 +359,16 @@ impl Operation for AddServiceAccount {
|
||||
|
||||
let is_svc_acc = target_user == req_user || target_user == req_parent_user;
|
||||
|
||||
// GHSA-5354: confine a non-owner caller to its own scope so it cannot mint
|
||||
// a root-parented (owner) service account. Evaluated on the original
|
||||
// `target_user` the caller submitted, before the derived-credential rewrite
|
||||
// to `req_parent_user` below, so the guard is bound to the parent actually
|
||||
// requested. Its allow set is exactly `owner || is_svc_acc`. See the helper's
|
||||
// doc comment.
|
||||
if !add_service_account_parent_within_scope(&target_user, &req_user, &req_parent_user, owner) {
|
||||
return Err(s3_error!(AccessDenied, "service account parent is outside requester scope"));
|
||||
}
|
||||
|
||||
let mut target_groups = None;
|
||||
let mut opts = NewServiceAccountOpts {
|
||||
access_key: create_req.access_key,
|
||||
@@ -1408,6 +1435,86 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ghsa_5354_non_owner_service_account_parent_confined_to_scope() {
|
||||
// Regression (GHSA-5354): a caller holding CreateServiceAccountAdminAction
|
||||
// but resolving to owner=false must not be able to parent a new service
|
||||
// account to the root credential (or any other user). A root-parented SA
|
||||
// authenticates as owner, so allowing this is a full-takeover escalation.
|
||||
let root = rustfs_credentials::DEFAULT_ACCESS_KEY; // "rustfsadmin"
|
||||
|
||||
// Attack: non-owner "alice" targets root as the parent -> DENY.
|
||||
assert!(
|
||||
!add_service_account_parent_within_scope(root, "alice", "alice", false),
|
||||
"non-owner must not parent a service account to root"
|
||||
);
|
||||
// Non-owner targeting an unrelated user -> DENY.
|
||||
assert!(
|
||||
!add_service_account_parent_within_scope("bob", "alice", "alice", false),
|
||||
"non-owner must not parent a service account to another user"
|
||||
);
|
||||
// Self-scoped creation (default target == own key) -> ALLOW.
|
||||
assert!(
|
||||
add_service_account_parent_within_scope("alice", "alice", "alice", false),
|
||||
"non-owner may create a service account for itself"
|
||||
);
|
||||
// Derived credential targeting its own parent -> ALLOW.
|
||||
assert!(
|
||||
add_service_account_parent_within_scope("parent-user", "svc-key", "parent-user", false),
|
||||
"derived credential may create a service account under its own parent"
|
||||
);
|
||||
// Owner may target any parent, including root (cross-user creation intact).
|
||||
assert!(
|
||||
add_service_account_parent_within_scope(root, root, root, true),
|
||||
"owner retains cross-user / root-parent creation"
|
||||
);
|
||||
assert!(
|
||||
add_service_account_parent_within_scope("bob", root, root, true),
|
||||
"owner may create a service account for another user"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ghsa_5354_scope_guard_matches_owner_or_self_scope_for_derived_credentials() {
|
||||
// The handler evaluates `add_service_account_parent_within_scope` on the
|
||||
// original `target_user`, before the `target_user = req_parent_user` rewrite
|
||||
// taken inside the `is_svc_acc` branch. The guard's allow set must therefore
|
||||
// stay exactly `owner || is_svc_acc`, where
|
||||
// `is_svc_acc == (target_user == req_user || target_user == req_parent_user)`.
|
||||
//
|
||||
// The named-boundary test above only exercises the self-scoped case with
|
||||
// `req_user == req_parent_user`. A *derived* credential has `req_user`
|
||||
// (its own key) distinct from `req_parent_user` (its parent), which is the
|
||||
// realistic attacker shape: a service account holding
|
||||
// CreateServiceAccountAdminAction. Pin the equivalence across that shape so a
|
||||
// later change to either the guard or the rewrite cannot silently let a
|
||||
// derived non-owner credential escape its own parent's scope.
|
||||
let root = rustfs_credentials::DEFAULT_ACCESS_KEY;
|
||||
let cases: &[(&str, &str, &str, bool)] = &[
|
||||
// Derived non-owner (own key != parent) aiming at root -> deny.
|
||||
(root, "attacker-sa", "alice", false),
|
||||
// Derived non-owner aiming at an unrelated user -> deny.
|
||||
("bob", "attacker-sa", "alice", false),
|
||||
// Derived non-owner aiming at its own parent -> allow.
|
||||
("alice", "attacker-sa", "alice", false),
|
||||
// Derived non-owner aiming at its own key -> allow.
|
||||
("attacker-sa", "attacker-sa", "alice", false),
|
||||
// Top-level non-owner aiming at itself -> allow.
|
||||
("alice", "alice", "alice", false),
|
||||
// Owner is unrestricted, including root and a derived shape.
|
||||
(root, "attacker-sa", "alice", true),
|
||||
("bob", root, root, true),
|
||||
];
|
||||
for &(target_user, req_user, req_parent_user, owner) in cases {
|
||||
let is_svc_acc = target_user == req_user || target_user == req_parent_user;
|
||||
assert_eq!(
|
||||
add_service_account_parent_within_scope(target_user, req_user, req_parent_user, owner),
|
||||
owner || is_svc_acc,
|
||||
"scope guard must equal `owner || is_svc_acc` for (target={target_user}, req_user={req_user}, req_parent={req_parent_user}, owner={owner})"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guess_user_provider_detects_builtin_accounts() {
|
||||
let credentials = StoredCredentials {
|
||||
|
||||
@@ -114,6 +114,8 @@ const SITE_REPL_REMOVE_SUCCESS: &str = "Requested site(s) were removed from clus
|
||||
const SITE_REPL_RESYNC_START: &str = "start";
|
||||
const SITE_REPL_RESYNC_CANCEL: &str = "cancel";
|
||||
const SITE_REPL_RESYNC_STATUS: &str = "status";
|
||||
const SITE_REPL_RESYNC_DEFAULT_PAGE_SIZE: usize = 100;
|
||||
const SITE_REPL_RESYNC_MAX_PAGE_SIZE: usize = 1000;
|
||||
const SITE_REPL_MIN_NETPERF_DURATION: Duration = Duration::from_secs(1);
|
||||
const SITE_REPL_MAX_NETPERF_DURATION: Duration = Duration::from_secs(30);
|
||||
const SITE_REPLICATION_PEER_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
@@ -187,7 +189,11 @@ fn site_replicator_service_account_policy() -> S3Result<Policy> {
|
||||
"s3:PutObjectTagging",
|
||||
"s3:PutObjectVersionTagging",
|
||||
"s3:DeleteObjectTagging",
|
||||
"s3:DeleteObjectVersionTagging"
|
||||
"s3:DeleteObjectVersionTagging",
|
||||
"s3:GetObjectRetention",
|
||||
"s3:PutObjectRetention",
|
||||
"s3:GetObjectLegalHold",
|
||||
"s3:PutObjectLegalHold"
|
||||
],
|
||||
"Resource": ["arn:aws:s3:::*/*"]
|
||||
}
|
||||
@@ -4890,32 +4896,160 @@ fn removed_deployment_ids_for_pending_remove(pending: &PendingRemove, local_peer
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn resync_status_for_state(
|
||||
state: &mut SiteReplicationState,
|
||||
op_type: &str,
|
||||
peer: &PeerInfo,
|
||||
bucket_names: Vec<String>,
|
||||
) -> SRResyncOpStatus {
|
||||
let status = SRResyncOpStatus {
|
||||
op_type: op_type.to_string(),
|
||||
resync_id: Uuid::new_v4().to_string(),
|
||||
status: "success".to_string(),
|
||||
buckets: bucket_names
|
||||
.into_iter()
|
||||
.map(|bucket| ResyncBucketStatus {
|
||||
bucket,
|
||||
status: if op_type == SITE_REPL_RESYNC_CANCEL {
|
||||
"canceled".to_string()
|
||||
} else {
|
||||
"started".to_string()
|
||||
},
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
..Default::default()
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct SiteResyncContinuationToken {
|
||||
id: String,
|
||||
generation: u64,
|
||||
offset: usize,
|
||||
}
|
||||
|
||||
fn site_resync_is_active(status: &SRResyncOpStatus) -> bool {
|
||||
matches!(status.state.as_str(), "pending" | "running" | "canceling")
|
||||
}
|
||||
|
||||
fn site_resync_cancel_is_idempotent(status: &SRResyncOpStatus) -> bool {
|
||||
status.state == "canceled"
|
||||
}
|
||||
|
||||
fn site_resync_nonnegative(value: i64) -> u64 {
|
||||
u64::try_from(value.max(0)).unwrap_or_default()
|
||||
}
|
||||
|
||||
fn site_resync_bucket_state(status: replication::ResyncStatusType) -> &'static str {
|
||||
match status {
|
||||
replication::ResyncStatusType::ResyncPending => "pending",
|
||||
replication::ResyncStatusType::ResyncStarted => "running",
|
||||
replication::ResyncStatusType::ResyncCompleted => "completed",
|
||||
replication::ResyncStatusType::ResyncCanceled => "canceled",
|
||||
replication::ResyncStatusType::ResyncFailed | replication::ResyncStatusType::NoResync => "failed",
|
||||
}
|
||||
}
|
||||
|
||||
fn site_bucket_resync_is_active(status: replication::ResyncStatusType) -> bool {
|
||||
matches!(
|
||||
status,
|
||||
replication::ResyncStatusType::ResyncPending | replication::ResyncStatusType::ResyncStarted
|
||||
)
|
||||
}
|
||||
|
||||
fn apply_site_resync_target_status(bucket: &mut ResyncBucketStatus, target: &replication::TargetReplicationResyncStatus) {
|
||||
bucket.status = site_resync_bucket_state(target.resync_status).to_string();
|
||||
bucket.started_at = target.start_time;
|
||||
bucket.updated_at = target.last_update;
|
||||
bucket.replicated_objects = site_resync_nonnegative(target.replicated_count);
|
||||
bucket.replicated_bytes = site_resync_nonnegative(target.replicated_size);
|
||||
bucket.failed_objects = site_resync_nonnegative(target.failed_count);
|
||||
bucket.failed_bytes = site_resync_nonnegative(target.failed_size);
|
||||
bucket.err_detail = target.error.as_deref().map(summarize_peer_error_detail).unwrap_or_default();
|
||||
if matches!(bucket.status.as_str(), "completed" | "canceled" | "failed") {
|
||||
bucket.completed_at = bucket.updated_at;
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_site_resync_status(status: &mut SRResyncOpStatus, now: OffsetDateTime) {
|
||||
status.total_buckets = status.buckets.len() as u64;
|
||||
status.pending_buckets = 0;
|
||||
status.running_buckets = 0;
|
||||
status.completed_buckets = 0;
|
||||
status.failed_buckets = 0;
|
||||
status.canceled_buckets = 0;
|
||||
status.replicated_objects = 0;
|
||||
status.replicated_bytes = 0;
|
||||
status.failed_objects = 0;
|
||||
status.failed_bytes = 0;
|
||||
|
||||
for bucket in &status.buckets {
|
||||
match bucket.status.as_str() {
|
||||
"pending" => status.pending_buckets += 1,
|
||||
"running" | "started" => status.running_buckets += 1,
|
||||
"completed" | "success" => status.completed_buckets += 1,
|
||||
"canceled" => status.canceled_buckets += 1,
|
||||
_ => status.failed_buckets += 1,
|
||||
}
|
||||
status.replicated_objects = status.replicated_objects.saturating_add(bucket.replicated_objects);
|
||||
status.replicated_bytes = status.replicated_bytes.saturating_add(bucket.replicated_bytes);
|
||||
status.failed_objects = status.failed_objects.saturating_add(bucket.failed_objects);
|
||||
status.failed_bytes = status.failed_bytes.saturating_add(bucket.failed_bytes);
|
||||
}
|
||||
|
||||
status.updated_at = Some(now);
|
||||
status.status = if status.failed_buckets > 0 { "failed" } else { "success" }.to_string();
|
||||
let has_active_buckets = status.pending_buckets > 0
|
||||
|| status.running_buckets > 0
|
||||
|| status.buckets.iter().any(|bucket| bucket.status == "conflict");
|
||||
status.state = if has_active_buckets {
|
||||
if status.op_type == SITE_REPL_RESYNC_CANCEL {
|
||||
"canceling"
|
||||
} else if status.running_buckets > 0 {
|
||||
"running"
|
||||
} else {
|
||||
"pending"
|
||||
}
|
||||
} else if status.failed_buckets > 0 {
|
||||
"failed"
|
||||
} else if status.op_type == SITE_REPL_RESYNC_CANCEL || status.canceled_buckets == status.total_buckets {
|
||||
"canceled"
|
||||
} else {
|
||||
"completed"
|
||||
}
|
||||
.to_string();
|
||||
if matches!(status.state.as_str(), "completed" | "canceled" | "failed") && status.completed_at.is_none() {
|
||||
status.completed_at = Some(now);
|
||||
}
|
||||
status.err_detail = if status.failed_buckets > 0 {
|
||||
format!("{} of {} buckets failed", status.failed_buckets, status.total_buckets)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
state.resync_status.insert(peer.deployment_id.clone(), status.clone());
|
||||
status
|
||||
}
|
||||
|
||||
fn site_resync_page(status: &SRResyncOpStatus, limit: usize, offset: usize) -> S3Result<SRResyncOpStatus> {
|
||||
if offset > status.buckets.len() {
|
||||
return Err(s3_error!(InvalidRequest, "invalid resync continuation token"));
|
||||
}
|
||||
let mut response = status.clone();
|
||||
let end = offset.saturating_add(limit).min(status.buckets.len());
|
||||
response.buckets = status.buckets[offset..end].to_vec();
|
||||
response.truncated = end < status.buckets.len();
|
||||
response.next_continuation_token = if response.truncated {
|
||||
let token = SiteResyncContinuationToken {
|
||||
id: status.resync_id.clone(),
|
||||
generation: status.generation,
|
||||
offset: end,
|
||||
};
|
||||
let encoded = serde_json::to_vec(&token)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("encode resync cursor failed: {err}")))?;
|
||||
URL_SAFE_NO_PAD.encode(encoded)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn parse_site_resync_page(query: &HashMap<String, String>, status: &SRResyncOpStatus) -> S3Result<(usize, usize)> {
|
||||
let limit = query
|
||||
.get("limit")
|
||||
.map(|value| value.parse::<usize>())
|
||||
.transpose()
|
||||
.map_err(|_| s3_error!(InvalidRequest, "invalid resync page limit"))?
|
||||
.unwrap_or(SITE_REPL_RESYNC_DEFAULT_PAGE_SIZE);
|
||||
if limit == 0 || limit > SITE_REPL_RESYNC_MAX_PAGE_SIZE {
|
||||
return Err(s3_error!(InvalidRequest, "invalid resync page limit"));
|
||||
}
|
||||
let offset = if let Some(value) = query.get("continuationToken") {
|
||||
let decoded = URL_SAFE_NO_PAD
|
||||
.decode(value)
|
||||
.map_err(|_| s3_error!(InvalidRequest, "invalid resync continuation token"))?;
|
||||
let token: SiteResyncContinuationToken =
|
||||
serde_json::from_slice(&decoded).map_err(|_| s3_error!(InvalidRequest, "invalid resync continuation token"))?;
|
||||
if token.id != status.resync_id || token.generation != status.generation {
|
||||
return Err(s3_error!(InvalidRequest, "stale resync continuation token"));
|
||||
}
|
||||
token.offset
|
||||
} else {
|
||||
0
|
||||
};
|
||||
Ok((limit, offset))
|
||||
}
|
||||
|
||||
fn bucket_target_endpoint(target: &BucketTarget) -> String {
|
||||
@@ -4924,8 +5058,10 @@ fn bucket_target_endpoint(target: &BucketTarget) -> String {
|
||||
}
|
||||
|
||||
fn bucket_target_matches_peer(target: &BucketTarget, peer: &PeerInfo) -> bool {
|
||||
(!target.deployment_id.is_empty() && target.deployment_id == peer.deployment_id)
|
||||
|| bucket_target_endpoint(target) == canonical_endpoint(&peer.endpoint)
|
||||
if !target.deployment_id.is_empty() {
|
||||
return target.deployment_id == peer.deployment_id;
|
||||
}
|
||||
bucket_target_endpoint(target) == canonical_endpoint(&peer.endpoint)
|
||||
}
|
||||
|
||||
fn site_replication_target_arns_by_peer(config: Option<&s3s::dto::ReplicationConfiguration>) -> HashMap<String, String> {
|
||||
@@ -5576,7 +5712,12 @@ async fn backfill_existing_buckets_after_add(
|
||||
if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) {
|
||||
continue;
|
||||
}
|
||||
let result = start_site_bucket_resync(name, peer, &resync_id).await;
|
||||
let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await;
|
||||
let result = if manifest.target_arn.is_empty() {
|
||||
manifest
|
||||
} else {
|
||||
start_site_bucket_resync(name, &manifest.target_arn, &resync_id).await
|
||||
};
|
||||
if result.status == "failed" {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
@@ -5657,10 +5798,60 @@ async fn refresh_bucket_targets_after_endpoint_edit(pending_id: &str, service_ac
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn start_site_bucket_resync(bucket: &str, peer: &PeerInfo, resync_id: &str) -> ResyncBucketStatus {
|
||||
async fn site_bucket_resync_manifest_entry(bucket: &str, peer: &PeerInfo, now: OffsetDateTime) -> ResyncBucketStatus {
|
||||
let mut entry = ResyncBucketStatus {
|
||||
bucket: bucket.to_string(),
|
||||
status: "pending".to_string(),
|
||||
created_at: Some(now),
|
||||
updated_at: Some(now),
|
||||
..Default::default()
|
||||
};
|
||||
let _targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
let (config, _) = match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
entry.status = "failed".to_string();
|
||||
entry.err_detail = summarize_peer_error_detail(&err.to_string());
|
||||
return entry;
|
||||
}
|
||||
};
|
||||
let targets = match metadata_sys::list_bucket_targets(bucket).await {
|
||||
Ok(targets) => targets,
|
||||
Err(err) => {
|
||||
entry.status = "failed".to_string();
|
||||
entry.err_detail = summarize_peer_error_detail(&err.to_string());
|
||||
return entry;
|
||||
}
|
||||
};
|
||||
let mut matching = targets
|
||||
.targets
|
||||
.iter()
|
||||
.filter(|target| target.target_type == BucketTargetType::ReplicationService && bucket_target_matches_peer(target, peer));
|
||||
let Some(target) = matching.next() else {
|
||||
entry.status = "failed".to_string();
|
||||
entry.err_detail = "no valid remote target found for peer".to_string();
|
||||
return entry;
|
||||
};
|
||||
if matching.next().is_some() {
|
||||
entry.status = "failed".to_string();
|
||||
entry.err_detail = "multiple remote targets matched peer".to_string();
|
||||
return entry;
|
||||
}
|
||||
let (has_arn, existing_object_enabled) = config.has_existing_object_replication(&target.arn);
|
||||
if !has_arn || !existing_object_enabled {
|
||||
entry.status = "failed".to_string();
|
||||
entry.err_detail = "existing object replication is not enabled for the peer target".to_string();
|
||||
return entry;
|
||||
}
|
||||
entry.target_arn = target.arn.clone();
|
||||
entry
|
||||
}
|
||||
|
||||
async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &str) -> ResyncBucketStatus {
|
||||
let mut bucket_status = ResyncBucketStatus {
|
||||
bucket: bucket.to_string(),
|
||||
status: "started".to_string(),
|
||||
target_arn: target_arn.to_string(),
|
||||
status: "running".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
@@ -5683,15 +5874,40 @@ async fn start_site_bucket_resync(bucket: &str, peer: &PeerInfo, resync_id: &str
|
||||
}
|
||||
};
|
||||
|
||||
let Some(pool) = current_replication_pool_handle() else {
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = "replication pool is not initialized".to_string();
|
||||
return bucket_status;
|
||||
};
|
||||
|
||||
let Some(target_index) = targets
|
||||
.targets
|
||||
.iter()
|
||||
.position(|target| target.target_type == BucketTargetType::ReplicationService && target.arn == target_arn)
|
||||
else {
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = "recorded remote target no longer exists".to_string();
|
||||
return bucket_status;
|
||||
};
|
||||
|
||||
let existing_reset_id = targets.targets[target_index].reset_id.clone();
|
||||
if !existing_reset_id.is_empty() && existing_reset_id != resync_id {
|
||||
let existing_is_active = pool
|
||||
.get_bucket_resync_status(bucket)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|status| status.targets_map.get(target_arn).cloned())
|
||||
.is_none_or(|target| target.resync_id != existing_reset_id || site_bucket_resync_is_active(target.resync_status));
|
||||
if existing_is_active {
|
||||
bucket_status.status = "conflict".to_string();
|
||||
bucket_status.err_detail = "target belongs to a different active resync operation".to_string();
|
||||
return bucket_status;
|
||||
}
|
||||
}
|
||||
|
||||
let reset_before = Some(OffsetDateTime::now_utc());
|
||||
let target_arn = {
|
||||
let Some(target) = targets.targets.iter_mut().find(|target| {
|
||||
target.target_type == BucketTargetType::ReplicationService && bucket_target_matches_peer(target, peer)
|
||||
}) else {
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = format!("no valid remote target found for peer {}", peer.deployment_id);
|
||||
return bucket_status;
|
||||
};
|
||||
let target = &mut targets.targets[target_index];
|
||||
|
||||
let (has_arn, existing_object_enabled) = config.has_existing_object_replication(&target.arn);
|
||||
if !has_arn || !existing_object_enabled {
|
||||
@@ -5722,12 +5938,6 @@ async fn start_site_bucket_resync(bucket: &str, peer: &PeerInfo, resync_id: &str
|
||||
BucketTargetSys::get().update_all_targets(bucket, Some(&targets)).await;
|
||||
drop(targets_guard);
|
||||
|
||||
let Some(pool) = current_replication_pool_handle() else {
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = "replication pool is not initialized".to_string();
|
||||
return bucket_status;
|
||||
};
|
||||
|
||||
if let Err(err) = pool
|
||||
.start_bucket_resync(replication::resync_opts(bucket, target_arn, resync_id, reset_before))
|
||||
.await
|
||||
@@ -5739,9 +5949,10 @@ async fn start_site_bucket_resync(bucket: &str, peer: &PeerInfo, resync_id: &str
|
||||
bucket_status
|
||||
}
|
||||
|
||||
async fn cancel_site_bucket_resync(bucket: &str, peer: &PeerInfo, resync_id: &str) -> ResyncBucketStatus {
|
||||
async fn cancel_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &str) -> ResyncBucketStatus {
|
||||
let mut bucket_status = ResyncBucketStatus {
|
||||
bucket: bucket.to_string(),
|
||||
target_arn: target_arn.to_string(),
|
||||
status: "canceled".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -5757,18 +5968,32 @@ async fn cancel_site_bucket_resync(bucket: &str, peer: &PeerInfo, resync_id: &st
|
||||
};
|
||||
|
||||
let Some(target) = targets.targets.iter_mut().find(|target| {
|
||||
target.target_type == BucketTargetType::ReplicationService
|
||||
&& bucket_target_matches_peer(target, peer)
|
||||
&& target.reset_id == resync_id
|
||||
target.target_type == BucketTargetType::ReplicationService && target.arn == target_arn && target.reset_id == resync_id
|
||||
}) else {
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = format!("no in-progress resync target found for peer {}", peer.deployment_id);
|
||||
bucket_status.err_detail = "recorded resync target is not in progress".to_string();
|
||||
return bucket_status;
|
||||
};
|
||||
|
||||
let target_arn = target.arn.clone();
|
||||
|
||||
let Some(pool) = current_replication_pool_handle() else {
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = "replication pool is not initialized".to_string();
|
||||
return bucket_status;
|
||||
};
|
||||
|
||||
if let Err(err) = pool
|
||||
.cancel_bucket_resync(replication::resync_opts(bucket, target_arn, resync_id, None))
|
||||
.await
|
||||
{
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = err.to_string();
|
||||
return bucket_status;
|
||||
}
|
||||
|
||||
target.reset_id.clear();
|
||||
target.reset_before_date = None;
|
||||
let target_arn = target.arn.clone();
|
||||
|
||||
let json_targets = match serde_json::to_vec(&targets) {
|
||||
Ok(json_targets) => json_targets,
|
||||
@@ -5787,23 +6012,91 @@ async fn cancel_site_bucket_resync(bucket: &str, peer: &PeerInfo, resync_id: &st
|
||||
BucketTargetSys::get().update_all_targets(bucket, Some(&targets)).await;
|
||||
drop(targets_guard);
|
||||
|
||||
let Some(pool) = current_replication_pool_handle() else {
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = "replication pool is not initialized".to_string();
|
||||
return bucket_status;
|
||||
};
|
||||
|
||||
if let Err(err) = pool
|
||||
.cancel_bucket_resync(replication::resync_opts(bucket, target_arn, resync_id, None))
|
||||
.await
|
||||
{
|
||||
bucket_status.status = "failed".to_string();
|
||||
bucket_status.err_detail = err.to_string();
|
||||
}
|
||||
|
||||
bucket_status
|
||||
}
|
||||
|
||||
async fn refresh_site_resync_status(mut status: SRResyncOpStatus, peer: &PeerInfo) -> SRResyncOpStatus {
|
||||
for bucket in &mut status.buckets {
|
||||
if bucket.target_arn.is_empty() && matches!(bucket.status.as_str(), "pending" | "running" | "started") {
|
||||
let resolved = site_bucket_resync_manifest_entry(&bucket.bucket, peer, OffsetDateTime::now_utc()).await;
|
||||
if resolved.target_arn.is_empty() {
|
||||
bucket.status = "failed".to_string();
|
||||
bucket.err_detail = resolved.err_detail;
|
||||
} else {
|
||||
bucket.target_arn = resolved.target_arn;
|
||||
bucket.status = "pending".to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(pool) = current_replication_pool_handle() {
|
||||
for bucket in &mut status.buckets {
|
||||
if bucket.target_arn.is_empty() || bucket.status == "failed" {
|
||||
continue;
|
||||
}
|
||||
match pool.get_bucket_resync_status(&bucket.bucket).await {
|
||||
Ok(live) => match live.targets_map.get(&bucket.target_arn) {
|
||||
Some(target) if target.resync_id == status.resync_id => {
|
||||
apply_site_resync_target_status(bucket, target);
|
||||
}
|
||||
Some(target) if !target.resync_id.is_empty() && site_bucket_resync_is_active(target.resync_status) => {
|
||||
bucket.status = "conflict".to_string();
|
||||
bucket.err_detail = "recorded target belongs to a different resync operation".to_string();
|
||||
bucket.updated_at = Some(OffsetDateTime::now_utc());
|
||||
}
|
||||
Some(target) if !target.resync_id.is_empty() => {
|
||||
bucket.status = "failed".to_string();
|
||||
bucket.err_detail = "recorded resync operation was superseded by a terminal bucket resync".to_string();
|
||||
bucket.updated_at = Some(OffsetDateTime::now_utc());
|
||||
bucket.completed_at = bucket.updated_at;
|
||||
}
|
||||
_ if matches!(bucket.status.as_str(), "pending" | "running" | "started") => {
|
||||
let previous = bucket.clone();
|
||||
let mut recovered =
|
||||
start_site_bucket_resync(&previous.bucket, &previous.target_arn, &status.resync_id).await;
|
||||
recovered.created_at = previous.created_at;
|
||||
recovered.started_at = previous.started_at.or(Some(OffsetDateTime::now_utc()));
|
||||
recovered.updated_at = Some(OffsetDateTime::now_utc());
|
||||
recovered.generation = status.generation;
|
||||
recovered.err_detail = summarize_peer_error_detail(&recovered.err_detail);
|
||||
*bucket = recovered;
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
Err(err) => {
|
||||
bucket.err_detail = summarize_peer_error_detail(&err.to_string());
|
||||
bucket.updated_at = Some(OffsetDateTime::now_utc());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
summarize_site_resync_status(&mut status, OffsetDateTime::now_utc());
|
||||
status
|
||||
}
|
||||
|
||||
async fn persist_site_resync_status(peer_id: &str, status: &SRResyncOpStatus) -> S3Result<()> {
|
||||
let _state_guard = SITE_REPLICATION_STATE_LOCK.lock().await;
|
||||
let mut state = load_site_replication_state().await?;
|
||||
if state
|
||||
.resync_status
|
||||
.get(peer_id)
|
||||
.is_some_and(|current| current.resync_id != status.resync_id || current.generation != status.generation)
|
||||
{
|
||||
return Err(s3_error!(InvalidRequest, "site replication resync state changed"));
|
||||
}
|
||||
state.resync_status.insert(peer_id.to_string(), status.clone());
|
||||
save_site_replication_state(&state).await
|
||||
}
|
||||
|
||||
async fn persist_new_site_resync_status(peer_id: &str, status: &SRResyncOpStatus) -> S3Result<()> {
|
||||
let _state_guard = SITE_REPLICATION_STATE_LOCK.lock().await;
|
||||
let mut state = load_site_replication_state().await?;
|
||||
if state.resync_status.get(peer_id).is_some_and(site_resync_is_active) {
|
||||
return Err(s3_error!(InvalidRequest, "site replication resync is already active"));
|
||||
}
|
||||
state.resync_status.insert(peer_id.to_string(), status.clone());
|
||||
save_site_replication_state(&state).await
|
||||
}
|
||||
|
||||
fn apply_state_edit_req(mut state: SiteReplicationState, body: SRStateEditReq) -> SiteReplicationState {
|
||||
let Some(incoming_updated_at) = body.updated_at else {
|
||||
return state;
|
||||
@@ -7261,89 +7554,162 @@ pub struct SiteReplicationResyncOpHandler {}
|
||||
impl Operation for SiteReplicationResyncOpHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
validate_site_replication_admin_request(&req, AdminAction::SiteReplicationResyncAction).await?;
|
||||
let operation = query_pairs(&req.uri).get("operation").cloned().unwrap_or_default();
|
||||
// Resolve before the request body is consumed below; the `else` stays
|
||||
// at its original position so error precedence is unchanged.
|
||||
let query = query_pairs(&req.uri);
|
||||
let operation = query.get("operation").cloned().unwrap_or_default();
|
||||
let resolved_store = object_store_from_req(&req);
|
||||
let peer: PeerInfo = read_site_replication_json(req, "", false).await?;
|
||||
let _state_guard = SITE_REPLICATION_STATE_LOCK.lock().await;
|
||||
let mut state = load_site_replication_state().await?;
|
||||
let local_peer = current_local_runtime_peer(&state);
|
||||
let peer = normalize_peer_info(peer);
|
||||
if peer.deployment_id == local_peer.deployment_id {
|
||||
return Err(s3_error!(InvalidRequest, "invalid peer specified - cannot resync to self"));
|
||||
}
|
||||
if !state.peers.contains_key(&peer.deployment_id) {
|
||||
return Err(s3_error!(InvalidRequest, "site replication peer not found"));
|
||||
}
|
||||
let Some(store) = resolved_store else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
let requested_peer: PeerInfo = read_site_replication_json(req, "", false).await?;
|
||||
let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await;
|
||||
let (peer, existing_status) = {
|
||||
let _state_guard = SITE_REPLICATION_STATE_LOCK.lock().await;
|
||||
let state = load_site_replication_state().await?;
|
||||
let local_peer = current_local_runtime_peer(&state);
|
||||
let requested_peer = normalize_peer_info(requested_peer);
|
||||
if requested_peer.deployment_id == local_peer.deployment_id {
|
||||
return Err(s3_error!(InvalidRequest, "invalid peer specified - cannot resync to self"));
|
||||
}
|
||||
let peer = state
|
||||
.peers
|
||||
.get(&requested_peer.deployment_id)
|
||||
.cloned()
|
||||
.ok_or_else(|| s3_error!(InvalidRequest, "site replication peer not found"))?;
|
||||
(peer, state.resync_status.get(&requested_peer.deployment_id).cloned())
|
||||
};
|
||||
let buckets = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?;
|
||||
let bucket_names: Vec<String> = buckets.into_iter().map(|bucket| bucket.name).collect();
|
||||
|
||||
let status = match operation.as_str() {
|
||||
let mut status = match operation.as_str() {
|
||||
SITE_REPL_RESYNC_START => {
|
||||
let mut status = resync_status_for_state(&mut state, &operation, &peer, vec![]);
|
||||
let mut bucket_statuses = Vec::new();
|
||||
if let Some(existing) = existing_status.as_ref() {
|
||||
let existing = refresh_site_resync_status(existing.clone(), &peer).await;
|
||||
persist_site_resync_status(&peer.deployment_id, &existing).await?;
|
||||
if site_resync_is_active(&existing) {
|
||||
return Err(s3_error!(InvalidRequest, "site replication resync is already active"));
|
||||
}
|
||||
}
|
||||
let Some(store) = resolved_store else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
let mut bucket_names: Vec<String> = store
|
||||
.list_bucket(&BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?
|
||||
.into_iter()
|
||||
.map(|bucket| bucket.name)
|
||||
.collect();
|
||||
bucket_names.sort();
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut bucket_statuses = Vec::with_capacity(bucket_names.len());
|
||||
for bucket in bucket_names {
|
||||
bucket_statuses.push(start_site_bucket_resync(&bucket, &peer, &status.resync_id).await);
|
||||
bucket_statuses.push(site_bucket_resync_manifest_entry(&bucket, &peer, now).await);
|
||||
}
|
||||
let failures = bucket_statuses.iter().filter(|bucket| bucket.status == "failed").count();
|
||||
if failures == bucket_statuses.len() && !bucket_statuses.is_empty() {
|
||||
status.status = "failed".to_string();
|
||||
status.err_detail = "all buckets resync failed".to_string();
|
||||
} else if failures > 0 {
|
||||
status.err_detail = "partial failure in starting site resync".to_string();
|
||||
let mut status = SRResyncOpStatus {
|
||||
op_type: SITE_REPL_RESYNC_START.to_string(),
|
||||
resync_id: Uuid::new_v4().to_string(),
|
||||
status: "success".to_string(),
|
||||
state: "pending".to_string(),
|
||||
buckets: bucket_statuses,
|
||||
created_at: Some(now),
|
||||
started_at: Some(now),
|
||||
updated_at: Some(now),
|
||||
generation: existing_status
|
||||
.as_ref()
|
||||
.map_or(1, |existing| existing.generation.saturating_add(1).max(1)),
|
||||
..Default::default()
|
||||
};
|
||||
summarize_site_resync_status(&mut status, now);
|
||||
persist_new_site_resync_status(&peer.deployment_id, &status).await?;
|
||||
for index in 0..status.buckets.len() {
|
||||
if status.buckets[index].target_arn.is_empty() || status.buckets[index].status == "failed" {
|
||||
continue;
|
||||
}
|
||||
let previous = status.buckets[index].clone();
|
||||
let mut result = start_site_bucket_resync(&previous.bucket, &previous.target_arn, &status.resync_id).await;
|
||||
result.created_at = previous.created_at;
|
||||
result.started_at = Some(OffsetDateTime::now_utc());
|
||||
result.updated_at = result.started_at;
|
||||
result.generation = status.generation;
|
||||
result.err_detail = summarize_peer_error_detail(&result.err_detail);
|
||||
status.buckets[index] = result;
|
||||
summarize_site_resync_status(&mut status, OffsetDateTime::now_utc());
|
||||
persist_site_resync_status(&peer.deployment_id, &status).await?;
|
||||
}
|
||||
status.buckets = bucket_statuses;
|
||||
state.resync_status.insert(peer.deployment_id.clone(), status.clone());
|
||||
status = refresh_site_resync_status(status, &peer).await;
|
||||
persist_site_resync_status(&peer.deployment_id, &status).await?;
|
||||
status
|
||||
}
|
||||
SITE_REPL_RESYNC_CANCEL => {
|
||||
let Some(existing_status) = state.resync_status.get(&peer.deployment_id).cloned() else {
|
||||
let Some(existing_status) = existing_status else {
|
||||
return Err(s3_error!(InvalidRequest, "no resync in progress"));
|
||||
};
|
||||
if existing_status.resync_id.is_empty() {
|
||||
return Err(s3_error!(InvalidRequest, "no resync in progress"));
|
||||
}
|
||||
let mut status = SRResyncOpStatus {
|
||||
op_type: operation.clone(),
|
||||
resync_id: existing_status.resync_id.clone(),
|
||||
status: "success".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut bucket_statuses = Vec::new();
|
||||
for bucket in bucket_names {
|
||||
bucket_statuses.push(cancel_site_bucket_resync(&bucket, &peer, &existing_status.resync_id).await);
|
||||
let mut status = refresh_site_resync_status(existing_status, &peer).await;
|
||||
if status.buckets.iter().any(|bucket| bucket.status == "conflict") {
|
||||
return Err(s3_error!(
|
||||
InvalidRequest,
|
||||
"site replication resync target belongs to a different active operation"
|
||||
));
|
||||
}
|
||||
let failures = bucket_statuses.iter().filter(|bucket| bucket.status == "failed").count();
|
||||
if failures == bucket_statuses.len() && !bucket_statuses.is_empty() {
|
||||
status.status = "failed".to_string();
|
||||
status.err_detail = "all buckets resync cancel failed".to_string();
|
||||
} else if failures > 0 {
|
||||
status.err_detail = "partial failure in canceling site resync".to_string();
|
||||
if site_resync_cancel_is_idempotent(&status) {
|
||||
status.op_type = SITE_REPL_RESYNC_CANCEL.to_string();
|
||||
for bucket in &status.buckets {
|
||||
if !bucket.target_arn.is_empty() {
|
||||
let _ = cancel_site_bucket_resync(&bucket.bucket, &bucket.target_arn, &status.resync_id).await;
|
||||
}
|
||||
}
|
||||
status
|
||||
} else {
|
||||
if !site_resync_is_active(&status) {
|
||||
return Err(s3_error!(InvalidRequest, "no active resync to cancel"));
|
||||
}
|
||||
status.op_type = SITE_REPL_RESYNC_CANCEL.to_string();
|
||||
status.state = "canceling".to_string();
|
||||
status.updated_at = Some(OffsetDateTime::now_utc());
|
||||
persist_site_resync_status(&peer.deployment_id, &status).await?;
|
||||
for index in 0..status.buckets.len() {
|
||||
if status.buckets[index].target_arn.is_empty()
|
||||
|| matches!(status.buckets[index].status.as_str(), "failed" | "canceled")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let previous = status.buckets[index].clone();
|
||||
let mut result =
|
||||
cancel_site_bucket_resync(&previous.bucket, &previous.target_arn, &status.resync_id).await;
|
||||
result.created_at = previous.created_at;
|
||||
result.started_at = previous.started_at;
|
||||
result.updated_at = Some(OffsetDateTime::now_utc());
|
||||
result.completed_at = result.updated_at;
|
||||
result.generation = status.generation;
|
||||
result.err_detail = summarize_peer_error_detail(&result.err_detail);
|
||||
status.buckets[index] = result;
|
||||
summarize_site_resync_status(&mut status, OffsetDateTime::now_utc());
|
||||
persist_site_resync_status(&peer.deployment_id, &status).await?;
|
||||
}
|
||||
status = refresh_site_resync_status(status, &peer).await;
|
||||
persist_site_resync_status(&peer.deployment_id, &status).await?;
|
||||
status
|
||||
}
|
||||
status.buckets = bucket_statuses;
|
||||
state.resync_status.insert(peer.deployment_id.clone(), status.clone());
|
||||
status
|
||||
}
|
||||
SITE_REPL_RESYNC_STATUS => {
|
||||
let status = state
|
||||
.resync_status
|
||||
.get(&peer.deployment_id)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| SRResyncOpStatus {
|
||||
op_type: SITE_REPL_RESYNC_STATUS.to_string(),
|
||||
status: "not-found".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return json_response(&status);
|
||||
let status = existing_status.unwrap_or_else(|| SRResyncOpStatus {
|
||||
op_type: SITE_REPL_RESYNC_STATUS.to_string(),
|
||||
status: "not-found".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
if status.resync_id.is_empty() {
|
||||
status
|
||||
} else {
|
||||
let status = refresh_site_resync_status(status, &peer).await;
|
||||
persist_site_resync_status(&peer.deployment_id, &status).await?;
|
||||
status
|
||||
}
|
||||
}
|
||||
_ => return Err(s3_error!(InvalidRequest, "unsupported resync operation")),
|
||||
};
|
||||
save_site_replication_state(&state).await?;
|
||||
json_response(&status)
|
||||
status
|
||||
.buckets
|
||||
.sort_by(|left, right| left.bucket.cmp(&right.bucket).then(left.target_arn.cmp(&right.target_arn)));
|
||||
let (limit, offset) = parse_site_resync_page(&query, &status)?;
|
||||
json_response(&site_resync_page(&status, limit, offset)?)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8860,6 +9226,57 @@ mod tests {
|
||||
assert!(!policy.is_allowed(&put_policy_args).await);
|
||||
}
|
||||
|
||||
// The replication service account must be able to carry object-lock metadata to the peer.
|
||||
// Without these actions the peer answers AccessDenied for any replicated object that has
|
||||
// retention or a legal hold, so a WORM-protected object never reaches the replica at all,
|
||||
// and a retention change made after upload never propagates.
|
||||
#[tokio::test]
|
||||
async fn test_site_replicator_policy_allows_object_lock_replication() {
|
||||
let policy = site_replicator_service_account_policy().expect("site replicator policy should parse");
|
||||
let groups: Option<Vec<String>> = None;
|
||||
let claims = HashMap::new();
|
||||
let conditions = HashMap::new();
|
||||
|
||||
let base_args = rustfs_policy::policy::Args {
|
||||
account: SITE_REPLICATOR_SERVICE_ACCOUNT,
|
||||
groups: &groups,
|
||||
action: Action::S3Action(S3Action::PutObjectRetentionAction),
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
claims: &claims,
|
||||
deny_only: false,
|
||||
bucket: "photos",
|
||||
object: "image.jpg",
|
||||
};
|
||||
|
||||
for action in [
|
||||
S3Action::PutObjectRetentionAction,
|
||||
S3Action::GetObjectRetentionAction,
|
||||
S3Action::PutObjectLegalHoldAction,
|
||||
S3Action::GetObjectLegalHoldAction,
|
||||
] {
|
||||
let args = rustfs_policy::policy::Args {
|
||||
action: Action::S3Action(action),
|
||||
..base_args
|
||||
};
|
||||
assert!(
|
||||
policy.is_allowed(&args).await,
|
||||
"site replicator must be allowed to replicate object-lock metadata: {action:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// Governance bypass stays denied: replication must not be able to erase a retained
|
||||
// version on the peer.
|
||||
let bypass_args = rustfs_policy::policy::Args {
|
||||
action: Action::S3Action(S3Action::BypassGovernanceRetentionAction),
|
||||
..base_args
|
||||
};
|
||||
assert!(
|
||||
!policy.is_allowed(&bypass_args).await,
|
||||
"site replicator must not be granted governance bypass"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sr_peer_edit_handler_uses_site_replication_operation_action() {
|
||||
let src = include_str!("site_replication.rs");
|
||||
@@ -11485,4 +11902,88 @@ mod tests {
|
||||
assert!(rule_ids.contains(&"site-repl-dep-b"));
|
||||
assert!(rule_ids.contains(&"site-repl-dep-c"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn site_resync_summary_reports_partial_failure_and_clamps_counters() {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut running = ResyncBucketStatus {
|
||||
bucket: "b".to_string(),
|
||||
target_arn: "arn-b".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
apply_site_resync_target_status(
|
||||
&mut running,
|
||||
&replication::TargetReplicationResyncStatus {
|
||||
resync_status: replication::ResyncStatusType::ResyncStarted,
|
||||
resync_id: "run-1".to_string(),
|
||||
replicated_count: 4,
|
||||
replicated_size: 16,
|
||||
failed_count: -1,
|
||||
failed_size: -2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let failed = ResyncBucketStatus {
|
||||
bucket: "a".to_string(),
|
||||
target_arn: "arn-a".to_string(),
|
||||
status: "conflict".to_string(),
|
||||
err_detail: "durable failure".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut status = SRResyncOpStatus {
|
||||
op_type: SITE_REPL_RESYNC_START.to_string(),
|
||||
resync_id: "run-1".to_string(),
|
||||
buckets: vec![running, failed],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
summarize_site_resync_status(&mut status, now);
|
||||
|
||||
assert_eq!(status.status, "failed");
|
||||
assert_eq!(status.state, "running");
|
||||
assert_eq!(status.running_buckets, 1);
|
||||
assert_eq!(status.failed_buckets, 1);
|
||||
assert_eq!(status.replicated_objects, 4);
|
||||
assert_eq!(status.replicated_bytes, 16);
|
||||
assert_eq!(status.failed_objects, 0);
|
||||
assert_eq!(status.failed_bytes, 0);
|
||||
assert_eq!(status.completed_at, None);
|
||||
assert!(site_resync_is_active(&status));
|
||||
assert!(site_resync_cancel_is_idempotent(&SRResyncOpStatus {
|
||||
state: "canceled".to_string(),
|
||||
..Default::default()
|
||||
}));
|
||||
assert!(site_bucket_resync_is_active(replication::ResyncStatusType::ResyncPending));
|
||||
assert!(site_bucket_resync_is_active(replication::ResyncStatusType::ResyncStarted));
|
||||
assert!(!site_bucket_resync_is_active(replication::ResyncStatusType::ResyncCompleted));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn site_resync_pagination_is_sorted_and_rejects_stale_cursor() {
|
||||
let status = SRResyncOpStatus {
|
||||
resync_id: "run-1".to_string(),
|
||||
generation: 3,
|
||||
buckets: ["a", "b", "c"]
|
||||
.into_iter()
|
||||
.map(|bucket| ResyncBucketStatus {
|
||||
bucket: bucket.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
let first = site_resync_page(&status, 2, 0).expect("first page should be valid");
|
||||
assert!(first.truncated);
|
||||
assert_eq!(first.buckets.iter().map(|bucket| bucket.bucket.as_str()).collect::<Vec<_>>(), ["a", "b"]);
|
||||
|
||||
let query = HashMap::from([
|
||||
("limit".to_string(), "2".to_string()),
|
||||
("continuationToken".to_string(), first.next_continuation_token),
|
||||
]);
|
||||
let (_, offset) = parse_site_resync_page(&query, &status).expect("cursor should match operation");
|
||||
assert_eq!(offset, 2);
|
||||
let mut newer = status;
|
||||
newer.generation += 1;
|
||||
assert!(parse_site_resync_page(&query, &newer).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,9 @@
|
||||
|
||||
use super::{cluster_snapshot, metrics};
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::route_policy::{
|
||||
ADMIN_ROUTE_POLICY_SPECS, DEFERRED_ADMIN_ROUTE_POLICIES, DeferredAdminRoutePolicy, DeferredRoutePolicyReason,
|
||||
};
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::{
|
||||
DefaultAdminUsecase, QueryServerInfoRequest, current_endpoints_handle, default_admin_usecase, object_store_from_req,
|
||||
@@ -31,9 +34,10 @@ use matchit::Params;
|
||||
use rustfs_concurrency::WorkloadAdmissionRegistrySnapshot;
|
||||
use rustfs_madmin::{InfoMessage, StorageInfo};
|
||||
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
||||
use rustfs_security_governance::{AdminRouteSpec, HttpMethod};
|
||||
use s3s::header::CONTENT_TYPE;
|
||||
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use serde::Serialize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
@@ -54,6 +58,9 @@ const OBSERVABILITY_SUMMARY_RESOLVED: &str = "observability summary resolved fro
|
||||
const TOPOLOGY_SUMMARY_RESOLVED: &str = "topology summary resolved from capability snapshot";
|
||||
const TOPOLOGY_SNAPSHOT_NOT_AVAILABLE: &str = "endpoint topology is not available before storage endpoint pools initialize";
|
||||
pub(crate) const RUNTIME_CAPABILITIES_ROUTE_SUFFIX: &str = "/v4/runtime/capabilities";
|
||||
const SITE_REPLICATION_INFO_ROUTE: &str = "/rustfs/admin/v3/site-replication/info";
|
||||
const SITE_REPLICATION_EDIT_ROUTE: &str = "/rustfs/admin/v3/site-replication/edit";
|
||||
const SITE_REPLICATION_RESYNC_ROUTE: &str = "/rustfs/admin/v3/site-replication/resync/op";
|
||||
|
||||
macro_rules! log_system_request_rejected {
|
||||
($operation:expr, $reason:expr) => {
|
||||
@@ -609,7 +616,7 @@ impl Operation for StorageInfoHandler {
|
||||
|
||||
pub struct DataUsageInfoHandler {}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RuntimeCapabilitiesSummary {
|
||||
pub observability: CapabilityStatus,
|
||||
pub userspace_profiling: CapabilityStatus,
|
||||
@@ -617,6 +624,12 @@ pub struct RuntimeCapabilitiesSummary {
|
||||
pub platform: CapabilityStatus,
|
||||
pub topology: CapabilityStatus,
|
||||
pub cluster_snapshot: CapabilityStatus,
|
||||
#[serde(default)]
|
||||
pub site_replication_info: CapabilityStatus,
|
||||
#[serde(default)]
|
||||
pub site_replication_edit: CapabilityStatus,
|
||||
#[serde(default)]
|
||||
pub site_replication_resync: CapabilityStatus,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
@@ -729,9 +742,42 @@ fn build_runtime_capabilities_summary(
|
||||
cluster_snapshot: cluster_snapshot_summary.cloned().unwrap_or_else(|| {
|
||||
CapabilityStatus::unknown().with_reason("cluster snapshot is not available before storage endpoint pools initialize")
|
||||
}),
|
||||
site_replication_info: admin_route_capability(HttpMethod::Get, SITE_REPLICATION_INFO_ROUTE),
|
||||
site_replication_edit: admin_route_capability(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE),
|
||||
site_replication_resync: admin_route_capability(HttpMethod::Put, SITE_REPLICATION_RESYNC_ROUTE),
|
||||
}
|
||||
}
|
||||
|
||||
fn admin_route_capability(method: HttpMethod, path: &str) -> CapabilityStatus {
|
||||
admin_route_capability_from_inventory(method, path, ADMIN_ROUTE_POLICY_SPECS, DEFERRED_ADMIN_ROUTE_POLICIES)
|
||||
}
|
||||
|
||||
fn admin_route_capability_from_inventory(
|
||||
method: HttpMethod,
|
||||
path: &str,
|
||||
direct: &[AdminRouteSpec],
|
||||
deferred: &[DeferredAdminRoutePolicy],
|
||||
) -> CapabilityStatus {
|
||||
if direct.iter().any(|spec| spec.method() == method && spec.path() == path) {
|
||||
return CapabilityStatus::supported().with_reason("public admin route is registered with an implemented handler");
|
||||
}
|
||||
|
||||
match deferred
|
||||
.iter()
|
||||
.find(|policy| policy.method() == method && policy.path() == path)
|
||||
.map(|policy| policy.reason())
|
||||
{
|
||||
Some(DeferredRoutePolicyReason::NotImplemented) | None => {
|
||||
CapabilityStatus::unsupported().with_reason("public admin route is absent or not implemented")
|
||||
}
|
||||
Some(_) => CapabilityStatus::supported().with_reason("public admin route is registered with contextual authorization"),
|
||||
}
|
||||
}
|
||||
|
||||
fn runtime_capabilities_gate_actions() -> Vec<Action> {
|
||||
vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]
|
||||
}
|
||||
|
||||
fn summarize_named_capability_statuses<const N: usize>(
|
||||
statuses: [(&'static str, &CapabilityStatus); N],
|
||||
subject: &'static str,
|
||||
@@ -767,15 +813,7 @@ impl Operation for RuntimeCapabilitiesHandler {
|
||||
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
|
||||
|
||||
let remote_addr = req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0));
|
||||
validate_admin_request(
|
||||
&req.headers,
|
||||
&cred,
|
||||
owner,
|
||||
false,
|
||||
vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)],
|
||||
remote_addr,
|
||||
)
|
||||
.await?;
|
||||
validate_admin_request(&req.headers, &cred, owner, false, runtime_capabilities_gate_actions(), remote_addr).await?;
|
||||
|
||||
let response = build_runtime_capabilities_response().await.map_err(|err| {
|
||||
log_system_request_failed!("runtime_capabilities", "build_runtime_capabilities_failed", err);
|
||||
@@ -838,18 +876,26 @@ impl Operation for DataUsageInfoHandler {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
OBSERVABILITY_SUMMARY_RESOLVED, ServerInfoResponse, TOPOLOGY_SNAPSHOT_NOT_AVAILABLE, TOPOLOGY_SUMMARY_RESOLVED,
|
||||
build_runtime_capabilities_response, build_runtime_capabilities_summary, data_usage_info_gate_actions,
|
||||
system_admin_discovery,
|
||||
OBSERVABILITY_SUMMARY_RESOLVED, RuntimeCapabilitiesHandler, SITE_REPLICATION_EDIT_ROUTE, SITE_REPLICATION_INFO_ROUTE,
|
||||
SITE_REPLICATION_RESYNC_ROUTE, ServerInfoResponse, TOPOLOGY_SNAPSHOT_NOT_AVAILABLE, TOPOLOGY_SUMMARY_RESOLVED,
|
||||
admin_route_capability_from_inventory, build_runtime_capabilities_response, build_runtime_capabilities_summary,
|
||||
data_usage_info_gate_actions, runtime_capabilities_gate_actions, system_admin_discovery,
|
||||
};
|
||||
use crate::admin::router::Operation;
|
||||
use crate::admin::runtime_sources::DefaultAdminUsecase;
|
||||
use crate::admin::storage_api::cluster::{
|
||||
CapabilityState, CapabilityStatus, MemorySamplingState, ObservabilitySnapshot, PlatformSupport, TopologyCapabilities,
|
||||
TopologySnapshot, UserspaceProfilingCapability,
|
||||
};
|
||||
use http::{Extensions, HeaderMap, Uri};
|
||||
use hyper::Method;
|
||||
use matchit::Params;
|
||||
use rustfs_concurrency::WorkloadClass;
|
||||
use rustfs_madmin::{InfoMessage, StorageInfo};
|
||||
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
||||
use rustfs_security_governance::HttpMethod;
|
||||
use s3s::{Body, S3ErrorCode, S3Request};
|
||||
use serde_json::json;
|
||||
|
||||
/// Authz regression pin (rustfs/backlog#1306): datausageinfo stays an
|
||||
/// any-of gate over exactly DataUsageInfoAdminAction OR ListBucketAction.
|
||||
@@ -882,6 +928,111 @@ mod tests {
|
||||
assert_eq!(response.summary.cluster_snapshot.state, CapabilityState::Unknown);
|
||||
assert_eq!(response.observability.platform.os.as_deref(), Some(std::env::consts::OS));
|
||||
assert_eq!(response.workload_admission.entries().len(), WorkloadClass::REQUIRED.len());
|
||||
assert_eq!(response.summary.site_replication_info.state, CapabilityState::Supported);
|
||||
assert_eq!(response.summary.site_replication_edit.state, CapabilityState::Supported);
|
||||
assert_eq!(response.summary.site_replication_resync.state, CapabilityState::Supported);
|
||||
|
||||
let value = serde_json::to_value(response).expect("runtime capability response should serialize");
|
||||
assert_eq!(value["summary"]["site_replication_info"]["state"], "supported");
|
||||
assert_eq!(value["summary"]["site_replication_edit"]["state"], "supported");
|
||||
assert_eq!(value["summary"]["site_replication_resync"]["state"], "supported");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn site_replication_capabilities_follow_public_route_inventory() {
|
||||
for (method, path) in [
|
||||
(HttpMethod::Get, SITE_REPLICATION_INFO_ROUTE),
|
||||
(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE),
|
||||
(HttpMethod::Put, SITE_REPLICATION_RESYNC_ROUTE),
|
||||
] {
|
||||
assert_eq!(
|
||||
admin_route_capability_from_inventory(
|
||||
method,
|
||||
path,
|
||||
crate::admin::route_policy::ADMIN_ROUTE_POLICY_SPECS,
|
||||
crate::admin::route_policy::DEFERRED_ADMIN_ROUTE_POLICIES,
|
||||
)
|
||||
.state,
|
||||
CapabilityState::Supported,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn site_replication_capabilities_reject_absent_internal_only_and_not_implemented_routes() {
|
||||
use crate::admin::route_policy::{DeferredAdminRoutePolicy, DeferredRoutePolicyReason};
|
||||
|
||||
let not_implemented = [DeferredAdminRoutePolicy::new(
|
||||
HttpMethod::Put,
|
||||
SITE_REPLICATION_EDIT_ROUTE,
|
||||
DeferredRoutePolicyReason::NotImplemented,
|
||||
)];
|
||||
assert_eq!(
|
||||
admin_route_capability_from_inventory(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE, &[], ¬_implemented).state,
|
||||
CapabilityState::Unsupported,
|
||||
);
|
||||
assert_eq!(
|
||||
admin_route_capability_from_inventory(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE, &[], &[]).state,
|
||||
CapabilityState::Unsupported,
|
||||
);
|
||||
|
||||
let internal_only = crate::admin::route_policy::ADMIN_ROUTE_POLICY_SPECS
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|spec| spec.path() == "/rustfs/admin/v3/site-replication/peer/edit-capabilities")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
admin_route_capability_from_inventory(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE, &internal_only, &[]).state,
|
||||
CapabilityState::Unsupported,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn older_runtime_summary_defaults_site_replication_capabilities_to_unknown() {
|
||||
let summary: super::RuntimeCapabilitiesSummary = serde_json::from_value(json!({
|
||||
"observability": { "state": "supported" },
|
||||
"userspace_profiling": { "state": "supported" },
|
||||
"memory_sampling": { "state": "supported" },
|
||||
"platform": { "state": "supported" },
|
||||
"topology": { "state": "supported" },
|
||||
"cluster_snapshot": { "state": "supported" }
|
||||
}))
|
||||
.expect("older runtime summary should remain compatible");
|
||||
|
||||
assert_eq!(summary.site_replication_info.state, CapabilityState::Unknown);
|
||||
assert_eq!(summary.site_replication_edit.state, CapabilityState::Unknown);
|
||||
assert_eq!(summary.site_replication_resync.state, CapabilityState::Unknown);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_capabilities_authentication_failure_is_not_reported_as_unsupported() {
|
||||
let request = S3Request {
|
||||
input: Body::empty(),
|
||||
method: Method::GET,
|
||||
uri: Uri::from_static("/rustfs/admin/v4/runtime/capabilities"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
|
||||
let error = RuntimeCapabilitiesHandler {}
|
||||
.call(request, Params::new())
|
||||
.await
|
||||
.expect_err("runtime capabilities must reject unauthenticated requests before capability resolution");
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidRequest);
|
||||
}
|
||||
|
||||
/// Authorization denial for this exact action is pinned to AccessDenied by
|
||||
/// `crate::admin::auth::tests::non_admin_credential_is_denied`.
|
||||
#[test]
|
||||
fn runtime_capabilities_gate_requires_server_info_authorization() {
|
||||
assert_eq!(
|
||||
runtime_capabilities_gate_actions(),
|
||||
vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use futures::StreamExt;
|
||||
use futures::future::BoxFuture;
|
||||
use hashbrown::HashSet as HbHashSet;
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
@@ -22,15 +21,14 @@ use rustfs_config::{
|
||||
MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC,
|
||||
MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR,
|
||||
};
|
||||
use rustfs_targets::SharedTarget;
|
||||
use rustfs_targets::{
|
||||
BuiltinTargetAdminDescriptor, TargetAdminMetadata, TargetDomain, TargetError, TargetRequestValidator,
|
||||
check_amqp_broker_available, check_kafka_broker_available, check_mqtt_broker_available_with_tls,
|
||||
check_mysql_server_available, check_nats_server_available, check_postgres_server_available, check_pulsar_broker_available,
|
||||
check_redis_server_available,
|
||||
BuiltinTargetAdminDescriptor, SharedTarget, TargetAdminMetadata, TargetDomain, TargetError, TargetHealthReason,
|
||||
TargetHealthState, TargetRequestValidator, check_amqp_broker_available, check_kafka_broker_available,
|
||||
check_mqtt_broker_available_with_tls, check_mysql_server_available, check_nats_server_available,
|
||||
check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
|
||||
config::{
|
||||
TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, build_amqp_args, build_kafka_args, build_mysql_args,
|
||||
build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, normalize_target_plugin_instances,
|
||||
build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, try_normalize_target_plugin_instances,
|
||||
validate_redis_config,
|
||||
},
|
||||
manifest::builtin_target_manifest,
|
||||
@@ -42,11 +40,36 @@ use std::collections::{HashMap, HashSet};
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
use url::Url;
|
||||
|
||||
pub(crate) type EndpointKey = (String, String);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct RuntimeHealthStatus {
|
||||
pub status: String,
|
||||
pub state: String,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
impl RuntimeHealthStatus {
|
||||
fn disabled() -> Self {
|
||||
Self {
|
||||
status: TargetHealthState::Disabled.status().to_string(),
|
||||
state: TargetHealthState::Disabled.as_str().to_string(),
|
||||
reason: TargetHealthReason::Disabled.as_str().to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn not_loaded() -> Self {
|
||||
Self {
|
||||
status: TargetHealthState::Offline.status().to_string(),
|
||||
state: TargetHealthState::Offline.as_str().to_string(),
|
||||
reason: TargetHealthReason::NotLoadedInRuntime.as_str().to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type AdminRequestValidatorFn =
|
||||
Arc<dyn for<'a> Fn(&'a HashMap<String, String>, &'a str) -> BoxFuture<'a, S3Result<()>> + Send + Sync>;
|
||||
type DomainScopedValidatorFn = for<'a> fn(&'a HashMap<String, String>, &'a str, TargetDomain) -> BoxFuture<'a, S3Result<()>>;
|
||||
@@ -64,6 +87,8 @@ pub(crate) struct MergedTargetEndpoint {
|
||||
pub account_id: String,
|
||||
pub service: String,
|
||||
pub status: String,
|
||||
pub health_state: String,
|
||||
pub health_reason: String,
|
||||
pub source: TargetEndpointSource,
|
||||
}
|
||||
|
||||
@@ -76,6 +101,8 @@ pub(crate) struct TargetInstanceReadModel {
|
||||
pub account_id: String,
|
||||
pub service: String,
|
||||
pub status: String,
|
||||
pub health_state: String,
|
||||
pub health_reason: String,
|
||||
pub runtime_present: bool,
|
||||
pub source: TargetEndpointSource,
|
||||
pub enabled: bool,
|
||||
@@ -223,11 +250,11 @@ pub(crate) fn endpoint_source(
|
||||
config: &Config,
|
||||
target_type: &str,
|
||||
target_name: &str,
|
||||
) -> TargetEndpointSource {
|
||||
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
|
||||
) -> S3Result<TargetEndpointSource> {
|
||||
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?;
|
||||
let service = target_service_name(specs, target_type).unwrap_or_default();
|
||||
let key = normalized_endpoint_key(target_name, service);
|
||||
classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &key)
|
||||
Ok(classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &key))
|
||||
}
|
||||
|
||||
pub(crate) fn target_mutation_block_reason(
|
||||
@@ -237,8 +264,8 @@ pub(crate) fn target_mutation_block_reason(
|
||||
target_type: &str,
|
||||
target_name: &str,
|
||||
target_label: &str,
|
||||
) -> Option<String> {
|
||||
match endpoint_source(specs, route_prefix, config, target_type, target_name) {
|
||||
) -> S3Result<Option<String>> {
|
||||
Ok(match endpoint_source(specs, route_prefix, config, target_type, target_name)? {
|
||||
TargetEndpointSource::Env => Some(format!(
|
||||
"{} '{}' is managed by environment variables and cannot be modified from the console",
|
||||
target_label, target_name
|
||||
@@ -248,7 +275,7 @@ pub(crate) fn target_mutation_block_reason(
|
||||
target_label, target_name
|
||||
)),
|
||||
TargetEndpointSource::Config | TargetEndpointSource::Runtime => None,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn target_module_disabled_reason(module_name: &str, env_key: &str, enabled: bool, action: &str) -> Option<String> {
|
||||
@@ -272,43 +299,36 @@ pub(crate) fn build_json_response(
|
||||
S3Response::with_headers((status, body), header)
|
||||
}
|
||||
|
||||
pub(crate) async fn collect_runtime_statuses<E>(targets: Vec<SharedTarget<E>>) -> HashMap<EndpointKey, String>
|
||||
pub(crate) async fn collect_runtime_statuses<E>(targets: Vec<SharedTarget<E>>) -> HashMap<EndpointKey, RuntimeHealthStatus>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + serde::Serialize + serde::de::DeserializeOwned,
|
||||
E: rustfs_targets::PluginEvent,
|
||||
{
|
||||
let semaphore = Arc::new(Semaphore::new(10));
|
||||
let mut futures = futures::stream::FuturesUnordered::new();
|
||||
|
||||
for target in targets {
|
||||
let sem = Arc::clone(&semaphore);
|
||||
futures.push(async move {
|
||||
let _permit = sem.acquire().await;
|
||||
let status = match tokio::time::timeout(Duration::from_secs(3), target.is_active()).await {
|
||||
Ok(Ok(true)) => "online",
|
||||
_ => "offline",
|
||||
};
|
||||
((target.id().id, target.id().name), status.to_string())
|
||||
});
|
||||
}
|
||||
|
||||
let mut runtime_statuses = HashMap::new();
|
||||
while let Some((key, status)) = futures.next().await {
|
||||
runtime_statuses.insert(key, status);
|
||||
}
|
||||
|
||||
runtime_statuses
|
||||
rustfs_targets::health_snapshots_for_targets(targets)
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|snapshot| {
|
||||
(
|
||||
(snapshot.account_id, snapshot.target_type),
|
||||
RuntimeHealthStatus {
|
||||
status: snapshot.state.status().to_string(),
|
||||
state: snapshot.state.as_str().to_string(),
|
||||
reason: snapshot.reason.as_str().to_string(),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn merge_target_endpoints(
|
||||
specs: &[AdminTargetSpec],
|
||||
route_prefix: &str,
|
||||
config: &Config,
|
||||
runtime_statuses: HashMap<EndpointKey, String>,
|
||||
) -> Vec<MergedTargetEndpoint> {
|
||||
runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
|
||||
) -> S3Result<Vec<MergedTargetEndpoint>> {
|
||||
let mut endpoints = Vec::new();
|
||||
let mut seen = HashSet::new();
|
||||
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
|
||||
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
|
||||
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?;
|
||||
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, RuntimeHealthStatus)> = HashMap::new();
|
||||
|
||||
for ((account_id, service), status) in runtime_statuses {
|
||||
let normalized = normalized_endpoint_key(&account_id, &service);
|
||||
@@ -323,15 +343,17 @@ pub(crate) fn merge_target_endpoints(
|
||||
continue;
|
||||
}
|
||||
|
||||
let status = normalized_runtime_statuses
|
||||
let health = normalized_runtime_statuses
|
||||
.remove(&normalized)
|
||||
.map(|(_, _, status)| status)
|
||||
.unwrap_or_else(|| "offline".to_string());
|
||||
.unwrap_or_else(RuntimeHealthStatus::not_loaded);
|
||||
|
||||
endpoints.push(MergedTargetEndpoint {
|
||||
account_id: key.0,
|
||||
service: key.1,
|
||||
status,
|
||||
status: health.status,
|
||||
health_state: health.state,
|
||||
health_reason: health.reason,
|
||||
source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &normalized),
|
||||
});
|
||||
}
|
||||
@@ -341,7 +363,9 @@ pub(crate) fn merge_target_endpoints(
|
||||
endpoints.push(MergedTargetEndpoint {
|
||||
account_id,
|
||||
service,
|
||||
status,
|
||||
status: status.status,
|
||||
health_state: status.state,
|
||||
health_reason: status.reason,
|
||||
source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &normalized),
|
||||
});
|
||||
}
|
||||
@@ -356,12 +380,14 @@ pub(crate) fn merge_target_endpoints(
|
||||
account_id: key.0.clone(),
|
||||
service: key.1.clone(),
|
||||
status: "offline".to_string(),
|
||||
health_state: TargetHealthState::Offline.as_str().to_string(),
|
||||
health_reason: TargetHealthReason::NotLoadedInRuntime.as_str().to_string(),
|
||||
source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, key),
|
||||
});
|
||||
}
|
||||
|
||||
endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
|
||||
endpoints
|
||||
Ok(endpoints)
|
||||
}
|
||||
|
||||
pub(crate) fn canonical_target_instance_id(plugin_id: &str, domain: TargetDomain, instance_id: &str) -> String {
|
||||
@@ -372,13 +398,13 @@ pub(crate) fn collect_target_instances(
|
||||
specs: &[AdminTargetSpec],
|
||||
route_prefix: &str,
|
||||
config: &Config,
|
||||
runtime_statuses: HashMap<EndpointKey, String>,
|
||||
) -> Vec<TargetInstanceReadModel> {
|
||||
runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
|
||||
) -> S3Result<Vec<TargetInstanceReadModel>> {
|
||||
let mut instances = Vec::new();
|
||||
let mut seen = HashSet::new();
|
||||
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
|
||||
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, RuntimeHealthStatus)> = HashMap::new();
|
||||
let domain = inferred_target_domain(route_prefix);
|
||||
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
|
||||
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?;
|
||||
|
||||
for ((account_id, service), status) in runtime_statuses {
|
||||
let normalized = normalized_endpoint_key(&account_id, &service);
|
||||
@@ -394,10 +420,16 @@ pub(crate) fn collect_target_instances(
|
||||
}
|
||||
|
||||
let runtime_present = normalized_runtime_statuses.contains_key(&key);
|
||||
let status = normalized_runtime_statuses
|
||||
let health = normalized_runtime_statuses
|
||||
.remove(&key)
|
||||
.map(|(_, _, status)| status)
|
||||
.unwrap_or_else(|| "offline".to_string());
|
||||
.unwrap_or_else(|| {
|
||||
if instance.enabled {
|
||||
RuntimeHealthStatus::not_loaded()
|
||||
} else {
|
||||
RuntimeHealthStatus::disabled()
|
||||
}
|
||||
});
|
||||
let source = classify_endpoint_source_flags(instance_has_config_entry(&instance), instance_has_env_entry(&instance));
|
||||
|
||||
instances.push(TargetInstanceReadModel {
|
||||
@@ -407,7 +439,9 @@ pub(crate) fn collect_target_instances(
|
||||
subsystem: instance.subsystem,
|
||||
account_id: instance.instance_id,
|
||||
service: instance.target_type,
|
||||
status,
|
||||
status: health.status,
|
||||
health_state: health.state,
|
||||
health_reason: health.reason,
|
||||
runtime_present,
|
||||
source,
|
||||
enabled: instance.enabled,
|
||||
@@ -430,7 +464,9 @@ pub(crate) fn collect_target_instances(
|
||||
subsystem,
|
||||
account_id,
|
||||
service,
|
||||
status,
|
||||
status: status.status,
|
||||
health_state: status.state,
|
||||
health_reason: status.reason,
|
||||
runtime_present: true,
|
||||
source: TargetEndpointSource::Runtime,
|
||||
enabled: true,
|
||||
@@ -439,19 +475,19 @@ pub(crate) fn collect_target_instances(
|
||||
}
|
||||
|
||||
instances.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
|
||||
instances
|
||||
Ok(instances)
|
||||
}
|
||||
|
||||
pub(crate) fn find_target_instance(
|
||||
specs: &[AdminTargetSpec],
|
||||
route_prefix: &str,
|
||||
config: &Config,
|
||||
runtime_statuses: HashMap<EndpointKey, String>,
|
||||
runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
|
||||
canonical_id: &str,
|
||||
) -> Option<TargetInstanceReadModel> {
|
||||
collect_target_instances(specs, route_prefix, config, runtime_statuses)
|
||||
) -> S3Result<Option<TargetInstanceReadModel>> {
|
||||
Ok(collect_target_instances(specs, route_prefix, config, runtime_statuses)?
|
||||
.into_iter()
|
||||
.find(|instance| instance.canonical_id == canonical_id)
|
||||
.find(|instance| instance.canonical_id == canonical_id))
|
||||
}
|
||||
|
||||
pub(crate) fn allowed_target_keys(specs: &[AdminTargetSpec], target_type: &str) -> HashSet<&'static str> {
|
||||
@@ -559,11 +595,11 @@ fn normalized_target_instances(
|
||||
specs: &[AdminTargetSpec],
|
||||
route_prefix: &str,
|
||||
config: &Config,
|
||||
) -> Vec<TargetPluginInstanceRecord> {
|
||||
specs
|
||||
.iter()
|
||||
.flat_map(|spec| {
|
||||
normalize_target_plugin_instances(
|
||||
) -> S3Result<Vec<TargetPluginInstanceRecord>> {
|
||||
let mut instances = Vec::new();
|
||||
for spec in specs {
|
||||
instances.extend(
|
||||
try_normalize_target_plugin_instances(
|
||||
config,
|
||||
&TargetPluginInstanceCompatDescriptor {
|
||||
domain: inferred_target_domain(route_prefix),
|
||||
@@ -574,8 +610,10 @@ fn normalized_target_instances(
|
||||
valid_fields: spec.valid_keys,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
.map_err(|err| s3_error!(InvalidRequest, "invalid {} target environment: {}", spec.service, err))?,
|
||||
);
|
||||
}
|
||||
Ok(instances)
|
||||
}
|
||||
|
||||
fn inferred_target_domain(route_prefix: &str) -> TargetDomain {
|
||||
@@ -597,8 +635,8 @@ fn target_spec_by_service<'a>(specs: &'a [AdminTargetSpec], service: &str) -> Op
|
||||
specs.iter().find(|spec| spec.service == service)
|
||||
}
|
||||
|
||||
fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, config: &Config) -> TargetEndpointSnapshot {
|
||||
let normalized_instances = normalized_target_instances(specs, route_prefix, config);
|
||||
fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, config: &Config) -> S3Result<TargetEndpointSnapshot> {
|
||||
let normalized_instances = normalized_target_instances(specs, route_prefix, config)?;
|
||||
let mut configured_keys = Vec::new();
|
||||
let mut config_targets = HbHashSet::new();
|
||||
let mut env_targets = HbHashSet::new();
|
||||
@@ -618,12 +656,12 @@ fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, conf
|
||||
}
|
||||
}
|
||||
|
||||
TargetEndpointSnapshot {
|
||||
Ok(TargetEndpointSnapshot {
|
||||
normalized_instances,
|
||||
configured_keys,
|
||||
config_targets,
|
||||
env_targets,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn retry_with_backoff<F, Fut, T>(mut operation: F, max_attempts: usize, base_delay: Duration) -> Result<T, Error>
|
||||
|
||||
@@ -323,6 +323,8 @@ pub(crate) struct PluginInstanceEntry {
|
||||
pub account_id: String,
|
||||
pub service: String,
|
||||
pub status: String,
|
||||
pub health_state: String,
|
||||
pub health_reason: String,
|
||||
pub source: PluginInstanceSource,
|
||||
pub enabled: bool,
|
||||
pub config: HashMap<String, String>,
|
||||
@@ -482,6 +484,8 @@ mod tests {
|
||||
account_id: "primary".to_string(),
|
||||
service: "webhook".to_string(),
|
||||
status: "offline".to_string(),
|
||||
health_state: "offline".to_string(),
|
||||
health_reason: "not_loaded_in_runtime".to_string(),
|
||||
source: PluginInstanceSource::Config,
|
||||
enabled: true,
|
||||
config: HashMap::from([
|
||||
@@ -512,6 +516,8 @@ mod tests {
|
||||
"account_id": "primary",
|
||||
"service": "webhook",
|
||||
"status": "offline",
|
||||
"health_state": "offline",
|
||||
"health_reason": "not_loaded_in_runtime",
|
||||
"source": "config",
|
||||
"enabled": true,
|
||||
"config": {
|
||||
@@ -541,6 +547,8 @@ mod tests {
|
||||
account_id: "primary".to_string(),
|
||||
service: "webhook".to_string(),
|
||||
status: "offline".to_string(),
|
||||
health_state: "offline".to_string(),
|
||||
health_reason: "not_loaded_in_runtime".to_string(),
|
||||
source: PluginInstanceSource::Config,
|
||||
enabled: true,
|
||||
config: HashMap::from([("endpoint".to_string(), "https://example.com/hook".to_string())]),
|
||||
@@ -565,6 +573,8 @@ mod tests {
|
||||
"account_id": "primary",
|
||||
"service": "webhook",
|
||||
"status": "offline",
|
||||
"health_state": "offline",
|
||||
"health_reason": "not_loaded_in_runtime",
|
||||
"source": "config",
|
||||
"enabled": true,
|
||||
"config": {
|
||||
|
||||
+303
-12
@@ -65,7 +65,7 @@ use rustfs_notify::{Event as NotificationEvent, notification_system};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_signer::pre_sign_v4;
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::egress::{OutboundDnsResolver, OutboundPolicy};
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||
SUFFIX_SOURCE_VERSION_ID, get_source_scheme, insert_header,
|
||||
@@ -229,7 +229,7 @@ struct ListenNotificationFilter {
|
||||
suffix: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone)]
|
||||
struct ObjectLambdaWebhookConfig {
|
||||
endpoint: Url,
|
||||
auth_token: String,
|
||||
@@ -238,6 +238,7 @@ struct ObjectLambdaWebhookConfig {
|
||||
client_ca: String,
|
||||
skip_tls_verify: bool,
|
||||
response_header_timeout: Option<Duration>,
|
||||
outbound_resolver: OutboundDnsResolver,
|
||||
}
|
||||
|
||||
const LAMBDA_WEBHOOK_SUB_SYS: &str = "lambda_webhook";
|
||||
@@ -612,7 +613,8 @@ fn resolve_object_lambda_webhook_config_from_server_config(
|
||||
|
||||
let parsed_endpoint =
|
||||
Url::parse(&endpoint).map_err(|_| s3_error!(InvalidRequest, "object lambda target endpoint is invalid"))?;
|
||||
validate_outbound_url(&parsed_endpoint)
|
||||
let outbound_resolver = outbound_policy()?
|
||||
.resolver_for(&parsed_endpoint)
|
||||
.map_err(|err| s3_error!(InvalidRequest, "object lambda target endpoint is not allowed: {}", err))?;
|
||||
|
||||
Ok(ObjectLambdaWebhookConfig {
|
||||
@@ -623,12 +625,13 @@ fn resolve_object_lambda_webhook_config_from_server_config(
|
||||
client_ca: kvs.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(),
|
||||
skip_tls_verify: config_enable_is_on(&kvs.lookup(WEBHOOK_SKIP_TLS_VERIFY).unwrap_or_default()),
|
||||
response_header_timeout,
|
||||
outbound_resolver,
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_current_server_config() -> S3Result<Config> {
|
||||
if let Some(system) = notification_system() {
|
||||
return Ok(system.config.read().await.clone());
|
||||
return Ok(system.config_snapshot().await);
|
||||
}
|
||||
|
||||
if let Some(store) = current_object_store_handle() {
|
||||
@@ -658,9 +661,18 @@ async fn resolve_object_lambda_webhook_config(uri: &Uri) -> S3Result<ObjectLambd
|
||||
}
|
||||
|
||||
fn build_object_lambda_http_client(config: &ObjectLambdaWebhookConfig) -> S3Result<reqwest::Client> {
|
||||
validate_outbound_url(&config.endpoint)
|
||||
.map_err(|err| s3_error!(InvalidRequest, "object lambda target endpoint is not allowed: {}", err))?;
|
||||
let mut builder = reqwest::Client::builder().user_agent(rustfs_targets::get_user_agent(rustfs_targets::ServiceType::Basis));
|
||||
build_object_lambda_http_client_with_resolver(config, config.outbound_resolver.clone())
|
||||
}
|
||||
|
||||
fn build_object_lambda_http_client_with_resolver(
|
||||
config: &ObjectLambdaWebhookConfig,
|
||||
resolver: impl reqwest::dns::Resolve + 'static,
|
||||
) -> S3Result<reqwest::Client> {
|
||||
let mut builder = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.dns_resolver(resolver)
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.user_agent(rustfs_targets::get_user_agent(rustfs_targets::ServiceType::Basis));
|
||||
|
||||
if let Some(timeout) = config.response_header_timeout {
|
||||
builder = builder.timeout(timeout);
|
||||
@@ -672,7 +684,7 @@ fn build_object_lambda_http_client(config: &ObjectLambdaWebhookConfig) -> S3Resu
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
|
||||
result = "tls_verification_disabled",
|
||||
endpoint = %config.endpoint,
|
||||
endpoint_origin = %config.endpoint.origin().ascii_serialization(),
|
||||
"admin router state"
|
||||
);
|
||||
builder = builder.danger_accept_invalid_certs(true);
|
||||
@@ -706,6 +718,17 @@ fn build_object_lambda_http_client(config: &ObjectLambdaWebhookConfig) -> S3Resu
|
||||
.map_err(|e| s3_error!(InternalError, "failed to build object lambda http client: {e}"))
|
||||
}
|
||||
|
||||
async fn send_object_lambda_request(request: reqwest::RequestBuilder) -> S3Result<reqwest::Response> {
|
||||
request
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| s3_error!(InternalError, "object lambda target request failed"))
|
||||
}
|
||||
|
||||
fn outbound_policy() -> S3Result<&'static OutboundPolicy> {
|
||||
OutboundPolicy::from_env_cached().map_err(|err| s3_error!(InvalidRequest, "invalid outbound policy: {}", err))
|
||||
}
|
||||
|
||||
fn extract_request_scheme(headers: &HeaderMap, uri: &Uri) -> String {
|
||||
get_source_scheme(headers)
|
||||
.and_then(|value| {
|
||||
@@ -1098,10 +1121,7 @@ async fn invoke_object_lambda_target(
|
||||
request_builder = request_builder.header("x-rustfs-object-lambda-version-id", version_id);
|
||||
}
|
||||
|
||||
let lambda_response = request_builder
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "object lambda target request failed: {e}"))?;
|
||||
let lambda_response = send_object_lambda_request(request_builder).await?;
|
||||
|
||||
let status = lambda_response.status();
|
||||
let lambda_headers = lambda_response.headers().clone();
|
||||
@@ -2576,7 +2596,70 @@ mod tests {
|
||||
use http::Method;
|
||||
use http::Uri;
|
||||
use s3s::S3Request;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use time::macros::datetime;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct StaticResolver(IpAddr);
|
||||
|
||||
impl reqwest::dns::Resolve for StaticResolver {
|
||||
fn resolve(&self, _name: reqwest::dns::Name) -> reqwest::dns::Resolving {
|
||||
let address = SocketAddr::new(self.0, 0);
|
||||
Box::pin(async move { Ok(Box::new(std::iter::once(address)) as reqwest::dns::Addrs) })
|
||||
}
|
||||
}
|
||||
|
||||
fn object_lambda_test_config(endpoint: Url) -> ObjectLambdaWebhookConfig {
|
||||
object_lambda_test_config_with_policy(endpoint, &OutboundPolicy::default())
|
||||
}
|
||||
|
||||
fn object_lambda_test_config_with_policy(endpoint: Url, policy: &OutboundPolicy) -> ObjectLambdaWebhookConfig {
|
||||
let outbound_resolver = policy
|
||||
.resolver_for(&endpoint)
|
||||
.expect("test endpoint should satisfy its outbound policy");
|
||||
ObjectLambdaWebhookConfig {
|
||||
endpoint,
|
||||
auth_token: String::new(),
|
||||
client_cert: String::new(),
|
||||
client_key: String::new(),
|
||||
client_ca: String::new(),
|
||||
skip_tls_verify: false,
|
||||
response_header_timeout: Some(Duration::from_secs(2)),
|
||||
outbound_resolver,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLog(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
|
||||
|
||||
struct CapturedLogWriter(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
|
||||
|
||||
impl std::io::Write for CapturedLogWriter {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
self.0.lock().expect("captured log lock").extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'writer> tracing_subscriber::fmt::MakeWriter<'writer> for CapturedLog {
|
||||
type Writer = CapturedLogWriter;
|
||||
|
||||
fn make_writer(&'writer self) -> Self::Writer {
|
||||
CapturedLogWriter(self.0.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl CapturedLog {
|
||||
fn contents(&self) -> String {
|
||||
String::from_utf8(self.0.lock().expect("captured log lock").clone()).expect("captured logs must be UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalize_admin_path_maps_compat_prefix_to_rustfs_prefix() {
|
||||
@@ -3778,6 +3861,214 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_lambda_client_ignores_proxies_and_does_not_follow_redirects() {
|
||||
const CHILD_ENV: &str = "RUSTFS_TEST_OBJECT_LAMBDA_PROXY_CHILD";
|
||||
const TARGET_URL_ENV: &str = "RUSTFS_TEST_OBJECT_LAMBDA_TARGET_URL";
|
||||
const TARGET_ADDR_ENV: &str = "RUSTFS_TEST_OBJECT_LAMBDA_TARGET_ADDR";
|
||||
|
||||
if std::env::var_os(CHILD_ENV).is_some() {
|
||||
let endpoint = Url::parse(&std::env::var(TARGET_URL_ENV).expect("child target URL")).expect("target URL");
|
||||
let address = std::env::var(TARGET_ADDR_ENV)
|
||||
.expect("child target address")
|
||||
.parse::<SocketAddr>()
|
||||
.expect("target address");
|
||||
let config = object_lambda_test_config(endpoint);
|
||||
let client = build_object_lambda_http_client_with_resolver(&config, StaticResolver(address.ip()))
|
||||
.expect("object lambda client should build");
|
||||
tokio::runtime::Runtime::new().expect("child runtime").block_on(async {
|
||||
let response = client
|
||||
.get(config.endpoint)
|
||||
.send()
|
||||
.await
|
||||
.expect("object lambda request should bypass environment proxy");
|
||||
assert_eq!(response.status(), reqwest::StatusCode::FOUND);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener as StdTcpListener;
|
||||
|
||||
let target_listener = StdTcpListener::bind("127.0.0.1:0").expect("bind object lambda target");
|
||||
let target_address = target_listener.local_addr().expect("object lambda target address");
|
||||
let redirect_listener = StdTcpListener::bind("127.0.0.1:0").expect("bind redirect destination");
|
||||
let redirect_address = redirect_listener.local_addr().expect("redirect destination address");
|
||||
drop(redirect_listener);
|
||||
let target = std::thread::spawn(move || {
|
||||
let (mut stream, _) = target_listener.accept().expect("accept object lambda request");
|
||||
let mut request = [0_u8; 1024];
|
||||
let read = stream.read(&mut request).expect("read object lambda request");
|
||||
assert!(String::from_utf8_lossy(&request[..read]).starts_with("GET /transform HTTP/1.1"));
|
||||
stream
|
||||
.write_all(
|
||||
format!(
|
||||
"HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:{}/metadata\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
|
||||
redirect_address.port()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.expect("write object lambda redirect");
|
||||
});
|
||||
let proxy_listener = StdTcpListener::bind("127.0.0.1:0").expect("reserve refused proxy address");
|
||||
let proxy_address = proxy_listener.local_addr().expect("proxy address");
|
||||
drop(proxy_listener);
|
||||
let target_url = format!("http://object-lambda.test:{}/transform", target_address.port());
|
||||
let proxy_url = format!("http://{proxy_address}");
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("resolve current test executable"))
|
||||
.arg("object_lambda_client_ignores_proxies_and_does_not_follow_redirects")
|
||||
.arg("--nocapture")
|
||||
.env(CHILD_ENV, "1")
|
||||
.env(TARGET_URL_ENV, target_url)
|
||||
.env(TARGET_ADDR_ENV, target_address.to_string())
|
||||
.env("HTTP_PROXY", &proxy_url)
|
||||
.env("HTTPS_PROXY", &proxy_url)
|
||||
.env("ALL_PROXY", &proxy_url)
|
||||
.env("NO_PROXY", "")
|
||||
.output()
|
||||
.expect("run isolated object lambda proxy test child");
|
||||
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"proxy test child failed: stdout={} stderr={}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
target.join().expect("object lambda target should finish");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_lambda_client_uses_configured_outbound_policy_resolver() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind object lambda policy target");
|
||||
let address = listener.local_addr().expect("object lambda policy target address");
|
||||
let endpoint = Url::parse(&format!("http://{address}/transform")).expect("object lambda endpoint");
|
||||
let policy = OutboundPolicy::from_allowed_origins(&endpoint.origin().ascii_serialization())
|
||||
.expect("loopback test origin should be explicitly allowed");
|
||||
let config = object_lambda_test_config_with_policy(endpoint.clone(), &policy);
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("accept object lambda policy request");
|
||||
let mut request = [0_u8; 1024];
|
||||
let read = stream.read(&mut request).await.expect("read object lambda policy request");
|
||||
assert!(String::from_utf8_lossy(&request[..read]).starts_with("GET /transform HTTP/1.1"));
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("write object lambda policy response");
|
||||
});
|
||||
|
||||
let response = build_object_lambda_http_client(&config)
|
||||
.expect("object lambda policy client should build")
|
||||
.get(endpoint)
|
||||
.send()
|
||||
.await
|
||||
.expect("configured policy resolver should reach the allowed origin");
|
||||
assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT);
|
||||
server.await.expect("object lambda policy target should finish");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_lambda_client_preserves_hostname_for_tls_sni() {
|
||||
use rustls::{
|
||||
ServerConfig, ServerConnection, StreamOwned,
|
||||
pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
|
||||
};
|
||||
use std::io::{Read, Write};
|
||||
use std::sync::{Arc, Once};
|
||||
|
||||
static INSTALL_CRYPTO_PROVIDER: Once = Once::new();
|
||||
INSTALL_CRYPTO_PROVIDER.call_once(|| {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
});
|
||||
let rcgen::CertifiedKey { cert, signing_key } =
|
||||
rcgen::generate_simple_self_signed(vec!["object-lambda.test".to_string()]).expect("cert should generate");
|
||||
let temp_dir = tempfile::tempdir().expect("create temp directory");
|
||||
let ca_path = temp_dir.path().join("object-lambda-ca.pem");
|
||||
std::fs::write(&ca_path, cert.pem()).expect("write object lambda CA");
|
||||
let server_config = Arc::new(
|
||||
ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_single_cert(
|
||||
vec![cert.der().clone()],
|
||||
PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(signing_key.serialize_der())),
|
||||
)
|
||||
.expect("server certificate should be valid"),
|
||||
);
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind object lambda TLS server");
|
||||
let address = listener.local_addr().expect("object lambda TLS address");
|
||||
let server = std::thread::spawn(move || {
|
||||
let (stream, _) = listener.accept().expect("accept object lambda TLS request");
|
||||
let connection = ServerConnection::new(server_config).expect("server TLS connection");
|
||||
let mut stream = StreamOwned::new(connection, stream);
|
||||
let mut request = [0_u8; 1024];
|
||||
let _ = stream.read(&mut request).expect("read object lambda TLS request");
|
||||
assert_eq!(stream.conn.server_name(), Some("object-lambda.test"));
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.expect("write object lambda TLS response");
|
||||
});
|
||||
let endpoint =
|
||||
Url::parse(&format!("https://object-lambda.test:{}/transform", address.port())).expect("object lambda TLS endpoint");
|
||||
let mut config = object_lambda_test_config(endpoint.clone());
|
||||
config.client_ca = ca_path.to_string_lossy().into_owned();
|
||||
|
||||
let response = build_object_lambda_http_client_with_resolver(&config, StaticResolver(address.ip()))
|
||||
.expect("object lambda TLS client should build")
|
||||
.get(endpoint)
|
||||
.send()
|
||||
.await
|
||||
.expect("TLS request should keep the configured hostname for SNI");
|
||||
assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT);
|
||||
server.join().expect("object lambda TLS server should finish");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_lambda_tls_warning_redacts_endpoint_details() {
|
||||
let captured = CapturedLog::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
.with_writer(captured.clone())
|
||||
.finish();
|
||||
let mut config = object_lambda_test_config(
|
||||
Url::parse("https://object-lambda.test/private?token=secret").expect("object lambda endpoint"),
|
||||
);
|
||||
config.skip_tls_verify = true;
|
||||
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
build_object_lambda_http_client_with_resolver(&config, StaticResolver(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)))
|
||||
.expect("object lambda client should build");
|
||||
});
|
||||
|
||||
let logs = captured.contents();
|
||||
assert!(logs.contains("https://object-lambda.test"));
|
||||
for secret in ["/private", "token=secret"] {
|
||||
assert!(!logs.contains(secret), "TLS warning leaked {secret}: {logs}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_lambda_request_error_does_not_expose_endpoint_details() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("reserve refused endpoint");
|
||||
let address = listener.local_addr().expect("refused endpoint address");
|
||||
drop(listener);
|
||||
let request = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("test client should build")
|
||||
.get(format!("http://{address}/private?token=secret"));
|
||||
|
||||
let error = send_object_lambda_request(request)
|
||||
.await
|
||||
.expect_err("refused object lambda request should fail");
|
||||
let rendered = error.to_string();
|
||||
assert!(rendered.contains("object lambda target request failed"));
|
||||
assert!(!rendered.contains("/private"));
|
||||
assert!(!rendered.contains("token=secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_object_lambda_get_request_removes_lambda_arn_and_preserves_request_inputs() {
|
||||
let mut req = S3Request {
|
||||
|
||||
@@ -19,18 +19,21 @@ use crate::admin::runtime_sources::{
|
||||
use crate::admin::storage_api::config::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate, storageclass};
|
||||
use crate::admin::storage_api::contract::admin::StorageAdminApi;
|
||||
use crate::admin::storage_api::runtime::ECStore;
|
||||
use crate::server::{
|
||||
MODULE_SWITCHES_SIGNAL_SUBSYSTEM, apply_audit_module_switch_for_context, reconcile_event_notifier_from_store,
|
||||
};
|
||||
use rustfs_audit::reload_audit_config;
|
||||
use rustfs_config::AUDIT_DEFAULT_DIR;
|
||||
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::notify::{NOTIFY_ROUTE_PREFIX, NOTIFY_SUB_SYSTEMS};
|
||||
use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
|
||||
use rustfs_config::server_config::{Config as ServerConfig, KVS};
|
||||
use rustfs_config::{AUDIT_DEFAULT_DIR, EVENT_DEFAULT_DIR};
|
||||
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
|
||||
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
|
||||
use rustfs_iam::oidc::load_oidc_provider_configs_from_server_config;
|
||||
use rustfs_targets::config::{
|
||||
validate_amqp_config, validate_kafka_config, validate_mqtt_config, validate_mysql_config, validate_nats_config,
|
||||
validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
|
||||
try_collect_target_configs, validate_amqp_config, validate_kafka_config, validate_mqtt_config, validate_mysql_config,
|
||||
validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
|
||||
};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
use std::future::Future;
|
||||
@@ -38,10 +41,11 @@ use tracing::warn;
|
||||
use url::Url;
|
||||
|
||||
pub fn is_dynamic_config_subsystem(sub_system: &str) -> bool {
|
||||
matches!(
|
||||
sub_system,
|
||||
STORAGE_CLASS_SUB_SYS | AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS | SCANNER_SUB_SYS | HEAL_SUB_SYS
|
||||
)
|
||||
NOTIFY_SUB_SYSTEMS.contains(&sub_system)
|
||||
|| matches!(
|
||||
sub_system,
|
||||
STORAGE_CLASS_SUB_SYS | AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS | SCANNER_SUB_SYS | HEAL_SUB_SYS
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) const FULL_CONFIG_WORKER_SUBSYSTEMS: [&str; 2] = [AUDIT_WEBHOOK_SUB_SYS, SCANNER_SUB_SYS];
|
||||
@@ -179,30 +183,20 @@ fn target_kvs(config: &ServerConfig, sub_system: &str, target: &str) -> KVS {
|
||||
}
|
||||
|
||||
fn validate_notify_subsystem_config(config: &ServerConfig, sub_system: &str) -> S3Result<()> {
|
||||
let Some(targets) = config.0.get(sub_system) else {
|
||||
let Some(descriptor) = rustfs_notify::factory::builtin_target_descriptors()
|
||||
.into_iter()
|
||||
.find(|descriptor| descriptor.subsystem() == sub_system)
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let plugin = descriptor.plugin();
|
||||
|
||||
for target in targets.keys() {
|
||||
let kvs = target_kvs(config, sub_system, target);
|
||||
if !target_enabled(&kvs) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let result = match sub_system {
|
||||
"notify_webhook" => validate_webhook_config(&kvs, EVENT_DEFAULT_DIR),
|
||||
"notify_amqp" => validate_amqp_config(&kvs, EVENT_DEFAULT_DIR),
|
||||
"notify_kafka" => validate_kafka_config(&kvs, EVENT_DEFAULT_DIR),
|
||||
"notify_mqtt" => validate_mqtt_config(&kvs),
|
||||
"notify_mysql" => validate_mysql_config(&kvs, EVENT_DEFAULT_DIR),
|
||||
"notify_nats" => validate_nats_config(&kvs, EVENT_DEFAULT_DIR),
|
||||
"notify_postgres" => validate_postgres_config(&kvs, EVENT_DEFAULT_DIR),
|
||||
"notify_pulsar" => validate_pulsar_config(&kvs, EVENT_DEFAULT_DIR),
|
||||
"notify_redis" => validate_redis_config(&kvs, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL),
|
||||
_ => return Ok(()),
|
||||
};
|
||||
|
||||
result.map_err(|err| invalid_request(format!("invalid {sub_system} config for target '{target}': {err}")))?;
|
||||
for (target, kvs) in try_collect_target_configs(config, NOTIFY_ROUTE_PREFIX, plugin.target_type(), plugin.valid_fields_set())
|
||||
.map_err(|err| invalid_request(format!("invalid {sub_system} config: {err}")))?
|
||||
{
|
||||
plugin
|
||||
.validate_config(&kvs)
|
||||
.map_err(|err| invalid_request(format!("invalid {sub_system} config for target '{target}': {err}")))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -314,8 +308,7 @@ pub(crate) async fn prepare_server_config_for_context(
|
||||
Some(STORAGE_CLASS_SUB_SYS) => {
|
||||
prepared.storage_class = Some(prepare_storage_class_runtime_config_for_context(context, config).await?);
|
||||
}
|
||||
Some(NOTIFY_WEBHOOK_SUB_SYS) => validate_notify_subsystem_config(config, NOTIFY_WEBHOOK_SUB_SYS)?,
|
||||
Some(NOTIFY_MQTT_SUB_SYS) => validate_notify_subsystem_config(config, NOTIFY_MQTT_SUB_SYS)?,
|
||||
Some(sub_system) if NOTIFY_SUB_SYSTEMS.contains(&sub_system) => validate_notify_subsystem_config(config, sub_system)?,
|
||||
Some(AUDIT_WEBHOOK_SUB_SYS) => validate_audit_subsystem_config(config, AUDIT_WEBHOOK_SUB_SYS)?,
|
||||
Some(AUDIT_MQTT_SUB_SYS) => validate_audit_subsystem_config(config, AUDIT_MQTT_SUB_SYS)?,
|
||||
Some(IDENTITY_OPENID_SUB_SYS) => validate_identity_openid_config(config)?,
|
||||
@@ -324,8 +317,9 @@ pub(crate) async fn prepare_server_config_for_context(
|
||||
Some(_) => {}
|
||||
None => {
|
||||
prepared.storage_class = Some(prepare_storage_class_runtime_config_for_context(context, config).await?);
|
||||
validate_notify_subsystem_config(config, NOTIFY_WEBHOOK_SUB_SYS)?;
|
||||
validate_notify_subsystem_config(config, NOTIFY_MQTT_SUB_SYS)?;
|
||||
for sub_system in NOTIFY_SUB_SYSTEMS {
|
||||
validate_notify_subsystem_config(config, sub_system)?;
|
||||
}
|
||||
validate_audit_subsystem_config(config, AUDIT_WEBHOOK_SUB_SYS)?;
|
||||
validate_audit_subsystem_config(config, AUDIT_MQTT_SUB_SYS)?;
|
||||
validate_identity_openid_config(config)?;
|
||||
@@ -371,6 +365,18 @@ pub async fn apply_dynamic_config_for_subsystem_for_context(
|
||||
AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS => reload_audit_config(config.clone())
|
||||
.await
|
||||
.map_err(|err| internal_error(format!("failed to reload audit config: {err}")))?,
|
||||
sub_system if NOTIFY_SUB_SYSTEMS.contains(&sub_system) => {
|
||||
// The notify lifecycle is intentionally process-wide. The explicit
|
||||
// store keeps the persisted snapshot bound to the request context;
|
||||
// the ECStore NotificationSys resolved below is the peer-broadcast
|
||||
// service, not the notify target runtime.
|
||||
let system = rustfs_notify::ensure_live_events();
|
||||
let store = resolve_runtime_config_store_for_context(context)?;
|
||||
system
|
||||
.reload_persisted_config_from_store(store)
|
||||
.await
|
||||
.map_err(|err| internal_error(format!("failed to reload notification config: {err}")))?;
|
||||
}
|
||||
SCANNER_SUB_SYS => rustfs_scanner::apply_scanner_runtime_config(config)
|
||||
.map_err(|err| internal_error(format!("failed to reload scanner config: {err}")))?,
|
||||
HEAL_SUB_SYS => rustfs_scanner::apply_scanner_runtime_config(config)
|
||||
@@ -387,6 +393,18 @@ pub async fn apply_dynamic_config_for_subsystem(config: &ServerConfig, sub_syste
|
||||
}
|
||||
|
||||
pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
|
||||
if sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM {
|
||||
let store = resolve_runtime_config_store_for_context(context)?;
|
||||
let notify_result = reconcile_event_notifier_from_store(store).await;
|
||||
let audit_result = apply_audit_module_switch_for_context(context).await;
|
||||
return match (notify_result, audit_result) {
|
||||
(Ok(()), Ok(())) => Ok(()),
|
||||
(Err(_), Ok(())) => Err(internal_error("failed to reconcile notification module switch")),
|
||||
(Ok(()), Err(_)) => Err(internal_error("failed to reconcile audit module switch")),
|
||||
(Err(_), Err(_)) => Err(internal_error("failed to reconcile notification and audit module switches")),
|
||||
};
|
||||
}
|
||||
|
||||
if !is_dynamic_config_subsystem(sub_system) {
|
||||
return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}")));
|
||||
}
|
||||
@@ -398,9 +416,8 @@ pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&Ap
|
||||
})?;
|
||||
apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
warn!("peer reload_dynamic_config: failed to apply {sub_system}: {err}");
|
||||
err
|
||||
.inspect_err(|_| {
|
||||
warn!(config_subsystem = sub_system, reason = "apply_failed", "Peer dynamic config apply failed");
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -428,19 +445,19 @@ where
|
||||
let (config, prepared) = prepare(config).await?;
|
||||
publish(&config, prepared)?;
|
||||
|
||||
// Worker reloads mutate live state and have no rollback contract. They are
|
||||
// therefore best-effort after the validated storage/server snapshots are
|
||||
// published; a transient worker failure must not leave this peer on stale
|
||||
// erasure geometry.
|
||||
// Worker reloads mutate live state and have no rollback contract, so the
|
||||
// validated snapshots stay published. The RPC still reports convergence
|
||||
// failure explicitly so the originating Admin request cannot claim success.
|
||||
if let Err(err) = apply_workers(config).await {
|
||||
warn!(
|
||||
event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
|
||||
error = ?err,
|
||||
reason = "apply_failed",
|
||||
"Runtime config snapshot was published but a worker reload failed"
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -468,20 +485,29 @@ pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppCont
|
||||
Ok(())
|
||||
},
|
||||
|config| async move {
|
||||
let mut failures = Vec::new();
|
||||
for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
|
||||
if let Err(err) = apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system).await {
|
||||
if apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
failures.push(sub_system);
|
||||
warn!(
|
||||
event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
config_subsystem = sub_system,
|
||||
state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
|
||||
error = ?err,
|
||||
reason = "apply_failed",
|
||||
"Peer runtime config snapshot was published but a subsystem worker reload failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
if failures.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(internal_error(format!("runtime worker reload failed: {}", failures.join("; "))))
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -492,20 +518,77 @@ pub async fn reload_runtime_config_snapshot() -> S3Result<()> {
|
||||
reload_runtime_config_snapshot_for_context(context.as_deref()).await
|
||||
}
|
||||
|
||||
pub async fn signal_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) {
|
||||
if !is_dynamic_config_subsystem(sub_system) {
|
||||
return;
|
||||
pub async fn preflight_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
|
||||
if sub_system != MODULE_SWITCHES_SIGNAL_SUBSYSTEM && !is_dynamic_config_subsystem(sub_system) {
|
||||
return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}")));
|
||||
}
|
||||
|
||||
let Some(notification_sys) = current_notification_system_for_context(context) else {
|
||||
return;
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
for failure in notification_sys.reload_dynamic_config(sub_system).await {
|
||||
if let Some(err) = failure.err {
|
||||
tracing::warn!("peer {} dynamic config reload for {} failed: {}", failure.host, sub_system, err);
|
||||
let mut failed = 0usize;
|
||||
for failure in notification_sys.preflight_dynamic_config(sub_system).await {
|
||||
if failure.err.is_some() {
|
||||
failed += 1;
|
||||
let host = if failure.host.is_empty() { "<unknown>" } else { &failure.host };
|
||||
warn!(
|
||||
peer = host,
|
||||
config_subsystem = sub_system,
|
||||
reason = "unsupported",
|
||||
"Peer does not support dynamic config convergence"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if failed == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(internal_error(format!(
|
||||
"{failed} peer(s) do not support dynamic config convergence for {sub_system}"
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn preflight_dynamic_config_reload(sub_system: &str) -> S3Result<()> {
|
||||
let context = current_app_context();
|
||||
preflight_dynamic_config_reload_for_context(context.as_deref(), sub_system).await
|
||||
}
|
||||
|
||||
pub async fn signal_dynamic_config_reload_checked_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
|
||||
if sub_system != MODULE_SWITCHES_SIGNAL_SUBSYSTEM && !is_dynamic_config_subsystem(sub_system) {
|
||||
return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}")));
|
||||
}
|
||||
|
||||
let Some(notification_sys) = current_notification_system_for_context(context) else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut failed = 0usize;
|
||||
for failure in notification_sys.reload_dynamic_config(sub_system).await {
|
||||
if failure.err.is_some() {
|
||||
failed += 1;
|
||||
let host = if failure.host.is_empty() { "<unknown>" } else { &failure.host };
|
||||
warn!(
|
||||
peer = host,
|
||||
config_subsystem = sub_system,
|
||||
reason = "reload_failed",
|
||||
"Peer dynamic config convergence failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if failed == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(internal_error(format!("{failed} peer(s) failed dynamic config reload for {sub_system}")))
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn signal_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) {
|
||||
if let Err(err) = signal_dynamic_config_reload_checked_for_context(context, sub_system).await {
|
||||
tracing::warn!("peer dynamic config reload for {} failed: {}", sub_system, err);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn signal_dynamic_config_reload(sub_system: &str) {
|
||||
@@ -513,16 +596,36 @@ pub async fn signal_dynamic_config_reload(sub_system: &str) {
|
||||
signal_dynamic_config_reload_for_context(context.as_deref(), sub_system).await;
|
||||
}
|
||||
|
||||
pub async fn signal_config_snapshot_reload_for_context(context: Option<&AppContext>) {
|
||||
pub async fn signal_dynamic_config_reload_checked(sub_system: &str) -> S3Result<()> {
|
||||
let context = current_app_context();
|
||||
signal_dynamic_config_reload_checked_for_context(context.as_deref(), sub_system).await
|
||||
}
|
||||
|
||||
pub async fn signal_config_snapshot_reload_checked_for_context(context: Option<&AppContext>) -> S3Result<()> {
|
||||
let Some(notification_sys) = current_notification_system_for_context(context) else {
|
||||
return;
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut failed = 0usize;
|
||||
for failure in notification_sys.refresh_config_snapshot().await {
|
||||
if let Some(err) = failure.err {
|
||||
tracing::warn!("peer config snapshot refresh failed for {}: {}", failure.host, err);
|
||||
if failure.err.is_some() {
|
||||
failed += 1;
|
||||
let host = if failure.host.is_empty() { "<unknown>" } else { &failure.host };
|
||||
warn!(peer = host, reason = "reload_failed", "Peer config snapshot refresh failed");
|
||||
}
|
||||
}
|
||||
|
||||
if failed == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(internal_error(format!("{failed} peer(s) failed config snapshot reload")))
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn signal_config_snapshot_reload_for_context(context: Option<&AppContext>) {
|
||||
if let Err(err) = signal_config_snapshot_reload_checked_for_context(context).await {
|
||||
tracing::warn!("peer config snapshot refresh failed: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn signal_config_snapshot_reload() {
|
||||
@@ -530,30 +633,46 @@ pub async fn signal_config_snapshot_reload() {
|
||||
signal_config_snapshot_reload_for_context(context.as_deref()).await;
|
||||
}
|
||||
|
||||
pub async fn signal_config_snapshot_reload_checked() -> S3Result<()> {
|
||||
let context = current_app_context();
|
||||
signal_config_snapshot_reload_checked_for_context(context.as_deref()).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::admin::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
|
||||
use crate::admin::storage_api::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
|
||||
use crate::admin::storage_api::config::save_admin_server_config;
|
||||
use crate::admin::storage_api::config::{
|
||||
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config,
|
||||
save_admin_server_config_no_lock, with_admin_server_config_write_lock,
|
||||
};
|
||||
use crate::admin::storage_api::error::StorageError;
|
||||
use crate::server::{
|
||||
ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot, is_event_notifier_reconciled,
|
||||
refresh_persisted_module_switches_from, save_persisted_module_switches_to,
|
||||
};
|
||||
use crate::storage_api::cluster::{Endpoint, EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::storage_api::startup::storage::{init_local_disks_with_instance_ctx, new_instance_ctx};
|
||||
use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
|
||||
use rustfs_config::notify::{ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES};
|
||||
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
|
||||
use rustfs_config::{MQTT_BROKER, MQTT_QUEUE_DIR, MQTT_TOPIC, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR};
|
||||
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
|
||||
use rustfs_kms::KmsServiceManager;
|
||||
use std::collections::HashMap;
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::path::Path;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::task::Poll;
|
||||
use tempfile::TempDir;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const LIFECYCLE_RELOAD_LABEL: &str = "lifecycle";
|
||||
const REAL_STORE_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
const REPLICATION_RELOAD_LABEL: &str = "replication";
|
||||
|
||||
fn without_storage_class_env<R>(f: impl FnOnce() -> R) -> R {
|
||||
@@ -748,6 +867,139 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn concurrent_real_store_config_rmw_preserves_notify_and_oidc_updates() {
|
||||
let temp_dir = TempDir::new().expect("server config RMW temp dir");
|
||||
let store = build_isolated_heterogeneous_store(temp_dir.path()).await;
|
||||
save_admin_server_config(store.clone(), &ServerConfig::new())
|
||||
.await
|
||||
.expect("persist baseline server config");
|
||||
|
||||
let notify_entered = Arc::new(tokio::sync::Notify::new());
|
||||
let release_notify = Arc::new(tokio::sync::Notify::new());
|
||||
let (oidc_polled_tx, oidc_polled_rx) = tokio::sync::oneshot::channel();
|
||||
let (oidc_entered_tx, mut oidc_entered_rx) = tokio::sync::oneshot::channel();
|
||||
let transaction_order = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
let notify_task = {
|
||||
let store = store.clone();
|
||||
let notify_entered = notify_entered.clone();
|
||||
let release_notify = release_notify.clone();
|
||||
let transaction_order = transaction_order.clone();
|
||||
tokio::spawn(async move {
|
||||
let transaction_store = store.clone();
|
||||
with_admin_server_config_write_lock(store, move || async move {
|
||||
let mut config = read_admin_config_without_migrate_no_lock(transaction_store.clone()).await?;
|
||||
transaction_order.lock().expect("transaction order lock").push("notify-read");
|
||||
notify_entered.notify_one();
|
||||
release_notify.notified().await;
|
||||
|
||||
let mut target = KVS::new();
|
||||
target.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
|
||||
target.insert(WEBHOOK_ENDPOINT.to_string(), "https://notify.example.test/hook".to_string());
|
||||
config
|
||||
.0
|
||||
.entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
|
||||
.or_default()
|
||||
.insert("concurrent-notify".to_string(), target);
|
||||
save_admin_server_config_no_lock(transaction_store, &config).await?;
|
||||
transaction_order.lock().expect("transaction order lock").push("notify-save");
|
||||
Ok::<(), StorageError>(())
|
||||
})
|
||||
.await
|
||||
.expect("notify transaction should acquire config locks")
|
||||
.expect("notify transaction should persist");
|
||||
})
|
||||
};
|
||||
|
||||
tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, notify_entered.notified())
|
||||
.await
|
||||
.expect("notify transaction should enter");
|
||||
|
||||
let oidc_task = {
|
||||
let store = store.clone();
|
||||
let transaction_order = transaction_order.clone();
|
||||
tokio::spawn(async move {
|
||||
let transaction_store = store.clone();
|
||||
let transaction = with_admin_server_config_write_lock(store, move || async move {
|
||||
let _ = oidc_entered_tx.send(());
|
||||
let mut config = read_admin_config_without_migrate_no_lock(transaction_store.clone()).await?;
|
||||
transaction_order.lock().expect("transaction order lock").push("oidc-read");
|
||||
|
||||
let mut provider = KVS::new();
|
||||
provider.insert(
|
||||
OIDC_CONFIG_URL.to_string(),
|
||||
"https://identity.example.test/.well-known/openid-configuration".to_string(),
|
||||
);
|
||||
provider.insert(OIDC_CLIENT_ID.to_string(), "console".to_string());
|
||||
config
|
||||
.0
|
||||
.entry(IDENTITY_OPENID_SUB_SYS.to_string())
|
||||
.or_default()
|
||||
.insert("concurrent-oidc".to_string(), provider);
|
||||
save_admin_server_config_no_lock(transaction_store, &config).await?;
|
||||
transaction_order.lock().expect("transaction order lock").push("oidc-save");
|
||||
Ok::<(), StorageError>(())
|
||||
});
|
||||
tokio::pin!(transaction);
|
||||
let mut oidc_polled_tx = Some(oidc_polled_tx);
|
||||
poll_fn(|cx| match transaction.as_mut().poll(cx) {
|
||||
Poll::Pending => {
|
||||
if let Some(tx) = oidc_polled_tx.take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
Poll::Ready(())
|
||||
}
|
||||
Poll::Ready(_) => panic!("OIDC transaction entered while notify held the config lock"),
|
||||
})
|
||||
.await;
|
||||
transaction
|
||||
.await
|
||||
.expect("OIDC transaction should acquire config locks")
|
||||
.expect("OIDC transaction should persist");
|
||||
})
|
||||
};
|
||||
|
||||
tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, oidc_polled_rx)
|
||||
.await
|
||||
.expect("OIDC transaction should be polled")
|
||||
.expect("OIDC transaction poll signal should be delivered");
|
||||
assert!(
|
||||
matches!(oidc_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)),
|
||||
"OIDC RMW must not enter while the notify RMW owns the server-config transaction"
|
||||
);
|
||||
|
||||
release_notify.notify_one();
|
||||
tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, async {
|
||||
notify_task.await.expect("notify task should not panic");
|
||||
oidc_task.await.expect("OIDC task should not panic");
|
||||
})
|
||||
.await
|
||||
.expect("both server-config transactions should finish");
|
||||
|
||||
let persisted = read_admin_config_without_migrate(store)
|
||||
.await
|
||||
.expect("read final server config");
|
||||
assert_eq!(
|
||||
persisted
|
||||
.get_value(NOTIFY_WEBHOOK_SUB_SYS, "concurrent-notify")
|
||||
.expect("notify update should survive")
|
||||
.get(WEBHOOK_ENDPOINT),
|
||||
"https://notify.example.test/hook"
|
||||
);
|
||||
assert_eq!(
|
||||
persisted
|
||||
.get_value(IDENTITY_OPENID_SUB_SYS, "concurrent-oidc")
|
||||
.expect("OIDC update should survive")
|
||||
.get(OIDC_CLIENT_ID),
|
||||
"console"
|
||||
);
|
||||
assert_eq!(
|
||||
*transaction_order.lock().expect("transaction order lock"),
|
||||
vec!["notify-read", "notify-save", "oidc-read", "oidc-save"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn peer_dynamic_reload_rejects_later_pool_without_publishing() {
|
||||
@@ -811,6 +1063,67 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn peer_module_switch_reload_uses_its_explicit_context_store() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_NOTIFY_ENABLE, None::<&str>),
|
||||
(rustfs_config::ENV_AUDIT_ENABLE, None::<&str>),
|
||||
],
|
||||
async {
|
||||
let selected = runtime_config_reload_fixture().await;
|
||||
let fallback = runtime_config_reload_fixture().await;
|
||||
save_persisted_module_switches_to(
|
||||
selected.context.object_store(),
|
||||
PersistedModuleSwitches {
|
||||
notify_enabled: true,
|
||||
audit_enabled: true,
|
||||
},
|
||||
|| (),
|
||||
)
|
||||
.await
|
||||
.expect("persist selected module switches");
|
||||
|
||||
refresh_persisted_module_switches_from(fallback.context.object_store())
|
||||
.await
|
||||
.expect("refresh absent fallback module switches");
|
||||
assert!(
|
||||
!current_module_switch_snapshot().notify_enabled,
|
||||
"fallback store should resolve the default"
|
||||
);
|
||||
|
||||
reload_dynamic_config_runtime_state_for_context(Some(&selected.context), MODULE_SWITCHES_SIGNAL_SUBSYSTEM)
|
||||
.await
|
||||
.expect("peer module switch reload should converge");
|
||||
|
||||
let selected_snapshot = current_module_switch_snapshot();
|
||||
assert!(selected_snapshot.notify_enabled);
|
||||
assert!(selected_snapshot.persisted_audit_enabled);
|
||||
assert_eq!(selected_snapshot.notify_source, ModuleSwitchSource::Console);
|
||||
assert!(matches!(
|
||||
rustfs_notify::notification_system()
|
||||
.expect("notification singleton should exist")
|
||||
.runtime_lifecycle_state(),
|
||||
rustfs_notify::NotificationRuntimeState::TargetsEnabled { .. }
|
||||
));
|
||||
assert!(is_event_notifier_reconciled());
|
||||
|
||||
reload_dynamic_config_runtime_state_for_context(Some(&fallback.context), MODULE_SWITCHES_SIGNAL_SUBSYSTEM)
|
||||
.await
|
||||
.expect("restore default module switch state");
|
||||
assert!(!current_module_switch_snapshot().notify_enabled);
|
||||
assert_eq!(
|
||||
rustfs_notify::notification_system()
|
||||
.expect("notification singleton should remain stable")
|
||||
.runtime_lifecycle_state(),
|
||||
rustfs_notify::NotificationRuntimeState::LiveOnly
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn peer_full_reload_rejects_later_pool_without_publishing() {
|
||||
@@ -835,12 +1148,12 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_reload_publishes_snapshots_before_best_effort_worker_failure() {
|
||||
async fn full_reload_publishes_snapshots_before_reporting_worker_failure() {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let publish_events = events.clone();
|
||||
let worker_events = events.clone();
|
||||
|
||||
reload_runtime_config_snapshot_with(
|
||||
let err = reload_runtime_config_snapshot_with(
|
||||
async { Ok(ServerConfig::new()) },
|
||||
|config| async { Ok((config, PreparedRuntimeConfig::default())) },
|
||||
move |_config, _prepared| {
|
||||
@@ -855,8 +1168,9 @@ mod tests {
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("worker failure must not roll back validated storage/server snapshots");
|
||||
.expect_err("worker failure must be reported after publishing validated snapshots");
|
||||
|
||||
assert_eq!(err.message(), Some("injected worker reload failure"));
|
||||
assert_eq!(
|
||||
*events.lock().expect("reload result lock"),
|
||||
["publish", "worker-1-applied", "worker-2-failed"]
|
||||
@@ -871,7 +1185,9 @@ mod tests {
|
||||
assert!(is_dynamic_config_subsystem(HEAL_SUB_SYS));
|
||||
assert!(is_dynamic_config_subsystem(STORAGE_CLASS_SUB_SYS));
|
||||
assert!(!is_dynamic_config_subsystem("identity_openid"));
|
||||
assert!(!is_dynamic_config_subsystem("notify_webhook"));
|
||||
for sub_system in NOTIFY_SUB_SYSTEMS {
|
||||
assert!(is_dynamic_config_subsystem(sub_system));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -885,6 +1201,7 @@ mod tests {
|
||||
STORAGE_CLASS_SUB_SYS,
|
||||
AUDIT_WEBHOOK_SUB_SYS,
|
||||
AUDIT_MQTT_SUB_SYS,
|
||||
NOTIFY_WEBHOOK_SUB_SYS,
|
||||
SCANNER_SUB_SYS,
|
||||
HEAL_SUB_SYS,
|
||||
] {
|
||||
@@ -1014,17 +1331,52 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(notify_config_env)]
|
||||
fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() {
|
||||
crate::admin::storage_api::config::init_admin_config_defaults();
|
||||
let mut config = ServerConfig::new();
|
||||
let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults");
|
||||
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
|
||||
let kvs = targets.entry("primary".to_string()).or_default();
|
||||
kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
|
||||
kvs.insert(WEBHOOK_ENDPOINT.to_string(), "not-a-url".to_string());
|
||||
kvs.insert(WEBHOOK_QUEUE_DIR.to_string(), "/tmp/rustfs-notify".to_string());
|
||||
|
||||
let err = validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS).expect_err("invalid endpoint should fail");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
temp_env::with_vars_unset(
|
||||
[
|
||||
format!("{ENV_NOTIFY_WEBHOOK_ENABLE}_PRIMARY"),
|
||||
format!("{ENV_NOTIFY_WEBHOOK_ENDPOINT}_PRIMARY"),
|
||||
],
|
||||
|| {
|
||||
let err =
|
||||
validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS).expect_err("invalid endpoint should fail");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(notify_config_env)]
|
||||
fn validate_notify_subsystem_config_uses_enabled_env_overlay() {
|
||||
crate::admin::storage_api::config::init_admin_config_defaults();
|
||||
let mut config = ServerConfig::new();
|
||||
let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults");
|
||||
let kvs = targets.entry("primary".to_string()).or_default();
|
||||
kvs.insert(ENABLE_KEY.to_string(), EnableState::Off.to_string());
|
||||
kvs.insert(WEBHOOK_ENDPOINT.to_string(), "https://example.com/hook".to_string());
|
||||
kvs.insert(WEBHOOK_QUEUE_DIR.to_string(), "/tmp/rustfs-notify".to_string());
|
||||
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(format!("{ENV_NOTIFY_WEBHOOK_ENABLE}_PRIMARY"), Some("on")),
|
||||
(format!("{ENV_NOTIFY_WEBHOOK_ENDPOINT}_PRIMARY"), Some("not-a-url")),
|
||||
],
|
||||
|| {
|
||||
let err = validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS)
|
||||
.expect_err("invalid environment endpoint should fail");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
assert!(err.message().is_some_and(|message| message.contains("target 'primary'")));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+2
@@ -21,6 +21,8 @@ expression: value
|
||||
],
|
||||
"domain": "notify",
|
||||
"enabled": true,
|
||||
"health_reason": "not_loaded_in_runtime",
|
||||
"health_state": "offline",
|
||||
"id": "builtin:webhook:notify:primary",
|
||||
"plugin_id": "builtin:webhook",
|
||||
"service": "webhook",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user