diff --git a/Cargo.lock b/Cargo.lock index 49bd58601..edb391254 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -332,9 +332,9 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" @@ -706,7 +706,7 @@ dependencies = [ "nuid", "pin-project", "portable-atomic", - "rand 0.10.1", + "rand 0.10.2", "regex", "ring", "rustls-native-certs", @@ -2447,9 +2447,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "5.0.0-rc.0" +version = "5.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f359e08ca85e7bd759e1fd933ff2bccd81864c60a8fba0e259c7f822b0924bf" +checksum = "c906a87e53a36ff795d72e06e8162a83c5436e3ea89e942a9cb9fc083f0a384f" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -3604,7 +3604,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3667,7 +3667,7 @@ dependencies = [ "http 1.4.2", "local-ip-address", "md5", - "rand 0.10.1", + "rand 0.10.2", "rcgen", "reqwest", "rmp-serde", @@ -3718,14 +3718,14 @@ dependencies = [ [[package]] name = "ecdsa" -version = "0.17.0-rc.18" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54fb064faabbee66e1fc8e5c5a9458d4269dc2d8b638fe86a425adb2510d1a96" +checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ "der 0.8.0", "digest 0.11.3", - "elliptic-curve 0.14.0-rc.33", - "rfc6979 0.5.0", + "elliptic-curve 0.14.1", + "rfc6979 0.6.0", "signature 3.0.0", "spki 0.8.0", "zeroize", @@ -3768,11 +3768,11 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "3.0.0-rc.0" +version = "3.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b011170fe4f04665565b4110afef66774fe9ffff278f3eb5b81cc73d26e27d60" +checksum = "1685663e23882cd8517dcbcb1c23a6ebff4433c22dfb681d760219b62cd1b849" dependencies = [ - "curve25519-dalek 5.0.0-rc.0", + "curve25519-dalek 5.0.0-rc.1", "ed25519 3.0.0", "rand_core 0.10.1", "serde", @@ -3811,9 +3811,9 @@ dependencies = [ [[package]] name = "elliptic-curve" -version = "0.14.0-rc.33" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "102d3643d30dd8b559613c5cced68317199597fffb278cdc88daa2ef7fafc935" +checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" dependencies = [ "base16ct 1.0.0", "crypto-bigint 0.7.5", @@ -3823,7 +3823,6 @@ dependencies = [ "group 0.14.0", "hkdf 0.13.0", "hybrid-array", - "once_cell", "pem-rfc7468 1.0.0", "pkcs8 0.11.0", "rand_core 0.10.1", @@ -3928,7 +3927,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4390,7 +4389,7 @@ dependencies = [ "google-cloud-wkt", "http 1.4.2", "pin-project", - "rand 0.10.1", + "rand 0.10.2", "serde", "serde_json", "thiserror 2.0.18", @@ -4770,7 +4769,7 @@ dependencies = [ "idna", "ipnet", "jni", - "rand 0.10.1", + "rand 0.10.2", "thiserror 2.0.18", "tinyvec", "tokio", @@ -4790,7 +4789,7 @@ dependencies = [ "jni", "once_cell", "prefix-trie", - "rand 0.10.1", + "rand 0.10.2", "ring", "thiserror 2.0.18", "tinyvec", @@ -4815,7 +4814,7 @@ dependencies = [ "ndk-context", "once_cell", "parking_lot", - "rand 0.10.1", + "rand 0.10.2", "resolv-conf", "smallvec", "system-configuration", @@ -4952,9 +4951,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" @@ -5243,7 +5242,7 @@ checksum = "ae8e22120c32fb4d19ec55fba35015f57095cd95a2e3b732e44457f5915b2ee8" dependencies = [ "num-integer", "num-traits", - "rand 0.10.1", + "rand 0.10.2", "rand_core 0.10.1", ] @@ -5363,9 +5362,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" +checksum = "6142247df1a93c2b3587402a19710be3e6e942f1581a1702e76408f2c21d6590" [[package]] name = "jiff-tzdb-platform" @@ -6280,7 +6279,7 @@ dependencies = [ "lru 0.18.0", "mysql_common", "percent-encoding", - "rand 0.10.1", + "rand 0.10.2", "rustls", "serde", "socket2", @@ -6481,7 +6480,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6495,9 +6494,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" dependencies = [ "num-integer", "num-traits", @@ -6619,7 +6618,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "chrono", "getrandom 0.2.17", "http 1.4.2", @@ -6960,7 +6959,7 @@ dependencies = [ "hmac 0.12.1", "pkcs12", "pkcs5 0.7.1", - "rand 0.10.1", + "rand 0.10.2", "rc2", "sha1 0.10.6", "sha2 0.10.9", @@ -6982,14 +6981,14 @@ dependencies = [ [[package]] name = "p256" -version = "0.14.0-rc.10" +version = "0.14.0-rc.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41adc63effe99d48837a8cc0e6d7a77e32ae6a07f6000df466178dbc2193093e" +checksum = "b6bb40a5099e2c38a09dd29321a7a7f045f165a54317679c7cdfb0cbaf8f6b1e" dependencies = [ - "ecdsa 0.17.0-rc.18", - "elliptic-curve 0.14.0-rc.33", + "ecdsa 0.17.0", + "elliptic-curve 0.14.1", "primefield", - "primeorder 0.14.0-rc.10", + "primeorder 0.14.0", "sha2 0.11.0", ] @@ -7007,29 +7006,29 @@ dependencies = [ [[package]] name = "p384" -version = "0.14.0-rc.10" +version = "0.14.0-rc.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd5333afa5ae0347f39e6a0f2c9c155da431583fd71fe5555bd0521b4ccaf02" +checksum = "492f329d7eb11d22dadc988626b9ea1f503b4ab043a8b1e4e2cc4ae45dabdd70" dependencies = [ - "ecdsa 0.17.0-rc.18", - "elliptic-curve 0.14.0-rc.33", + "ecdsa 0.17.0", + "elliptic-curve 0.14.1", "fiat-crypto 0.3.0", "primefield", - "primeorder 0.14.0-rc.10", + "primeorder 0.14.0", "sha2 0.11.0", ] [[package]] name = "p521" -version = "0.14.0-rc.10" +version = "0.14.0-rc.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3a5297f53dc16d35909060ba3032cff7867e8809f01e273ff325579d5f0ceae" +checksum = "ff42e4ace5424e3b6d7cb82514be89866b85af87015f80341e37dcc21a66ce6e" dependencies = [ "base16ct 1.0.0", - "ecdsa 0.17.0-rc.18", - "elliptic-curve 0.14.0-rc.33", + "ecdsa 0.17.0", + "elliptic-curve 0.14.1", "primefield", - "primeorder 0.14.0-rc.10", + "primeorder 0.14.0", "sha2 0.11.0", ] @@ -7055,7 +7054,7 @@ dependencies = [ "delegate", "futures", "log", - "rand 0.10.1", + "rand 0.10.2", "sha2 0.11.0", "thiserror 2.0.18", "tokio", @@ -7527,7 +7526,7 @@ dependencies = [ "hmac 0.13.0", "md-5 0.11.0", "memchr", - "rand 0.10.1", + "rand 0.10.2", "sha2 0.11.0", "stringprep", ] @@ -7631,11 +7630,15 @@ dependencies = [ [[package]] name = "primeorder" -version = "0.14.0-rc.10" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d2793f22b9b6fd11ef3ac1d59bf003c2573593e4968702341605c2748fd90bf" +checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" dependencies = [ - "elliptic-curve 0.14.0-rc.33", + "elliptic-curve 0.14.1", + "once_cell", + "primefield", + "serdect", + "wnaf", ] [[package]] @@ -7749,7 +7752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -7769,7 +7772,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "petgraph 0.8.3", @@ -7790,7 +7793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.118", @@ -7803,7 +7806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.118", @@ -8020,7 +8023,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -8077,9 +8080,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -8441,12 +8444,12 @@ dependencies = [ [[package]] name = "rfc6979" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5236ce872cac07e0fb3969b0cbf468c7d2f37d432f1b627dcb7b8d34563fb0c3" +checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ + "crypto-bigint 0.7.5", "hmac 0.13.0", - "subtle", ] [[package]] @@ -8581,9 +8584,9 @@ dependencies = [ [[package]] name = "russh" -version = "0.61.2" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf893f64684e58da8a68d56a5e84d1cf0440226274c515770fe267707a7d0b0" +checksum = "20d2039c3e50abb1ec3bdadea27098679da12a33befa9ab4e62adad9a13ebcb2" dependencies = [ "aes 0.9.1", "aws-lc-rs", @@ -8595,14 +8598,14 @@ dependencies = [ "cipher 0.5.2", "crypto-bigint 0.7.5", "ctr", - "curve25519-dalek 5.0.0-rc.0", + "curve25519-dalek 5.0.0-rc.1", "data-encoding", "delegate", "der 0.8.0", "digest 0.11.3", - "ecdsa 0.17.0-rc.18", - "ed25519-dalek 3.0.0-rc.0", - "elliptic-curve 0.14.0-rc.33", + "ecdsa 0.17.0", + "ed25519-dalek 3.0.0-rc.1", + "elliptic-curve 0.14.1", "enum_dispatch", "flate2", "futures", @@ -8619,8 +8622,8 @@ dependencies = [ "ml-kem", "module-lattice", "num-bigint", - "p256 0.14.0-rc.10", - "p384 0.14.0-rc.10", + "p256 0.14.0-rc.15", + "p384 0.14.0-rc.15", "p521", "pageant", "pbkdf2 0.13.0", @@ -8628,7 +8631,7 @@ dependencies = [ "pkcs5 0.8.1", "pkcs8 0.11.0", "polyval", - "rand 0.10.1", + "rand 0.10.2", "rand_core 0.10.1", "rsa 0.10.0-rc.18", "russh-cryptovec", @@ -8653,9 +8656,9 @@ dependencies = [ [[package]] name = "russh-cryptovec" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443f6bbcfacb34a1aab2b12b99bf08e0c63abdc5a0db261901365df9d57fff51" +checksum = "3aec6cb630dbe85d72ffd7bcd95f07e1bd69f9f270ee8adfa1afe443a6331438" dependencies = [ "log", "nix 0.31.3", @@ -8738,9 +8741,9 @@ checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -8802,7 +8805,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "proptest", - "rand 0.10.1", + "rand 0.10.2", "rcgen", "reqwest", "rmp-serde", @@ -8961,7 +8964,7 @@ version = "1.0.0-beta.8" dependencies = [ "base64-simd", "hmac 0.13.0", - "rand 0.10.1", + "rand 0.10.2", "serde", "serde_json", "sha2 0.11.0", @@ -8978,7 +8981,7 @@ dependencies = [ "chacha20poly1305", "jsonwebtoken", "pbkdf2 0.13.0", - "rand 0.10.1", + "rand 0.10.2", "rsa 0.10.0-rc.18", "serde", "serde_json", @@ -9052,7 +9055,7 @@ dependencies = [ "pin-project-lite", "proptest", "quick-xml", - "rand 0.10.1", + "rand 0.10.2", "ratelimit", "rcgen", "reed-solomon-simd", @@ -9266,7 +9269,7 @@ dependencies = [ "kafka-protocol", "metrics", "pbkdf2 0.13.0", - "rand 0.10.1", + "rand 0.10.2", "rustls", "rustls-native-certs", "sha2 0.11.0", @@ -9289,7 +9292,7 @@ dependencies = [ "kafka-protocol", "metrics", "pbkdf2 0.13.0", - "rand 0.10.1", + "rand 0.10.2", "rustfs-kafka", "rustls", "rustls-native-certs", @@ -9340,7 +9343,7 @@ dependencies = [ "jiff", "md5", "moka", - "rand 0.10.1", + "rand 0.10.2", "reqwest", "rustfs-security-governance", "rustfs-utils", @@ -9630,7 +9633,7 @@ dependencies = [ "http-body-util", "md-5 0.11.0", "pin-project-lite", - "rand 0.10.1", + "rand 0.10.2", "reqwest", "rustfs-config", "rustfs-io-metrics", @@ -9660,7 +9663,7 @@ dependencies = [ "hmac 0.13.0", "minlz", "pin-project-lite", - "rand 0.10.1", + "rand 0.10.2", "rustfs-filemeta", "rustfs-rio", "rustfs-utils", @@ -9739,7 +9742,7 @@ dependencies = [ "futures", "http 1.4.2", "metrics", - "rand 0.10.1", + "rand 0.10.2", "rmp-serde", "rustfs-common", "rustfs-config", @@ -10010,7 +10013,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10084,7 +10087,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10141,7 +10144,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" version = "0.14.0" -source = "git+https://github.com/rustfs/s3s?rev=921e9c5911951ae9efb91dc237e11be9f2f10a14#921e9c5911951ae9efb91dc237e11be9f2f10a14" +source = "git+https://github.com/rustfs/s3s?rev=b365107038a633ce0f7a199f507507a8c5e64665#b365107038a633ce0f7a199f507507a8c5e64665" dependencies = [ "arc-swap", "arrayvec", @@ -10912,17 +10915,15 @@ dependencies = [ [[package]] name = "ssh-cipher" -version = "0.3.0-rc.9" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10db6f219196a8528f9ec904d9d45cdad692d65b0e57e72be4dedd1c5fddce36" +checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597" dependencies = [ "aead", "aes 0.9.1", "aes-gcm", - "cbc 0.2.1", "chacha20", "cipher 0.5.2", - "ctr", "ctutils", "des 0.9.0", "poly1305", @@ -10932,9 +10933,9 @@ dependencies = [ [[package]] name = "ssh-encoding" -version = "0.3.0-rc.9" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abf34aa716da5d5b4c496936d042ea282ab392092cd68a72ef6a8863ff8c96a" +checksum = "7b54d0ed0498daf3f78d82e00e28c8eec9d75a067c4cfbcc7a0f7d0f4077749e" dependencies = [ "base64ct", "bytes", @@ -10947,18 +10948,18 @@ dependencies = [ [[package]] name = "ssh-key" -version = "0.7.0-rc.10" +version = "0.7.0-rc.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45735ce3dea95690e4a9e414c4cfde7f79835063c3dcd35881df85a84118e74b" +checksum = "f9a32fae177b74a22aa9c5b01bf7e68b33545be32d9e381e248058d2adc15ce3" dependencies = [ "argon2", "bcrypt-pbkdf", "ctutils", - "ed25519-dalek 3.0.0-rc.0", + "ed25519-dalek 3.0.0-rc.1", "hex", "hmac 0.13.0", - "p256 0.14.0-rc.10", - "p384 0.14.0-rc.10", + "p256 0.14.0-rc.15", + "p384 0.14.0-rc.15", "p521", "rand_core 0.10.1", "rsa 0.10.0-rc.18", @@ -11114,9 +11115,9 @@ dependencies = [ [[package]] name = "symbolic-common" -version = "13.7.0" +version = "13.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82bfbe03a011e4d3e2a94f03e054c33b101767f6905ec2a608dd85aa0cd71c97" +checksum = "dddd4e6e0693d88f584394027d97d4c4b7748c2fa86be4441703db5fa8b78a95" dependencies = [ "debugid", "memmap2", @@ -11126,9 +11127,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "13.7.0" +version = "13.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d2ed98c0fd513dd4788554e1a62d9736f3642f8aaf7923644ec647158c1ded" +checksum = "6cad904b0202151337e3dca9187593aadf6b70ca1e495a2ff420b5bf287f1b3b" dependencies = [ "rustc-demangle", "symbolic-common", @@ -11270,7 +11271,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11511,7 +11512,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.10.1", + "rand 0.10.2", "socket2", "tokio", "tokio-util", @@ -12085,7 +12086,7 @@ checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" dependencies = [ "getrandom 0.4.3", "js-sys", - "rand 0.10.1", + "rand 0.10.2", "serde_core", "wasm-bindgen", ] @@ -12353,7 +12354,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -12480,7 +12481,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -12489,7 +12490,16 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -12507,14 +12517,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -12532,48 +12559,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "1.0.3" @@ -12589,6 +12664,17 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wnaf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1" +dependencies = [ + "ff 0.14.0", + "group 0.14.0", + "hybrid-array", +] + [[package]] name = "wrapcenum-derive" version = "0.4.1" diff --git a/Cargo.toml b/Cargo.toml index a22abfc57..4ae15f2e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -202,7 +202,7 @@ zeroize = { version = "1.9.0", features = ["derive"] } # Time and Date chrono = { version = "0.4.45", features = ["serde"] } -humantime = "2.3.0" +humantime = "2.4.0" jiff = { version = "0.2.31", features = ["serde"] } time = { version = "0.3.53", features = ["std", "parsing", "formatting", "macros", "serde"] } @@ -267,7 +267,7 @@ path-clean = "1.0.1" percent-encoding = "2.3.2" pin-project-lite = "0.2.17" pretty_assertions = "1.4.1" -rand = { version = "0.10.1", features = ["serde"] } +rand = { version = "0.10.2", features = ["serde"] } ratelimit = "0.10.1" rayon = "1.12.0" reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "7.0.1", features = ["simd-accel"] } @@ -278,8 +278,8 @@ rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket redis = { version = "1.3.0", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] } rustix = { version = "1.1.4", features = ["fs"] } rust-embed = { version = "8.11.0" } -rustc-hash = { version = "2.1.2" } -s3s = { git = "https://github.com/rustfs/s3s", rev = "921e9c5911951ae9efb91dc237e11be9f2f10a14", features = ["minio"] } +rustc-hash = { version = "2.1.3" } +s3s = { git = "https://github.com/rustfs/s3s", rev = "b365107038a633ce0f7a199f507507a8c5e64665", features = ["minio"] } serial_test = "3.5.0" shadow-rs = { version = "2.0.0", default-features = false } siphasher = "1.0.3" @@ -327,7 +327,7 @@ libunftp = { version = "0.23.0", features = ["experimental"] } unftp-core = "0.1.0" suppaftp = { version = "10.0.0", features = ["tokio", "tokio-rustls-aws-lc-rs"] } rcgen = "0.14.8" -russh = { version = "0.61.2", features = ["serde"] } +russh = { version = "0.62.1", features = ["serde"] } russh-sftp = "2.3.0" # WebDAV diff --git a/crates/config/src/constants/internode.rs b/crates/config/src/constants/internode.rs index 003a7207e..14b570da2 100644 --- a/crates/config/src/constants/internode.rs +++ b/crates/config/src/constants/internode.rs @@ -32,6 +32,28 @@ pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3; pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS"; pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10; +/// Profile selector for conservative internode HTTP data-plane client tuning. +pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE"; +pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy"; + +/// Internode HTTP connection pool maximum idle connections per host. +pub const ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST: &str = "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST"; + +/// Internode HTTP connection pool idle timeout in seconds. +pub const ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS"; + +/// Internode HTTP/2 initial stream window size in bytes. +pub const ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE"; + +/// Internode HTTP/2 initial connection window size in bytes. +pub const ENV_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE"; + +/// Whether internode HTTP/2 adaptive window sizing is enabled. +pub const ENV_INTERNODE_HTTP2_ADAPTIVE_WINDOW: &str = "RUSTFS_INTERNODE_HTTP2_ADAPTIVE_WINDOW"; + +/// Internode HTTP proxy mode: legacy, off, or system. +pub const ENV_INTERNODE_HTTP_PROXY: &str = "RUSTFS_INTERNODE_HTTP_PROXY"; + /// Environment variable for selecting the internode data-plane transport backend. pub const ENV_RUSTFS_INTERNODE_DATA_TRANSPORT: &str = "RUSTFS_INTERNODE_DATA_TRANSPORT"; pub const DEFAULT_INTERNODE_DATA_TRANSPORT: &str = "tcp-http"; @@ -53,6 +75,7 @@ mod tests { assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5); assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 3); assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 10); + assert_eq!(DEFAULT_INTERNODE_HTTP_TUNING_PROFILE, "legacy"); } #[test] @@ -68,6 +91,19 @@ mod tests { "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS" ); assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS"); + assert_eq!(ENV_INTERNODE_HTTP_TUNING_PROFILE, "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE"); + assert_eq!(ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST, "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST"); + assert_eq!(ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS, "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS"); + assert_eq!( + ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE, + "RUSTFS_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE" + ); + assert_eq!( + ENV_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE, + "RUSTFS_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE" + ); + assert_eq!(ENV_INTERNODE_HTTP2_ADAPTIVE_WINDOW, "RUSTFS_INTERNODE_HTTP2_ADAPTIVE_WINDOW"); + assert_eq!(ENV_INTERNODE_HTTP_PROXY, "RUSTFS_INTERNODE_HTTP_PROXY"); assert_eq!(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, "RUSTFS_INTERNODE_DATA_TRANSPORT"); assert_eq!(DEFAULT_INTERNODE_DATA_TRANSPORT, "tcp-http"); assert_eq!(INTERNODE_DATA_TRANSPORT_TCP, "tcp"); diff --git a/crates/config/src/constants/object.rs b/crates/config/src/constants/object.rs index a77dc0274..62180919c 100644 --- a/crates/config/src/constants/object.rs +++ b/crates/config/src/constants/object.rs @@ -191,8 +191,9 @@ pub const DEFAULT_OBJECT_IO_BUFFER_SIZE: usize = 128 * 1024; /// Environment variable to enable/disable lock optimization. /// -/// When enabled, fully materialized reads may release read locks before the -/// reader is returned. Streaming reads keep the lock until EOF or drop. +/// When enabled, read locks may be released before the reader is returned. +/// Disable this only when streaming readers must keep the object namespace +/// locked until EOF or drop. /// /// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE`). pub const ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE: &str = "RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE"; diff --git a/crates/e2e_test/src/reliant/grpc_lock_server.rs b/crates/e2e_test/src/reliant/grpc_lock_server.rs index 01849428f..97b92c383 100644 --- a/crates/e2e_test/src/reliant/grpc_lock_server.rs +++ b/crates/e2e_test/src/reliant/grpc_lock_server.rs @@ -21,8 +21,9 @@ use rustfs_lock::{LockClient, LockRequest}; use rustfs_protos::{ models::PingBodyBuilder, proto_gen::node_service::{ - BatchGenerallyLockRequest, BatchGenerallyLockResponse, GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, - PingRequest, PingResponse, node_service_server::NodeService, + BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse, + GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse, + node_service_server::NodeService, }, }; use std::pin::Pin; @@ -96,6 +97,13 @@ impl NodeService for MinimalLockNodeService { })) } + async fn batch_read_version( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs")) + } + async fn lock(&self, request: Request) -> Result, Status> { let request = request.into_inner(); let args: LockRequest = match serde_json::from_str(&request.args) { diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 1bd46194f..67b5e44dc 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -84,9 +84,11 @@ pub mod disk { pub use crate::disk::error_reduce::is_all_buckets_not_found; pub use crate::disk::local::ScanGuard; pub use crate::disk::{ - BUCKET_META_PREFIX, CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, - DiskStore, FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, + BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, + CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, + FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, + validate_batch_read_version_item_count, }; pub use crate::disk::{endpoint, error, error_reduce}; pub use bytes::Bytes; diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 1640d7a3d..ee75fac1b 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -20,8 +20,9 @@ use crate::cluster::rpc::internode_data_transport::{ }; use crate::disk::error::{Error, Result}; use crate::disk::{ - CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, - FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, + BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, + DiskOption, FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, + UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one, disk_store::{ DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE, get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout, @@ -30,6 +31,7 @@ use crate::disk::{ }, endpoint::Endpoint, health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state}, + validate_batch_read_version_item_count, }; use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard}; use crate::set_disk::DEFAULT_READ_BUFFER_SIZE; @@ -40,11 +42,11 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; use rustfs_protos::evict_failed_connection; use rustfs_protos::proto_gen::node_service::RenamePartRequest; use rustfs_protos::proto_gen::node_service::{ - CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, - DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, - ReadMetadataRequest, ReadMultipleRequest, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, - RenameFileRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, - node_service_client::NodeServiceClient, + BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest, + DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest, + MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, + ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, StatVolumeRequest, + UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient, }; use serde::{Serialize, de::DeserializeOwned}; use std::{ @@ -63,7 +65,7 @@ use tokio::{ time::timeout, }; use tokio_util::sync::CancellationToken; -use tonic::{Request, service::interceptor::InterceptedService, transport::Channel}; +use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel}; use tracing::{Instrument, debug, warn}; use uuid::Uuid; @@ -75,11 +77,78 @@ enum FailureHealthAction { const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2; const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20); +const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ"; +const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC"; +const BATCH_METADATA_RPC_OFF: &str = "off"; +const BATCH_METADATA_RPC_AUTO: &str = "auto"; +const BATCH_METADATA_RPC_ON: &str = "on"; +const BATCH_READ_VERSION_GATE_ATTEMPT: &str = "attempt"; +const BATCH_READ_VERSION_GATE_OFF_UNARY: &str = "off_unary"; +const BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED: &str = "fallback_unimplemented"; +const BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK: &str = "unsupported_no_fallback"; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk"; const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health"; const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc"; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum BatchMetadataRpcMode { + Off, + Auto, + On, +} + +impl BatchMetadataRpcMode { + fn as_str(self) -> &'static str { + match self { + Self::Off => BATCH_METADATA_RPC_OFF, + Self::Auto => BATCH_METADATA_RPC_AUTO, + Self::On => BATCH_METADATA_RPC_ON, + } + } + + fn should_attempt(self) -> bool { + matches!(self, Self::Auto | Self::On) + } + + fn should_fallback_on_unimplemented(self) -> bool { + matches!(self, Self::Auto) + } +} + +fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode { + match raw.trim() { + value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_AUTO) => BatchMetadataRpcMode::Auto, + value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_ON) => BatchMetadataRpcMode::On, + value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_OFF) => BatchMetadataRpcMode::Off, + _ => BatchMetadataRpcMode::Off, + } +} + +fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode { + rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ) + .or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC)) + .as_deref() + .map(parse_batch_metadata_rpc_mode) + .unwrap_or(BatchMetadataRpcMode::Off) +} + +fn batch_metadata_rpc_mode() -> BatchMetadataRpcMode { + // The gate cannot change at runtime; parse it once instead of re-reading + // the environment on every batch RPC. + static MODE: std::sync::LazyLock = std::sync::LazyLock::new(batch_metadata_rpc_mode_from_env); + *MODE +} + +fn record_batch_read_version_gate_decision(mode: BatchMetadataRpcMode, decision: &'static str) { + counter!( + "rustfs_remote_disk_batch_read_version_gate_total", + "mode" => mode.as_str(), + "decision" => decision + ) + .increment(1); +} + async fn copy_stream_with_buffer(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result where R: AsyncRead + Unpin, @@ -769,6 +838,86 @@ fn decode_msgpack_or_json(binary: &[u8], json: &str) -> Res serde_json::from_str(json).map_err(Error::from) } +fn decode_read_multiple_response_items(response: ReadMultipleResponse, endpoint: &Endpoint) -> Result> { + if !response.read_multiple_resps_bin.is_empty() { + if !response.read_multiple_resps.is_empty() + && response.read_multiple_resps.len() != response.read_multiple_resps_bin.len() + { + warn!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %endpoint, + json_count = response.read_multiple_resps.len(), + msgpack_count = response.read_multiple_resps_bin.len(), + op = "read_multiple", + state = "response_count_mismatch", + "Remote disk ReadMultiple compatibility payload counts differ" + ); + } + + let mut read_multiple_resps = Vec::with_capacity(response.read_multiple_resps_bin.len()); + for (index, buf) in response.read_multiple_resps_bin.iter().enumerate() { + let resp = decode_msgpack_or_json::(buf, "").map_err(|err| { + Error::other(format!("decode ReadMultipleResp msgpack item {index} from {endpoint} failed: {err}")) + })?; + read_multiple_resps.push(resp); + } + return Ok(read_multiple_resps); + } + + let mut read_multiple_resps = Vec::with_capacity(response.read_multiple_resps.len()); + for (index, json_str) in response.read_multiple_resps.iter().enumerate() { + let resp = serde_json::from_str::(json_str) + .map_err(|err| Error::other(format!("decode ReadMultipleResp json item {index} from {endpoint} failed: {err}")))?; + read_multiple_resps.push(resp); + } + + Ok(read_multiple_resps) +} + +fn decode_batch_read_version_response_items( + response: BatchReadVersionResponse, + endpoint: &Endpoint, +) -> Result> { + if !response.batch_read_version_resps_bin.is_empty() { + if !response.batch_read_version_resps.is_empty() + && response.batch_read_version_resps.len() != response.batch_read_version_resps_bin.len() + { + warn!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %endpoint, + json_count = response.batch_read_version_resps.len(), + msgpack_count = response.batch_read_version_resps_bin.len(), + op = "batch_read_version", + state = "response_count_mismatch", + "Remote disk BatchReadVersion compatibility payload counts differ" + ); + } + + let mut batch_read_version_resps = Vec::with_capacity(response.batch_read_version_resps_bin.len()); + for (index, buf) in response.batch_read_version_resps_bin.iter().enumerate() { + let resp = decode_msgpack_or_json::(buf, "").map_err(|err| { + Error::other(format!("decode BatchReadVersionResp msgpack item {index} from {endpoint} failed: {err}")) + })?; + batch_read_version_resps.push(resp); + } + return Ok(batch_read_version_resps); + } + + let mut batch_read_version_resps = Vec::with_capacity(response.batch_read_version_resps.len()); + for (index, json_str) in response.batch_read_version_resps.iter().enumerate() { + let resp = serde_json::from_str::(json_str).map_err(|err| { + Error::other(format!("decode BatchReadVersionResp json item {index} from {endpoint} failed: {err}")) + })?; + batch_read_version_resps.push(resp); + } + + Ok(batch_read_version_resps) +} + // TODO: all api need to handle errors #[async_trait::async_trait] impl DiskAPI for RemoteDisk { @@ -1462,6 +1611,99 @@ impl DiskAPI for RemoteDisk { .await } + #[tracing::instrument(skip(self, req))] + async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result> { + validate_batch_read_version_item_count(req.items.len())?; + + let mode = batch_metadata_rpc_mode(); + if !mode.should_attempt() { + record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_OFF_UNARY); + return batch_read_version_one_by_one(self, req).await; + } + record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_ATTEMPT); + + debug!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + item_count = req.items.len(), + batch_metadata_rpc_mode = mode.as_str(), + op = "batch_read_version", + state = "started", + "Remote disk RPC started" + ); + let batch_read_version_req = serde_json::to_string(&req)?; + let batch_read_version_req_bin = encode_msgpack(&req)?; + + let batch_result = self + .execute_with_timeout_for_op( + "batch_read_version", + move || async move { + let disk = self.disk_ref().await; + let mut client = self + .get_client() + .await + .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; + let request = Request::new(BatchReadVersionRequest { + disk, + batch_read_version_req, + batch_read_version_req_bin: batch_read_version_req_bin.into(), + }); + + let response = match client.batch_read_version(request).await { + Ok(response) => response.into_inner(), + Err(status) if status.code() == Code::Unimplemented => { + if mode.should_fallback_on_unimplemented() { + record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED); + warn!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + batch_metadata_rpc_mode = mode.as_str(), + op = "batch_read_version", + state = "fallback_unimplemented", + "Remote disk BatchReadVersion unsupported; falling back to unary read_version" + ); + return Ok(None); + } + + record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK); + warn!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + batch_metadata_rpc_mode = mode.as_str(), + op = "batch_read_version", + state = "unsupported_no_fallback", + "Remote disk BatchReadVersion unsupported and explicit batch RPC mode forbids fallback" + ); + return Err(Error::from(status)); + } + Err(status) => return Err(Error::from(status)), + }; + + if !response.success { + return Err(response.error.unwrap_or_default().into()); + } + + decode_batch_read_version_response_items(response, &self.endpoint).map(Some) + }, + get_max_timeout_duration(), + ) + .await?; + + match batch_result { + Some(batch_read_version_resps) => Ok(batch_read_version_resps), + // Run the unary fallback outside the batch RPC deadline so each + // read_version keeps its own per-op timeout and health accounting + // instead of racing the whole batch against one drive timeout. + None => batch_read_version_one_by_one(self, req).await, + } + } + #[tracing::instrument(level = "debug", skip(self))] async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { debug!( @@ -2069,19 +2311,7 @@ impl DiskAPI for RemoteDisk { return Err(response.error.unwrap_or_default().into()); } - let read_multiple_resps = if !response.read_multiple_resps_bin.is_empty() { - response - .read_multiple_resps_bin - .into_iter() - .filter_map(|buf| decode_msgpack_or_json::(&buf, "").ok()) - .collect() - } else { - response - .read_multiple_resps - .into_iter() - .filter_map(|json_str| serde_json::from_str::(&json_str).ok()) - .collect() - }; + let read_multiple_resps = decode_read_multiple_response_items(response, &self.endpoint)?; Ok(read_multiple_resps) }, @@ -2413,6 +2643,196 @@ mod tests { } } + fn sample_read_multiple_resp(file: &str, data: &[u8]) -> ReadMultipleResp { + ReadMultipleResp { + bucket: "bucket".to_string(), + prefix: "prefix".to_string(), + file: file.to_string(), + exists: true, + data: data.to_vec(), + ..Default::default() + } + } + + fn sample_remote_endpoint() -> Endpoint { + Endpoint { + url: url::Url::parse("http://server:9000/disk-a").expect("endpoint URL should parse"), + is_local: false, + pool_idx: 0, + set_idx: 0, + disk_idx: 0, + } + } + + #[test] + fn read_multiple_response_decode_prefers_msgpack_payloads() { + let endpoint = sample_remote_endpoint(); + let msgpack_resp = sample_read_multiple_resp("msgpack", b"binary"); + let json_resp = sample_read_multiple_resp("json", b"fallback"); + let response = ReadMultipleResponse { + success: true, + read_multiple_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")], + read_multiple_resps_bin: vec![encode_msgpack(&msgpack_resp).expect("msgpack response should encode").into()], + error: None, + }; + + let decoded = decode_read_multiple_response_items(response, &endpoint).expect("msgpack response should decode"); + + assert_eq!(decoded.len(), 1); + assert_eq!(decoded[0].file, "msgpack"); + assert_eq!(decoded[0].data, b"binary"); + } + + #[test] + fn read_multiple_response_decode_falls_back_to_json_payloads() { + let endpoint = sample_remote_endpoint(); + let json_resp = sample_read_multiple_resp("json", b"fallback"); + let response = ReadMultipleResponse { + success: true, + read_multiple_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")], + read_multiple_resps_bin: Vec::new(), + error: None, + }; + + let decoded = decode_read_multiple_response_items(response, &endpoint).expect("json response should decode"); + + assert_eq!(decoded.len(), 1); + assert_eq!(decoded[0].file, "json"); + assert_eq!(decoded[0].data, b"fallback"); + } + + #[test] + fn read_multiple_response_decode_reports_corrupt_msgpack_item() { + let endpoint = sample_remote_endpoint(); + let response = ReadMultipleResponse { + success: true, + read_multiple_resps: Vec::new(), + read_multiple_resps_bin: vec![ + encode_msgpack(&sample_read_multiple_resp("ok", b"data")) + .expect("msgpack response should encode") + .into(), + bytes::Bytes::from_static(b"not-msgpack"), + ], + error: None, + }; + + let err = decode_read_multiple_response_items(response, &endpoint).expect_err("corrupt msgpack item should fail"); + let err = err.to_string(); + + assert!(err.contains("ReadMultipleResp msgpack item 1"), "unexpected error: {err}"); + assert!(err.contains("server:9000"), "unexpected error: {err}"); + } + + fn sample_batch_read_version_resp(index: usize, path: &str, success: bool) -> BatchReadVersionResp { + BatchReadVersionResp { + index, + path: path.to_string(), + version_id: "version-a".to_string(), + success, + error: if success { + String::new() + } else { + "file version not found".to_string() + }, + ..Default::default() + } + } + + #[test] + fn batch_read_version_response_decode_prefers_msgpack_payloads() { + let endpoint = sample_remote_endpoint(); + let msgpack_resp = sample_batch_read_version_resp(7, "msgpack-object", true); + let json_resp = sample_batch_read_version_resp(1, "json-object", false); + let response = BatchReadVersionResponse { + success: true, + batch_read_version_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")], + batch_read_version_resps_bin: vec![encode_msgpack(&msgpack_resp).expect("msgpack response should encode").into()], + error: None, + }; + + let decoded = decode_batch_read_version_response_items(response, &endpoint).expect("msgpack response should decode"); + + assert_eq!(decoded.len(), 1); + assert_eq!(decoded[0].index, 7); + assert_eq!(decoded[0].path, "msgpack-object"); + assert!(decoded[0].success); + } + + #[test] + fn batch_read_version_response_decode_reports_corrupt_msgpack_item() { + let endpoint = sample_remote_endpoint(); + let response = BatchReadVersionResponse { + success: true, + batch_read_version_resps: Vec::new(), + batch_read_version_resps_bin: vec![ + encode_msgpack(&sample_batch_read_version_resp(0, "ok", true)) + .expect("msgpack response should encode") + .into(), + bytes::Bytes::from_static(b"not-msgpack"), + ], + error: None, + }; + + let err = decode_batch_read_version_response_items(response, &endpoint) + .expect_err("corrupt msgpack item should fail") + .to_string(); + + assert!(err.contains("BatchReadVersionResp msgpack item 1"), "unexpected error: {err}"); + assert!(err.contains("server:9000"), "unexpected error: {err}"); + } + + #[test] + fn batch_metadata_rpc_mode_defaults_to_off_and_parses_supported_values() { + assert_eq!(parse_batch_metadata_rpc_mode(""), BatchMetadataRpcMode::Off); + assert_eq!(parse_batch_metadata_rpc_mode("off"), BatchMetadataRpcMode::Off); + assert_eq!(parse_batch_metadata_rpc_mode("auto"), BatchMetadataRpcMode::Auto); + assert_eq!(parse_batch_metadata_rpc_mode("on"), BatchMetadataRpcMode::On); + assert_eq!(parse_batch_metadata_rpc_mode("unknown"), BatchMetadataRpcMode::Off); + assert_eq!(BatchMetadataRpcMode::Off.as_str(), "off"); + assert_eq!(BatchMetadataRpcMode::Auto.as_str(), "auto"); + assert_eq!(BatchMetadataRpcMode::On.as_str(), "on"); + assert!(!BatchMetadataRpcMode::Off.should_attempt()); + assert!(BatchMetadataRpcMode::Auto.should_attempt()); + assert!(BatchMetadataRpcMode::On.should_attempt()); + assert_eq!(BATCH_READ_VERSION_GATE_ATTEMPT, "attempt"); + assert_eq!(BATCH_READ_VERSION_GATE_OFF_UNARY, "off_unary"); + assert_eq!(BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED, "fallback_unimplemented"); + assert_eq!(BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK, "unsupported_no_fallback"); + } + + #[test] + fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() { + temp_env::with_vars( + [ + (ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")), + (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")), + ], + || { + assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::Auto); + }, + ); + } + + #[test] + fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() { + temp_env::with_vars( + [ + (ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>), + (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")), + ], + || { + assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On); + }, + ); + } + + #[test] + fn batch_metadata_rpc_mode_only_auto_falls_back_on_unimplemented() { + assert!(!BatchMetadataRpcMode::Off.should_fallback_on_unimplemented()); + assert!(BatchMetadataRpcMode::Auto.should_fallback_on_unimplemented()); + assert!(!BatchMetadataRpcMode::On.should_fallback_on_unimplemented()); + } + #[test] fn rename_data_file_info_named_msgpack_is_smaller_than_json() { let file_info = sample_rename_data_file_info(); diff --git a/crates/ecstore/src/cluster/rpc/remote_locker.rs b/crates/ecstore/src/cluster/rpc/remote_locker.rs index 593026c6d..9aefa8e27 100644 --- a/crates/ecstore/src/cluster/rpc/remote_locker.rs +++ b/crates/ecstore/src/cluster/rpc/remote_locker.rs @@ -249,6 +249,22 @@ impl RemoteClient { } } } + + fn unknown_lock_info(lock_id: &LockId) -> LockInfo { + LockInfo { + id: lock_id.clone(), + resource: lock_id.resource.clone(), + lock_type: LockType::Exclusive, + status: LockStatus::Acquired, + owner: "unknown".to_string(), + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600), + last_refreshed: std::time::SystemTime::now(), + metadata: LockMetadata::default(), + priority: LockPriority::Normal, + wait_start_time: None, + } + } } #[async_trait] @@ -338,11 +354,11 @@ impl LockClient for RemoteClient { let request_string = serde_json::to_string(&unlock_request) .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?; let mut client = self.get_client().await?; + let resource_summary = unlock_request.resource.to_string(); let req = Request::new(GenerallyLockRequest { args: request_string }); - let resp = client - .un_lock(req) - .await - .map_err(|e| LockError::internal(e.to_string()))? + let resp = self + .execute_rpc("release", &resource_summary, client.un_lock(req)) + .await? .into_inner(); if let Some(error_info) = resp.error_info { return Err(LockError::internal(error_info)); @@ -351,21 +367,25 @@ impl LockClient for RemoteClient { } async fn release_locks_batch(&self, lock_ids: &[LockId]) -> Result> { + if lock_ids.is_empty() { + return Ok(Vec::new()); + } + + let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::>(); let mut client = self.get_client().await?; + let resource_summary = Self::summarize_resources(&unlock_requests); let req = Request::new(BatchGenerallyLockRequest { - args: lock_ids + args: unlock_requests .iter() - .map(|lock_id| { - serde_json::to_string(&Self::create_unlock_request(lock_id)) - .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}"))) + .map(|request| { + serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}"))) }) .collect::>>()?, }); - let resp = client - .un_lock_batch(req) - .await - .map_err(|e| LockError::internal(e.to_string()))? + let resp = self + .execute_rpc("release_batch", &resource_summary, client.un_lock_batch(req)) + .await? .into_inner(); Ok(lock_ids @@ -379,14 +399,14 @@ impl LockClient for RemoteClient { info!("remote refresh for {}", lock_id); let refresh_request = Self::create_unlock_request(lock_id); let mut client = self.get_client().await?; + let resource_summary = refresh_request.resource.to_string(); let req = Request::new(GenerallyLockRequest { args: serde_json::to_string(&refresh_request) .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, }); - let resp = client - .refresh(req) - .await - .map_err(|e| LockError::internal(e.to_string()))? + let resp = self + .execute_rpc("refresh", &resource_summary, client.refresh(req)) + .await? .into_inner(); if let Some(error_info) = resp.error_info { return Err(LockError::internal(error_info)); @@ -398,14 +418,14 @@ impl LockClient for RemoteClient { info!("remote force_release for {}", lock_id); let force_request = Self::create_unlock_request(lock_id); let mut client = self.get_client().await?; + let resource_summary = force_request.resource.to_string(); let req = Request::new(GenerallyLockRequest { args: serde_json::to_string(&force_request) .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, }); - let resp = client - .force_un_lock(req) - .await - .map_err(|e| LockError::internal(e.to_string()))? + let resp = self + .execute_rpc("force_release", &resource_summary, client.force_un_lock(req)) + .await? .into_inner(); if let Some(error_info) = resp.error_info { return Err(LockError::internal(error_info)); @@ -419,6 +439,7 @@ impl LockClient for RemoteClient { // Since there's no direct status query in the gRPC service, // we attempt a non-blocking lock acquisition to check if the resource is available let status_request = Self::create_unlock_request(lock_id); + let resource_summary = status_request.resource.to_string(); let mut client = self.get_client().await?; // Try to acquire a very short-lived lock to test availability @@ -428,56 +449,27 @@ impl LockClient for RemoteClient { }); // Try exclusive lock first with very short timeout - let resp = client.lock(req).await; + let resp = match self.execute_rpc("check_status", &resource_summary, client.lock(req)).await { + Ok(response) => response.into_inner(), + Err(_) => return Ok(Some(Self::unknown_lock_info(lock_id))), + }; - match resp { - Ok(response) => { - let resp = response.into_inner(); - if resp.success { - // If we successfully acquired the lock, the resource was free - // Immediately release it - let release_req = Request::new(GenerallyLockRequest { - args: serde_json::to_string(&status_request) - .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, - }); - let _ = client.un_lock(release_req).await; // Best effort release + if resp.success { + // If we successfully acquired the lock, the resource was free. + // Immediately release it on a best-effort basis. + let release_req = Request::new(GenerallyLockRequest { + args: serde_json::to_string(&status_request) + .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, + }); + let _ = self + .execute_rpc("check_status_release", &resource_summary, client.un_lock(release_req)) + .await; - // Return None since no one was holding the lock - Ok(None) - } else { - // Lock acquisition failed, meaning someone is holding it - // We can't determine the exact details remotely, so return a generic status - Ok(Some(LockInfo { - id: lock_id.clone(), - resource: lock_id.resource.clone(), - lock_type: LockType::Exclusive, // We can't know the exact type - status: LockStatus::Acquired, - owner: "unknown".to_string(), // Remote client can't determine owner - acquired_at: std::time::SystemTime::now(), - expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600), - last_refreshed: std::time::SystemTime::now(), - metadata: LockMetadata::default(), - priority: LockPriority::Normal, - wait_start_time: None, - })) - } - } - Err(_) => { - // Communication error or lock is held - Ok(Some(LockInfo { - id: lock_id.clone(), - resource: lock_id.resource.clone(), - lock_type: LockType::Exclusive, - status: LockStatus::Acquired, - owner: "unknown".to_string(), - acquired_at: std::time::SystemTime::now(), - expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600), - last_refreshed: std::time::SystemTime::now(), - metadata: LockMetadata::default(), - priority: LockPriority::Normal, - wait_start_time: None, - })) - } + Ok(None) + } else { + // Lock acquisition failed, meaning someone is holding it. + // We can't determine the exact details remotely, so return a generic status. + Ok(Some(Self::unknown_lock_info(lock_id))) } } @@ -558,6 +550,17 @@ mod tests { Some((addr, task)) } + async fn closed_listener_addr() -> Option { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, + Err(err) => panic!("test listener should bind: {err}"), + }; + let addr = format!("http://{}", listener.local_addr().expect("listener local address should be available")); + drop(listener); + Some(addr) + } + async fn cache_lazy_channel(addr: &str) { let channel = TonicEndpoint::from_shared(addr.to_string()).unwrap().connect_lazy(); runtime_sources::cache_test_node_channel(addr.to_string(), channel).await; @@ -664,6 +667,113 @@ mod tests { accept_task.abort(); } + #[tokio::test] + #[serial_test::serial] + async fn test_remote_client_release_uses_rpc_timeout_and_evicts_connection() { + ensure_test_rpc_secret(); + let Some((addr, accept_task)) = spawn_hanging_listener().await else { + return; + }; + cache_lazy_channel(&addr).await; + assert!(runtime_sources::test_node_channel_is_cached(&addr).await); + + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("50"))], async { + let client = RemoteClient::new(addr.clone()); + let request = test_lock_request(Duration::from_millis(5)); + let started_at = tokio::time::Instant::now(); + + let err = client.release(&request.lock_id).await.expect_err("release should time out"); + let elapsed = started_at.elapsed(); + + assert!( + elapsed >= Duration::from_millis(40), + "remote release RPC should use configured transport timeout, got {elapsed:?}" + ); + assert!( + elapsed < Duration::from_secs(1), + "test RPC timeout should keep the test fast, got {elapsed:?}" + ); + assert!(matches!(err, LockError::Timeout { .. }), "expected remote release timeout, got {err:?}"); + assert!( + !runtime_sources::test_node_channel_is_cached(&addr).await, + "release timeout should evict cached connection" + ); + }) + .await; + + accept_task.abort(); + } + + #[tokio::test] + #[serial_test::serial] + async fn test_remote_client_refresh_tonic_error_evicts_connection() { + ensure_test_rpc_secret(); + let Some(addr) = closed_listener_addr().await else { + return; + }; + cache_lazy_channel(&addr).await; + assert!(runtime_sources::test_node_channel_is_cached(&addr).await); + + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("500"))], async { + let client = RemoteClient::new(addr.clone()); + let request = test_lock_request(Duration::from_millis(5)); + + let err = client + .refresh(&request.lock_id) + .await + .expect_err("refresh should report tonic failure"); + + assert!( + err.to_string().contains("refresh RPC failed"), + "expected refresh RPC failure marker, got {err}" + ); + assert!( + !runtime_sources::test_node_channel_is_cached(&addr).await, + "refresh tonic error should evict cached connection" + ); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn test_remote_client_check_status_timeout_evicts_connection_and_preserves_status_shape() { + ensure_test_rpc_secret(); + let Some((addr, accept_task)) = spawn_hanging_listener().await else { + return; + }; + cache_lazy_channel(&addr).await; + assert!(runtime_sources::test_node_channel_is_cached(&addr).await); + + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("50"))], async { + let client = RemoteClient::new(addr.clone()); + let request = test_lock_request(Duration::from_millis(5)); + let started_at = tokio::time::Instant::now(); + + let status = client.check_status(&request.lock_id).await.unwrap(); + let elapsed = started_at.elapsed(); + + assert!( + elapsed >= Duration::from_millis(40), + "remote check_status RPC should use configured transport timeout, got {elapsed:?}" + ); + assert!( + elapsed < Duration::from_secs(1), + "test RPC timeout should keep the test fast, got {elapsed:?}" + ); + let info = status.expect("communication failure should preserve unknown lock status shape"); + assert_eq!(info.id, request.lock_id); + assert_eq!(info.owner, "unknown"); + assert!( + !runtime_sources::test_node_channel_is_cached(&addr).await, + "check_status timeout should evict cached connection" + ); + }) + .await; + + accept_task.abort(); + } + #[test] #[serial_test::serial] fn test_remote_client_rpc_timeout_honors_configured_deadline() { diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 0ecf7b21b..6c09fe848 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -577,6 +577,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { version_id: &str, opts: &ReadOptions, ) -> Result; + async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result> { + batch_read_version_one_by_one(self, req).await + } async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result; async fn read_metadata(&self, volume: &str, path: &str) -> Result; async fn rename_data( @@ -637,6 +640,41 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { fn start_scan(&self) -> ScanGuard; } +pub async fn batch_read_version_one_by_one(disk: &D, req: BatchReadVersionReq) -> Result> +where + D: DiskAPI + ?Sized, +{ + validate_batch_read_version_item_count(req.items.len())?; + + let mut responses = Vec::with_capacity(req.items.len()); + for (index, item) in req.items.iter().enumerate() { + let response = match disk + .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts) + .await + { + Ok(file_info) => BatchReadVersionResp { + index, + path: item.path.clone(), + version_id: item.version_id.clone(), + success: true, + file_info, + error: String::new(), + }, + Err(err) => BatchReadVersionResp { + index, + path: item.path.clone(), + version_id: item.version_id.clone(), + success: false, + file_info: FileInfo::default(), + error: err.to_string(), + }, + }; + responses.push(response); + } + + Ok(responses) +} + #[derive(Debug, Default, Serialize, Deserialize)] pub struct CheckPartsResp { pub results: Vec, @@ -807,6 +845,41 @@ pub struct ReadMultipleResp { pub mod_time: Option, } +pub const BATCH_READ_VERSION_MAX_ITEMS: usize = 128; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchReadVersionItem { + pub org_volume: String, + pub volume: String, + pub path: String, + pub version_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchReadVersionReq { + pub items: Vec, + pub opts: ReadOptions, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct BatchReadVersionResp { + pub index: usize, + pub path: String, + pub version_id: String, + pub success: bool, + pub file_info: FileInfo, + pub error: String, +} + +pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> { + if item_count > BATCH_READ_VERSION_MAX_ITEMS { + return Err(DiskError::other(format!( + "batch read version item count {item_count} exceeds limit {BATCH_READ_VERSION_MAX_ITEMS}" + ))); + } + Ok(()) +} + #[derive(Debug, Deserialize, Serialize)] pub struct VolumeInfo { pub name: String, diff --git a/crates/ecstore/src/erasure/coding/decode.rs b/crates/ecstore/src/erasure/coding/decode.rs index c618c5958..2ce2b3d30 100644 --- a/crates/ecstore/src/erasure/coding/decode.rs +++ b/crates/ecstore/src/erasure/coding/decode.rs @@ -40,18 +40,54 @@ use tracing::{error, warn}; type ShardReadFuture<'a> = Pin, Error>, bool)> + Send + 'a>>; +const ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING: &str = "RUSTFS_SHARD_LOCALITY_SCHEDULING"; const ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: &str = "RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE"; -const DEFAULT_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: bool = false; +const SHARD_LOCALITY_SCHEDULING_OFF: &str = "off"; +const SHARD_LOCALITY_SCHEDULING_OBSERVE: &str = "observe"; +const SHARD_LOCALITY_SCHEDULING_ON: &str = "on"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ShardLocalitySchedulingMode { + Off, + Observe, + On, +} + +impl ShardLocalitySchedulingMode { + fn is_on(self) -> bool { + matches!(self, ShardLocalitySchedulingMode::On) + } +} + +fn parse_shard_locality_scheduling_mode(value: &str) -> ShardLocalitySchedulingMode { + match value.trim() { + value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_OFF) => ShardLocalitySchedulingMode::Off, + value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_OBSERVE) => ShardLocalitySchedulingMode::Observe, + value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_ON) => ShardLocalitySchedulingMode::On, + _ => ShardLocalitySchedulingMode::Off, + } +} + +fn get_shard_locality_scheduling_mode() -> ShardLocalitySchedulingMode { + if let Some(value) = rustfs_utils::get_env_opt_str(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING) { + return parse_shard_locality_scheduling_mode(&value); + } + + if rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, false) { + return ShardLocalitySchedulingMode::On; + } + + ShardLocalitySchedulingMode::Off +} fn get_shard_locality_preference_enabled() -> bool { - rustfs_utils::get_env_bool( - ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, - DEFAULT_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, - ) + get_shard_locality_scheduling_mode().is_on() } pub(crate) fn should_collect_shard_read_costs() -> bool { - rustfs_io_metrics::get_stage_metrics_enabled() || get_shard_locality_preference_enabled() + // `observe` mode only feeds the stage-metrics histograms, so skip the + // cost-collection overhead when those metrics cannot be reported anyway. + rustfs_io_metrics::get_stage_metrics_enabled() || get_shard_locality_scheduling_mode().is_on() } /// Number of stripes to prefetch in the legacy decode path. @@ -108,9 +144,10 @@ impl ShardReadCostCounts { } fn shard_read_launch_rank(cost: ShardReadCost) -> u8 { match cost { - ShardReadCost::Local | ShardReadCost::SameNode => 0, - ShardReadCost::Unknown => 1, + ShardReadCost::Local => 0, + ShardReadCost::SameNode => 1, ShardReadCost::Remote => 2, + ShardReadCost::Unknown => 3, } } @@ -533,6 +570,11 @@ fn shard_read_hedge_delay(read_timeout: Duration) -> Option { } } +fn shard_locality_remote_avoid_potential(remote_scheduled: usize, low_cost_available: usize, data_shards: usize) -> usize { + let theoretical_remote_needed = data_shards.saturating_sub(low_cost_available); + remote_scheduled.saturating_sub(theoretical_remote_needed) +} + #[allow(clippy::too_many_arguments)] fn record_scheduled_read_cost( read_cost: ShardReadCost, @@ -544,15 +586,13 @@ fn record_scheduled_read_cost( remote_scheduled: &mut usize, fallback_to_remote: &mut usize, ) { - if !locality_preference_enabled { - return; - } - if read_cost.is_low_cost() { - *local_preferred += 1; + if locality_preference_enabled { + *local_preferred += 1; + } } else if read_cost.is_remote() { *remote_scheduled += 1; - if count_remote_as_fallback || low_cost_available < data_shards { + if locality_preference_enabled && (count_remote_as_fallback || low_cost_available < data_shards) { *fallback_to_remote += 1; } } @@ -889,6 +929,13 @@ where fallback_to_remote, ); } else { + let remote_avoid_potential = + shard_locality_remote_avoid_potential(remote_scheduled, low_cost_available, self.data_shards); + rustfs_io_metrics::record_get_object_shard_locality_observe_only( + path, + remote_scheduled, + remote_avoid_potential, + ); rustfs_io_metrics::record_get_object_shard_locality_policy_disabled(path); } } @@ -1598,14 +1645,138 @@ mod tests { } #[test] + #[serial_test::serial] fn test_shard_locality_preference_gate_defaults_disabled() { - temp_env::with_var(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>, || { - assert!(!get_shard_locality_preference_enabled()); - }); + temp_env::with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>), + ], + || { + assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Off); + assert!(!get_shard_locality_preference_enabled()); + }, + ); } #[test] - fn test_shard_read_launch_order_is_gated() { + #[serial_test::serial] + fn test_shard_locality_scheduling_mode_parses_supported_values() { + temp_env::with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("observe")), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>), + ], + || { + assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Observe); + assert!(!get_shard_locality_preference_enabled()); + }, + ); + + temp_env::with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>), + ], + || { + assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::On); + assert!(get_shard_locality_preference_enabled()); + assert!(should_collect_shard_read_costs()); + }, + ); + + temp_env::with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("unexpected")), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")), + ], + || { + assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Off); + assert!(!get_shard_locality_preference_enabled()); + }, + ); + + temp_env::with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")), + ], + || { + assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::On); + assert!(get_shard_locality_preference_enabled()); + }, + ); + } + + #[test] + #[serial_test::serial] + fn test_shard_locality_scheduling_off_does_not_collect_without_metrics() { + temp_env::with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("off")), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>), + ], + || { + rustfs_io_metrics::set_get_stage_metrics_enabled(false); + assert!(!should_collect_shard_read_costs()); + }, + ); + } + + #[test] + #[serial_test::serial] + fn test_shard_locality_scheduling_observe_collects_only_with_stage_metrics() { + temp_env::with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("observe")), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>), + ], + || { + // Without stage metrics there is no reporting channel, so + // observe mode must not pay the cost-collection overhead. + rustfs_io_metrics::set_get_stage_metrics_enabled(false); + assert!(!should_collect_shard_read_costs()); + + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + assert!(should_collect_shard_read_costs()); + assert!(!get_shard_locality_preference_enabled()); + let read_costs = [ShardReadCost::Remote, ShardReadCost::Local, ShardReadCost::SameNode]; + assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), false), vec![0, 1, 2]); + rustfs_io_metrics::set_get_stage_metrics_enabled(false); + }, + ); + } + + #[test] + #[serial_test::serial] + fn test_shard_locality_scheduling_on_enables_reordering() { + temp_env::with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>), + ], + || { + assert!(get_shard_locality_preference_enabled()); + }, + ); + } + + #[test] + #[serial_test::serial] + fn test_shard_locality_legacy_preference_gate_still_enables_on() { + temp_env::with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")), + ], + || { + assert!(get_shard_locality_preference_enabled()); + }, + ); + } + + #[test] + fn test_shard_locality_read_launch_order_is_gated() { let read_costs = [ ShardReadCost::Remote, ShardReadCost::Local, @@ -1615,7 +1786,14 @@ mod tests { ]; assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), false), vec![0, 1, 2, 3, 4]); - assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), true), vec![1, 3, 2, 0, 4]); + assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), true), vec![1, 3, 0, 4, 2]); + } + + #[test] + fn test_shard_locality_remote_avoid_potential_is_observe_only() { + assert_eq!(shard_locality_remote_avoid_potential(2, 4, 4), 2); + assert_eq!(shard_locality_remote_avoid_potential(2, 2, 4), 0); + assert_eq!(shard_locality_remote_avoid_potential(3, 3, 4), 2); } #[test] @@ -1639,90 +1817,107 @@ mod tests { #[tokio::test] #[serial_test::serial] async fn test_parallel_reader_local_first_avoids_remote_when_local_quorum_exists() { - temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async { - const NUM_SHARDS: usize = 1; - const BLOCK_SIZE: usize = 64; - const DATA_SHARDS: usize = 4; - const PARITY_SHARDS: usize = 2; - const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS; + temp_env::async_with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>), + ], + async { + const NUM_SHARDS: usize = 1; + const BLOCK_SIZE: usize = 64; + const DATA_SHARDS: usize = 4; + const PARITY_SHARDS: usize = 2; + const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS; - let hash_algo = HashAlgorithm::HighwayHash256; - let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[]).await; - let read_costs = vec![ - ShardReadCost::Remote, - ShardReadCost::Remote, - ShardReadCost::Local, - ShardReadCost::SameNode, - ShardReadCost::Local, - ShardReadCost::SameNode, - ]; - let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE); - let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs( - readers, - erasure, - 0, - NUM_SHARDS * BLOCK_SIZE, - None, - read_costs, - ); + let hash_algo = HashAlgorithm::HighwayHash256; + let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[]).await; + let read_costs = vec![ + ShardReadCost::Remote, + ShardReadCost::Remote, + ShardReadCost::Local, + ShardReadCost::SameNode, + ShardReadCost::Local, + ShardReadCost::SameNode, + ]; + let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE); + let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs( + readers, + erasure, + 0, + NUM_SHARDS * BLOCK_SIZE, + None, + read_costs, + ); - let (bufs, errs) = parallel_reader.read().await; + let (bufs, errs) = parallel_reader.read().await; - assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count()); - assert!(bufs[0].is_none()); - assert!(bufs[1].is_none()); - for (index, buf) in bufs.iter().enumerate().take(DATA_SHARDS + PARITY_SHARDS).skip(2) { - assert_eq!(buf.as_deref(), Some(&[(index % 256) as u8; SHARD_SIZE][..])); - } - assert!(errs.iter().all(Option::is_none)); - }) + assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count()); + assert!(bufs[0].is_none()); + assert!(bufs[1].is_none()); + for (index, buf) in bufs.iter().enumerate().take(DATA_SHARDS + PARITY_SHARDS).skip(2) { + assert_eq!(buf.as_deref(), Some(&[(index % 256) as u8; SHARD_SIZE][..])); + } + assert!(errs.iter().all(Option::is_none)); + }, + ) .await; } #[tokio::test] #[serial_test::serial] async fn test_parallel_reader_local_missing_falls_back_to_remote() { - temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async { - const NUM_SHARDS: usize = 1; - const BLOCK_SIZE: usize = 64; - const DATA_SHARDS: usize = 4; - const PARITY_SHARDS: usize = 2; - const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS; + temp_env::async_with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>), + ], + async { + const NUM_SHARDS: usize = 1; + const BLOCK_SIZE: usize = 64; + const DATA_SHARDS: usize = 4; + const PARITY_SHARDS: usize = 2; + const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS; - let hash_algo = HashAlgorithm::HighwayHash256; - let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[1], &[]).await; - let read_costs = vec![ - ShardReadCost::Local, - ShardReadCost::Local, - ShardReadCost::Local, - ShardReadCost::Local, - ShardReadCost::Remote, - ShardReadCost::Remote, - ]; - let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE); - let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs( - readers, - erasure, - 0, - NUM_SHARDS * BLOCK_SIZE, - None, - read_costs, - ); + let hash_algo = HashAlgorithm::HighwayHash256; + let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[1], &[]).await; + let read_costs = vec![ + ShardReadCost::Local, + ShardReadCost::Local, + ShardReadCost::Local, + ShardReadCost::Local, + ShardReadCost::Remote, + ShardReadCost::Remote, + ]; + let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE); + let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs( + readers, + erasure, + 0, + NUM_SHARDS * BLOCK_SIZE, + None, + read_costs, + ); - let (bufs, errs) = parallel_reader.read().await; + let (bufs, errs) = parallel_reader.read().await; - assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count()); - assert!(matches!(errs[1], Some(Error::FileNotFound))); - assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..])); - assert!(bufs[5].is_none()); - }) + assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count()); + assert!(matches!(errs[1], Some(Error::FileNotFound))); + assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..])); + assert!(bufs[5].is_none()); + }, + ) .await; } #[tokio::test] #[serial_test::serial] async fn test_parallel_reader_local_corrupt_falls_back_to_remote() { - temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async { + temp_env::async_with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>), + ], + async { const NUM_SHARDS: usize = 1; const BLOCK_SIZE: usize = 64; const DATA_SHARDS: usize = 4; @@ -1757,45 +1952,52 @@ mod tests { ); assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..])); assert!(bufs[5].is_none()); - }) + }, + ) .await; } #[tokio::test] #[serial_test::serial] async fn test_erasure_decode_local_first_preserves_output_order() { - temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async { - const DATA_SHARDS: usize = 4; - const PARITY_SHARDS: usize = 2; - const BLOCK_SIZE: usize = 64; + temp_env::async_with_vars( + [ + (ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")), + (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>), + ], + async { + const DATA_SHARDS: usize = 4; + const PARITY_SHARDS: usize = 2; + const BLOCK_SIZE: usize = 64; - let total_data: Vec = (0..BLOCK_SIZE as u32).map(|i| i as u8).collect(); - let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE); - let shard_size = erasure.shard_size(); - let hash_algo = HashAlgorithm::HighwayHash256; - let shard_bufs = encode_test_object(&erasure, &total_data, shard_size, &hash_algo).await; - let readers = shard_bufs - .iter() - .map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false))) - .collect(); - let read_costs = vec![ - ShardReadCost::Remote, - ShardReadCost::Remote, - ShardReadCost::Local, - ShardReadCost::Local, - ShardReadCost::SameNode, - ShardReadCost::SameNode, - ]; + let total_data: Vec = (0..BLOCK_SIZE as u32).map(|i| i as u8).collect(); + let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE); + let shard_size = erasure.shard_size(); + let hash_algo = HashAlgorithm::HighwayHash256; + let shard_bufs = encode_test_object(&erasure, &total_data, shard_size, &hash_algo).await; + let readers = shard_bufs + .iter() + .map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false))) + .collect(); + let read_costs = vec![ + ShardReadCost::Remote, + ShardReadCost::Remote, + ShardReadCost::Local, + ShardReadCost::Local, + ShardReadCost::SameNode, + ShardReadCost::SameNode, + ]; - let mut output = Vec::new(); - let (written, err) = erasure - .decode_with_read_costs(&mut output, readers, 0, total_data.len(), total_data.len(), read_costs) - .await; + let mut output = Vec::new(); + let (written, err) = erasure + .decode_with_read_costs(&mut output, readers, 0, total_data.len(), total_data.len(), read_costs) + .await; - assert!(err.is_none(), "unexpected decode error: {err:?}"); - assert_eq!(written, total_data.len()); - assert_eq!(output, total_data); - }) + assert!(err.is_none(), "unexpected decode error: {err:?}"); + assert_eq!(written, total_data.len()); + assert_eq!(output, total_data); + }, + ) .await; } diff --git a/crates/ecstore/src/services/batch_processor.rs b/crates/ecstore/src/services/batch_processor.rs index 1c678dea8..61212ec00 100644 --- a/crates/ecstore/src/services/batch_processor.rs +++ b/crates/ecstore/src/services/batch_processor.rs @@ -19,17 +19,255 @@ use crate::disk::error::{Error, Result}; use std::future::Future; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; use tokio::task::JoinSet; +const BATCH_PROCESSOR_OPERATION_CUSTOM: &str = "custom"; +const BATCH_PROCESSOR_OPERATION_READ: &str = "read"; +const BATCH_PROCESSOR_OPERATION_WRITE: &str = "write"; +const BATCH_PROCESSOR_OPERATION_METADATA: &str = "metadata"; +const ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE: &str = "RUSTFS_BATCH_PROCESSOR_ADAPTIVE"; +const BATCH_PROCESSOR_ADAPTIVE_OFF: &str = "off"; +const BATCH_PROCESSOR_ADAPTIVE_OBSERVE: &str = "observe"; +const BATCH_PROCESSOR_ADAPTIVE_ON: &str = "on"; +const BATCH_SUGGESTION_REASON_IMPROVING: &str = "improving"; +const BATCH_SUGGESTION_REASON_DEGRADING: &str = "degrading"; +const BATCH_SUGGESTION_REASON_STABLE: &str = "stable"; +const BATCH_SUGGESTION_REASON_COOLDOWN: &str = "cooldown"; +const BATCH_OBSERVATION_IMPROVING_LATENCY: Duration = Duration::from_millis(50); +const BATCH_OBSERVATION_DEGRADING_LATENCY: Duration = Duration::from_millis(250); +const BATCH_OBSERVATION_COOLDOWN: Duration = Duration::from_secs(30); +const BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR: usize = 4; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BatchProcessorAdaptiveMode { + Off, + Observe, + On, +} + +impl BatchProcessorAdaptiveMode { + fn should_observe(self) -> bool { + matches!(self, Self::Observe | Self::On) + } + + fn should_apply(self) -> bool { + matches!(self, Self::On) + } +} + +fn parse_batch_processor_adaptive_mode(raw: &str) -> BatchProcessorAdaptiveMode { + match raw.trim() { + value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OBSERVE) => BatchProcessorAdaptiveMode::Observe, + value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_ON) => BatchProcessorAdaptiveMode::On, + value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OFF) => BatchProcessorAdaptiveMode::Off, + _ => BatchProcessorAdaptiveMode::Off, + } +} + +fn batch_processor_adaptive_mode_from_env() -> BatchProcessorAdaptiveMode { + rustfs_utils::get_env_opt_str(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE) + .as_deref() + .map(parse_batch_processor_adaptive_mode) + .unwrap_or(BatchProcessorAdaptiveMode::Off) +} + +fn batch_processor_adaptive_mode() -> BatchProcessorAdaptiveMode { + // The gate cannot change at runtime; parse it once instead of re-reading + // the environment on every batch. + static MODE: std::sync::LazyLock = + std::sync::LazyLock::new(batch_processor_adaptive_mode_from_env); + *MODE +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BatchConcurrencySuggestion { + concurrency: usize, + reason: &'static str, +} + +#[derive(Debug, Clone, Copy)] +struct BatchObservation { + batch_size: usize, + success_count: usize, + error_count: usize, + timeout_count: usize, + max_queue_wait: Duration, + execution_latency: Duration, +} + +#[derive(Debug)] +struct BatchObservationState { + last_suggestion_at: Option, + last_suggested_concurrency: usize, +} + +impl BatchObservationState { + fn new(configured_concurrency: usize) -> Self { + Self { + last_suggestion_at: None, + last_suggested_concurrency: configured_concurrency, + } + } + + fn suggest( + &mut self, + current_concurrency: usize, + configured_concurrency: usize, + observation: BatchObservation, + now: Instant, + ) -> BatchConcurrencySuggestion { + let raw = calculate_batch_concurrency_suggestion(current_concurrency, configured_concurrency, observation); + if raw.concurrency == current_concurrency { + self.last_suggested_concurrency = current_concurrency; + return raw; + } + + if self + .last_suggestion_at + .is_some_and(|last_suggestion_at| now.duration_since(last_suggestion_at) < BATCH_OBSERVATION_COOLDOWN) + { + return BatchConcurrencySuggestion { + concurrency: self.last_suggested_concurrency, + reason: BATCH_SUGGESTION_REASON_COOLDOWN, + }; + } + + self.last_suggestion_at = Some(now); + self.last_suggested_concurrency = raw.concurrency; + raw + } +} + +fn calculate_batch_concurrency_suggestion( + current_concurrency: usize, + configured_concurrency: usize, + observation: BatchObservation, +) -> BatchConcurrencySuggestion { + let current_concurrency = current_concurrency.max(1); + let configured_concurrency = configured_concurrency.max(1); + + if observation.batch_size == 0 { + return BatchConcurrencySuggestion { + concurrency: current_concurrency, + reason: BATCH_SUGGESTION_REASON_STABLE, + }; + } + + if observation.timeout_count > 0 + || observation.error_count > 0 + || (observation.batch_size >= current_concurrency && observation.execution_latency >= BATCH_OBSERVATION_DEGRADING_LATENCY) + { + return BatchConcurrencySuggestion { + concurrency: decrease_batch_concurrency(current_concurrency), + reason: BATCH_SUGGESTION_REASON_DEGRADING, + }; + } + + if observation.success_count == observation.batch_size + && observation.batch_size >= current_concurrency + && observation.execution_latency <= BATCH_OBSERVATION_IMPROVING_LATENCY + { + return BatchConcurrencySuggestion { + concurrency: increase_batch_concurrency(current_concurrency, configured_concurrency, observation.batch_size), + reason: BATCH_SUGGESTION_REASON_IMPROVING, + }; + } + + BatchConcurrencySuggestion { + concurrency: current_concurrency, + reason: BATCH_SUGGESTION_REASON_STABLE, + } +} + +fn decrease_batch_concurrency(current_concurrency: usize) -> usize { + (current_concurrency.saturating_mul(3) / 4).max(1) +} + +fn increase_batch_concurrency(current_concurrency: usize, configured_concurrency: usize, batch_size: usize) -> usize { + let step = (current_concurrency / 4).max(1); + // Suggestions build on the previous suggestion, so bound the ratchet at a + // hard multiple of the configured baseline instead of the current value. + let upper_bound = configured_concurrency + .saturating_mul(BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR) + .max(current_concurrency); + current_concurrency + .saturating_add(step) + .min(upper_bound) + .min(batch_size.max(current_concurrency)) +} + +fn is_timeout_error(err: &Error) -> bool { + matches!(err, Error::Timeout | Error::SourceStalled) + || matches!(err, Error::Io(io_err) if io_err.kind() == std::io::ErrorKind::TimedOut) +} + /// Batch processor that executes tasks concurrently with a semaphore pub struct AsyncBatchProcessor { max_concurrent: usize, + operation: &'static str, + observation_state: Mutex, } impl AsyncBatchProcessor { pub fn new(max_concurrent: usize) -> Self { - Self { max_concurrent } + Self::new_with_operation(max_concurrent, BATCH_PROCESSOR_OPERATION_CUSTOM) + } + + fn new_with_operation(max_concurrent: usize, operation: &'static str) -> Self { + let max_concurrent = max_concurrent.max(1); + Self { + max_concurrent, + operation, + observation_state: Mutex::new(BatchObservationState::new(max_concurrent)), + } + } + + fn execution_concurrency(&self, mode: BatchProcessorAdaptiveMode) -> usize { + if !mode.should_apply() { + return self.max_concurrent; + } + + self.observation_state + .lock() + .map(|state| state.last_suggested_concurrency.max(1)) + .unwrap_or(self.max_concurrent) + } + + fn observe_batch_with_mode( + &self, + mode: BatchProcessorAdaptiveMode, + execution_concurrency: usize, + observation: BatchObservation, + ) -> BatchConcurrencySuggestion { + let execution_concurrency = execution_concurrency.max(1); + if !mode.should_observe() { + return BatchConcurrencySuggestion { + concurrency: execution_concurrency, + reason: BATCH_SUGGESTION_REASON_STABLE, + }; + } + + let suggestion = match self.observation_state.lock() { + Ok(mut state) => state.suggest(execution_concurrency, self.max_concurrent, observation, Instant::now()), + Err(_) => calculate_batch_concurrency_suggestion(execution_concurrency, self.max_concurrent, observation), + }; + + rustfs_io_metrics::record_batch_processor_observation(rustfs_io_metrics::BatchProcessorObservation { + operation: self.operation, + batch_size: observation.batch_size, + configured_concurrency: execution_concurrency, + max_queue_wait_secs: observation.max_queue_wait.as_secs_f64(), + execution_latency_secs: observation.execution_latency.as_secs_f64(), + successes: observation.success_count, + errors: observation.error_count, + timeouts: observation.timeout_count, + suggested_concurrency: suggestion.concurrency, + suggestion_reason: suggestion.reason, + }); + + suggestion } /// Execute a batch of tasks concurrently with concurrency control @@ -42,27 +280,35 @@ impl AsyncBatchProcessor { return Vec::new(); } - let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent)); + let batch_size = tasks.len(); + let batch_started_at = Instant::now(); + let adaptive_mode = batch_processor_adaptive_mode(); + let execution_concurrency = self.execution_concurrency(adaptive_mode); + let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency)); let mut join_set = JoinSet::new(); - let mut results = Vec::with_capacity(tasks.len()); - for _ in 0..tasks.len() { + let mut results = Vec::with_capacity(batch_size); + for _ in 0..batch_size { results.push(Err(Error::other("Not completed"))); } // Spawn all tasks with semaphore control for (i, task) in tasks.into_iter().enumerate() { let sem = semaphore.clone(); + let queued_at = Instant::now(); join_set.spawn(async move { let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?; + let queue_wait = queued_at.elapsed(); let result = task.await; - Ok::<(usize, Result), Error>((i, result)) + Ok::<(usize, Result, Duration), Error>((i, result, queue_wait)) }); } // Collect results + let mut max_queue_wait = Duration::ZERO; while let Some(join_result) = join_set.join_next().await { match join_result { - Ok(Ok((index, task_result))) => { + Ok(Ok((index, task_result, queue_wait))) => { + max_queue_wait = max_queue_wait.max(queue_wait); if index < results.len() { results[index] = task_result; } @@ -78,6 +324,29 @@ impl AsyncBatchProcessor { } } + let mut success_count = 0; + let mut error_count = 0; + let mut timeout_count = 0; + for result in &results { + match result { + Ok(_) => success_count += 1, + Err(err) if is_timeout_error(err) => timeout_count += 1, + Err(_) => error_count += 1, + } + } + self.observe_batch_with_mode( + adaptive_mode, + execution_concurrency, + BatchObservation { + batch_size, + success_count, + error_count, + timeout_count, + max_queue_wait, + execution_latency: batch_started_at.elapsed(), + }, + ); + results } @@ -97,17 +366,29 @@ impl AsyncBatchProcessor { ))); } - let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent)); + let batch_size = tasks.len(); + let batch_started_at = Instant::now(); + let adaptive_mode = batch_processor_adaptive_mode(); + let execution_concurrency = self.execution_concurrency(adaptive_mode); + let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency)); let mut join_set = JoinSet::new(); let mut successes = Vec::new(); - let mut pending_tasks = tasks.len(); + let mut pending_tasks = batch_size; let mut first_error = None; + let mut error_count = 0; + let mut timeout_count = 0; + let mut max_queue_wait = Duration::ZERO; for task in tasks { let sem = semaphore.clone(); + let queued_at = Instant::now(); join_set.spawn(async move { - let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?; - task.await + let _permit = sem + .acquire() + .await + .map_err(|_| (Error::other("Semaphore error"), queued_at.elapsed()))?; + let queue_wait = queued_at.elapsed(); + task.await.map(|value| (value, queue_wait)).map_err(|err| (err, queue_wait)) }); } @@ -115,18 +396,38 @@ impl AsyncBatchProcessor { pending_tasks = pending_tasks.saturating_sub(1); match join_result { - Ok(Ok(value)) => { + Ok(Ok((value, queue_wait))) => { + max_queue_wait = max_queue_wait.max(queue_wait); successes.push(value); if successes.len() >= required_successes { + self.observe_batch_with_mode( + adaptive_mode, + execution_concurrency, + BatchObservation { + batch_size, + success_count: successes.len(), + error_count, + timeout_count, + max_queue_wait, + execution_latency: batch_started_at.elapsed(), + }, + ); return Ok(successes); } } - Ok(Err(err)) => { + Ok(Err((err, queue_wait))) => { + max_queue_wait = max_queue_wait.max(queue_wait); + if is_timeout_error(&err) { + timeout_count += 1; + } else { + error_count += 1; + } if first_error.is_none() { first_error = Some(err); } } Err(join_error) => { + error_count += 1; if first_error.is_none() { first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}"))); } @@ -134,6 +435,18 @@ impl AsyncBatchProcessor { } if successes.len() + pending_tasks < required_successes { + self.observe_batch_with_mode( + adaptive_mode, + execution_concurrency, + BatchObservation { + batch_size, + success_count: successes.len(), + error_count, + timeout_count, + max_queue_wait, + execution_latency: batch_started_at.elapsed(), + }, + ); return Err(first_error.unwrap_or_else(|| { Error::other(format!( "Insufficient successful results: got {}, needed {}", @@ -144,6 +457,18 @@ impl AsyncBatchProcessor { } } + self.observe_batch_with_mode( + adaptive_mode, + execution_concurrency, + BatchObservation { + batch_size, + success_count: successes.len(), + error_count, + timeout_count, + max_queue_wait, + execution_latency: batch_started_at.elapsed(), + }, + ); Err(first_error.unwrap_or_else(|| { Error::other(format!( "Insufficient successful results: got {}, needed {}", @@ -164,9 +489,9 @@ pub struct GlobalBatchProcessors { impl GlobalBatchProcessors { pub fn new() -> Self { Self { - read_processor: AsyncBatchProcessor::new(16), // Higher concurrency for reads - write_processor: AsyncBatchProcessor::new(8), // Lower concurrency for writes - metadata_processor: AsyncBatchProcessor::new(12), // Medium concurrency for metadata + read_processor: AsyncBatchProcessor::new_with_operation(16, BATCH_PROCESSOR_OPERATION_READ), + write_processor: AsyncBatchProcessor::new_with_operation(8, BATCH_PROCESSOR_OPERATION_WRITE), + metadata_processor: AsyncBatchProcessor::new_with_operation(12, BATCH_PROCESSOR_OPERATION_METADATA), } } @@ -321,4 +646,208 @@ mod tests { assert!(err.to_string().contains("first failure")); assert!(started.elapsed() < Duration::from_millis(120)); } + + #[tokio::test] + async fn test_batch_processor_observe_only_keeps_configured_concurrency() { + let processor = AsyncBatchProcessor::new(2); + + let tasks: Vec<_> = (0..4) + .map(|i| async move { + tokio::time::sleep(Duration::from_millis(5)).await; + Ok::(i) + }) + .collect(); + + let results = processor.execute_batch(tasks).await; + assert_eq!(results.len(), 4); + assert_eq!(processor.max_concurrent, 2); + } + + #[test] + fn batch_processor_adaptive_mode_defaults_to_off_and_parses_supported_values() { + assert_eq!(parse_batch_processor_adaptive_mode(""), BatchProcessorAdaptiveMode::Off); + assert_eq!(parse_batch_processor_adaptive_mode("off"), BatchProcessorAdaptiveMode::Off); + assert_eq!(parse_batch_processor_adaptive_mode("observe"), BatchProcessorAdaptiveMode::Observe); + assert_eq!(parse_batch_processor_adaptive_mode("on"), BatchProcessorAdaptiveMode::On); + assert_eq!(parse_batch_processor_adaptive_mode("unknown"), BatchProcessorAdaptiveMode::Off); + assert!(!BatchProcessorAdaptiveMode::Off.should_observe()); + assert!(BatchProcessorAdaptiveMode::Observe.should_observe()); + assert!(BatchProcessorAdaptiveMode::On.should_observe()); + assert!(!BatchProcessorAdaptiveMode::Observe.should_apply()); + assert!(BatchProcessorAdaptiveMode::On.should_apply()); + } + + #[test] + fn batch_processor_adaptive_mode_env_gate_is_parsed_from_env() { + temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, None::<&str>, || { + assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Off); + }); + temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, Some("observe"), || { + assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Observe); + }); + } + + #[test] + fn batch_processor_observation_gate_keeps_default_off_stable() { + let processor = AsyncBatchProcessor::new(8); + let suggestion = processor.observe_batch_with_mode( + BatchProcessorAdaptiveMode::Off, + processor.execution_concurrency(BatchProcessorAdaptiveMode::Off), + BatchObservation { + batch_size: 16, + success_count: 16, + error_count: 0, + timeout_count: 0, + max_queue_wait: Duration::ZERO, + execution_latency: Duration::from_millis(20), + }, + ); + + assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_STABLE); + assert_eq!(suggestion.concurrency, 8); + } + + #[test] + fn batch_processor_observation_gate_observe_records_suggestion() { + let processor = AsyncBatchProcessor::new(8); + let suggestion = processor.observe_batch_with_mode( + BatchProcessorAdaptiveMode::Observe, + processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe), + BatchObservation { + batch_size: 16, + success_count: 16, + error_count: 0, + timeout_count: 0, + max_queue_wait: Duration::ZERO, + execution_latency: Duration::from_millis(20), + }, + ); + + assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING); + assert!(suggestion.concurrency > 8); + assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe), 8); + } + + #[test] + fn batch_processor_adaptive_on_applies_last_suggestion_to_next_batch() { + let processor = AsyncBatchProcessor::new(8); + let suggestion = processor.observe_batch_with_mode( + BatchProcessorAdaptiveMode::On, + processor.execution_concurrency(BatchProcessorAdaptiveMode::On), + BatchObservation { + batch_size: 16, + success_count: 16, + error_count: 0, + timeout_count: 0, + max_queue_wait: Duration::ZERO, + execution_latency: Duration::from_millis(20), + }, + ); + + assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING); + assert!(suggestion.concurrency > 8); + assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::On), suggestion.concurrency); + } + + #[test] + fn batch_processor_suggestion_increases_for_fast_successful_batches() { + let suggestion = calculate_batch_concurrency_suggestion( + 8, + 8, + BatchObservation { + batch_size: 16, + success_count: 16, + error_count: 0, + timeout_count: 0, + max_queue_wait: Duration::ZERO, + execution_latency: Duration::from_millis(20), + }, + ); + + assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING); + assert!(suggestion.concurrency > 8); + } + + #[test] + fn batch_processor_suggestion_decreases_for_timeout_batches() { + let suggestion = calculate_batch_concurrency_suggestion( + 8, + 8, + BatchObservation { + batch_size: 16, + success_count: 14, + error_count: 0, + timeout_count: 2, + max_queue_wait: Duration::ZERO, + execution_latency: Duration::from_millis(20), + }, + ); + + assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_DEGRADING); + assert!(suggestion.concurrency < 8); + assert!(suggestion.concurrency >= 1); + } + + #[test] + fn batch_processor_suggestion_clamps_to_batch_size() { + let suggestion = calculate_batch_concurrency_suggestion( + 64, + 64, + BatchObservation { + batch_size: 65, + success_count: 65, + error_count: 0, + timeout_count: 0, + max_queue_wait: Duration::ZERO, + execution_latency: Duration::from_millis(20), + }, + ); + + assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING); + assert_eq!(suggestion.concurrency, 65); + } + + #[test] + fn batch_processor_suggestion_cooldown_prevents_rapid_oscillation() { + let mut state = BatchObservationState::new(8); + let now = Instant::now(); + let observation = BatchObservation { + batch_size: 16, + success_count: 16, + error_count: 0, + timeout_count: 0, + max_queue_wait: Duration::ZERO, + execution_latency: Duration::from_millis(20), + }; + + let first = state.suggest(8, 8, observation, now); + let second = state.suggest(8, 8, observation, now + Duration::from_secs(1)); + + assert_eq!(first.reason, BATCH_SUGGESTION_REASON_IMPROVING); + assert_eq!(second.reason, BATCH_SUGGESTION_REASON_COOLDOWN); + assert_eq!(second.concurrency, first.concurrency); + } + + #[test] + fn batch_processor_suggestion_growth_is_capped_relative_to_configured_concurrency() { + const CONFIGURED: usize = 8; + let mut state = BatchObservationState::new(CONFIGURED); + let mut current = CONFIGURED; + let mut now = Instant::now(); + let observation = BatchObservation { + batch_size: 1024, + success_count: 1024, + error_count: 0, + timeout_count: 0, + max_queue_wait: Duration::ZERO, + execution_latency: Duration::from_millis(20), + }; + + for _ in 0..64 { + current = state.suggest(current, CONFIGURED, observation, now).concurrency; + now += BATCH_OBSERVATION_COOLDOWN; + } + + assert_eq!(current, CONFIGURED * BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR); + } } diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index e1f6e5413..4b627c132 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -270,10 +270,19 @@ impl AsyncRead for SetDiskLockGuardedReader { } } -fn attach_set_disk_read_lock_guard(mut reader: GetObjectReader, read_lock_guard: Option) -> GetObjectReader { - if let Some(guard) = read_lock_guard - && reader.buffered_body.is_none() - { +fn finish_set_disk_read_lock( + mut reader: GetObjectReader, + read_lock_guard: Option, + lock_optimization_enabled: bool, + bucket: &str, + object: &str, +) -> GetObjectReader { + if lock_optimization_enabled || reader.buffered_body.is_some() { + release_materialized_read_lock(bucket, object, read_lock_guard); + return reader; + } + + if let Some(guard) = read_lock_guard { reader.stream = Box::new(SetDiskLockGuardedReader { inner: reader.stream, guard: Some(guard), @@ -2314,7 +2323,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { opts.part_number = Some(1); } let gr = get_transitioned_object_reader(bucket, object, &range, &h, &object_info, &opts).await?; - return Ok(attach_set_disk_read_lock_guard(gr, read_lock_guard.take())); + return Ok(finish_set_disk_read_lock( + gr, + read_lock_guard.take(), + lock_optimization_enabled, + bucket, + object, + )); } if is_get_small_object_direct_memory_eligible(&range, &object_info, &fi, opts) { @@ -2418,7 +2433,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { ); record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket); let (reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?; - return Ok(attach_set_disk_read_lock_guard(reader, read_lock_guard.take())); + return Ok(finish_set_disk_read_lock( + reader, + read_lock_guard.take(), + lock_optimization_enabled, + bucket, + object, + )); } read::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => { record_get_codec_streaming_gate_decision( @@ -2454,8 +2475,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let set_index = self.set_index; let pool_index = self.pool_index; let skip_verify = opts.skip_verify_bitrot; - // Move the read-lock guard into the task so it lives for the duration of the read. - // Fully materialized paths release it before returning; streaming paths keep it. + if lock_optimization_enabled { + release_materialized_read_lock(&bucket, &object, read_lock_guard.take()); + debug!(bucket, object, "Lock optimization: released read lock before streaming read"); + } + + // When lock optimization is disabled, keep the read-lock guard in the + // task so it lives for the duration of the streaming read. tokio::spawn(async move { let _guard = read_lock_guard; let mut writer = wd; diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 65656867d..ae05c2430 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -595,14 +595,64 @@ fn resolve_read_part_from_responses( Err(DiskError::ErasureReadQuorum) } -fn shard_read_cost_for_disk(disk: Option<&DiskStore>) -> ShardReadCost { +fn shard_read_costs_for_disks(disks: &[Option]) -> Vec { + let local_endpoint_hosts = local_endpoint_hosts_for_shard_costs(); + disks + .iter() + .map(|disk| shard_read_cost_for_disk(disk.as_ref(), local_endpoint_hosts)) + .collect() +} + +fn shard_read_cost_for_disk(disk: Option<&DiskStore>, local_endpoint_hosts: &[String]) -> ShardReadCost { match disk { Some(disk) if disk.is_local() => ShardReadCost::Local, - Some(_) => ShardReadCost::Remote, + Some(disk) => shard_read_cost_for_endpoint(false, &disk.host_name(), local_endpoint_hosts), None => ShardReadCost::Unknown, } } +fn shard_read_cost_for_endpoint(is_local: bool, host_name: &str, local_endpoint_hosts: &[String]) -> ShardReadCost { + if is_local { + return ShardReadCost::Local; + } + + if !host_name.is_empty() && local_endpoint_hosts.iter().any(|host| host == host_name) { + return ShardReadCost::SameNode; + } + + ShardReadCost::Remote +} + +fn local_endpoint_hosts_for_shard_costs() -> &'static [String] { + // Endpoint pools are immutable after startup, so build the host list once + // instead of walking every pool on each read. Do not cache the empty + // pre-startup answer: only memoize once the pools are published. + static LOCAL_ENDPOINT_HOSTS: std::sync::OnceLock> = std::sync::OnceLock::new(); + + if let Some(hosts) = LOCAL_ENDPOINT_HOSTS.get() { + return hosts; + } + + let Some(endpoint_pools) = runtime_sources::endpoint_pools() else { + return &[]; + }; + + let mut hosts = Vec::new(); + for pool in endpoint_pools.as_ref() { + for endpoint in pool.endpoints.as_ref() { + if !endpoint.is_local { + continue; + } + + let host = endpoint.host_port(); + if !host.is_empty() && !hosts.contains(&host) { + hosts.push(host); + } + } + } + LOCAL_ENDPOINT_HOSTS.get_or_init(|| hosts) +} + #[derive(Clone, Debug, Eq, Hash, PartialEq)] struct ReadRepairHealCacheKey { bucket: String, @@ -2813,12 +2863,7 @@ impl SetDisks { let use_mmap_read = object_mmap_read_enabled(); let reader_setup_stage_start = Instant::now(); - let read_costs = coding::decode::should_collect_shard_read_costs().then(|| { - disks - .iter() - .map(|disk| shard_read_cost_for_disk(disk.as_ref())) - .collect::>() - }); + let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(&disks)); let reader_setup = create_bitrot_readers_until_quorum_with_preference( &files, &disks, @@ -3244,12 +3289,7 @@ impl SetDisks { bitrot_reader_init_stage: GET_STAGE_READER_TASK_BITROT_READER_INIT, }); let reader_setup_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); - let read_costs = coding::decode::should_collect_shard_read_costs().then(|| { - disks - .iter() - .map(|disk| shard_read_cost_for_disk(disk.as_ref())) - .collect::>() - }); + let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(disks)); let reader_setup = create_bitrot_readers_until_quorum_with_preference( files, disks, @@ -3835,6 +3875,17 @@ mod tests { const CODEC_STREAMING_TEST_BUCKET: &str = "bucket"; const CODEC_STREAMING_TEST_OBJECT: &str = "object"; + #[test] + fn shard_read_cost_for_endpoint_maps_topology_classes() { + let local_hosts = vec!["node-a:9000".to_string()]; + + assert_eq!(shard_read_cost_for_endpoint(true, "node-a:9000", &local_hosts), ShardReadCost::Local); + assert_eq!(shard_read_cost_for_endpoint(false, "node-a:9000", &local_hosts), ShardReadCost::SameNode); + assert_eq!(shard_read_cost_for_endpoint(false, "node-b:9000", &local_hosts), ShardReadCost::Remote); + assert_eq!(shard_read_cost_for_endpoint(false, "", &local_hosts), ShardReadCost::Remote); + assert_eq!(shard_read_cost_for_disk(None, &local_hosts), ShardReadCost::Unknown); + } + fn metadata_fanout_test_fileinfo(object: &str) -> FileInfo { let mut fi = FileInfo::new(object, 2, 2); fi.volume = "bucket".to_string(); diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 1ab4f0922..d6a5cf6a0 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -15,7 +15,7 @@ use super::*; use crate::set_disk::{ get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold, - is_object_lock_diag_enabled, + is_lock_optimization_enabled, is_object_lock_diag_enabled, }; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; use rustfs_io_metrics::{ @@ -442,7 +442,7 @@ impl ECStore { } fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option) -> GetObjectReader { - if reader.buffered_body.is_some() { + if is_lock_optimization_enabled() || reader.buffered_body.is_some() { return reader; } @@ -1978,7 +1978,7 @@ mod tests { #[tokio::test] #[serial_test::serial] - async fn reader_lock_is_held_for_stream_when_optimization_is_enabled() { + async fn reader_lock_is_not_held_for_stream_when_optimization_is_enabled() { temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async { let manager = Arc::new(rustfs_lock::GlobalLockManager::new()); let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager); @@ -2004,13 +2004,10 @@ mod tests { let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard)); - lock.get_write_lock(key.clone(), "writer", Duration::from_millis(20)) - .await - .expect_err("streaming reader should hold the read lock"); - drop(reader); lock.get_write_lock(key, "writer", Duration::from_secs(1)) .await - .expect("dropping the reader should release the read lock"); + .expect("lock optimization should release the read lock before returning the stream"); + drop(reader); }) .await; } diff --git a/crates/io-metrics/src/internode_metrics.rs b/crates/io-metrics/src/internode_metrics.rs index 1593a8c50..9e9dbf6a3 100644 --- a/crates/io-metrics/src/internode_metrics.rs +++ b/crates/io-metrics/src/internode_metrics.rs @@ -33,6 +33,7 @@ const BACKEND_LABEL: &str = "backend"; const CLASSIFICATION_LABEL: &str = "classification"; const STAGE_LABEL: &str = "stage"; const DOMINANT_ERROR_LABEL: &str = "dominant_error"; +const HTTP_VERSION_LABEL: &str = "http_version"; const INTERNODE_OPERATION_SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_sent_bytes_total"; const INTERNODE_OPERATION_RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_recv_bytes_total"; const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_outgoing_total"; @@ -42,6 +43,10 @@ const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_o const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total"; const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total"; const INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL: &str = "rustfs_system_network_internode_operation_retry_successes_total"; +const INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL: &str = "rustfs_system_network_internode_operation_http_versions_total"; +const INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL: &str = "rustfs_system_network_internode_operation_stall_timeouts_total"; +const INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL: &str = + "rustfs_system_network_internode_operation_write_shutdown_errors_total"; const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -52,6 +57,7 @@ pub struct InternodeOperationMetricDescriptor { const OPERATION_BACKEND_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL]; const OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]; +const OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL]; const QUORUM_FAILURE_LABELS: &[&str] = &[STAGE_LABEL, DOMINANT_ERROR_LABEL]; pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &[ @@ -91,6 +97,18 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = & name: INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL, labels: OPERATION_BACKEND_CLASSIFICATION_LABELS, }, + InternodeOperationMetricDescriptor { + name: INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL, + labels: OPERATION_BACKEND_HTTP_VERSION_LABELS, + }, + InternodeOperationMetricDescriptor { + name: INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL, + labels: OPERATION_BACKEND_LABELS, + }, + InternodeOperationMetricDescriptor { + name: INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL, + labels: OPERATION_BACKEND_LABELS, + }, InternodeOperationMetricDescriptor { name: ERASURE_WRITE_QUORUM_FAILURES_TOTAL, labels: QUORUM_FAILURE_LABELS, @@ -107,6 +125,9 @@ pub struct InternodeMetricsSnapshot { pub dial_errors_total: u64, pub dial_avg_time_nanos: u64, pub last_dial_unix_millis: u64, + pub operation_http_versions_total: u64, + pub operation_stall_timeouts_total: u64, + pub operation_write_shutdown_errors_total: u64, } #[derive(Debug, Default)] @@ -120,6 +141,9 @@ pub struct InternodeMetrics { dial_total_time_nanos: AtomicU64, dial_samples_total: AtomicU64, last_dial_unix_millis: AtomicU64, + operation_http_versions_total: AtomicU64, + operation_stall_timeouts_total: AtomicU64, + operation_write_shutdown_errors_total: AtomicU64, } impl InternodeMetrics { @@ -264,6 +288,33 @@ impl InternodeMetrics { .increment(1); } + pub fn record_http_version_for_operation_and_backend( + &self, + operation: &'static str, + backend: &'static str, + http_version: &'static str, + ) { + self.operation_http_versions_total.fetch_add(1, Ordering::Relaxed); + counter!( + INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL, + OPERATION_LABEL => operation, + BACKEND_LABEL => backend, + HTTP_VERSION_LABEL => http_version + ) + .increment(1); + } + + pub fn record_stall_timeout_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) { + self.operation_stall_timeouts_total.fetch_add(1, Ordering::Relaxed); + counter!(INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1); + } + + pub fn record_write_shutdown_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) { + self.operation_write_shutdown_errors_total.fetch_add(1, Ordering::Relaxed); + counter!(INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend) + .increment(1); + } + pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) { counter!( ERASURE_WRITE_QUORUM_FAILURES_TOTAL, @@ -307,6 +358,9 @@ impl InternodeMetrics { dial_errors_total: self.dial_errors_total.load(Ordering::Relaxed), dial_avg_time_nanos, last_dial_unix_millis: self.last_dial_unix_millis.load(Ordering::Relaxed), + operation_http_versions_total: self.operation_http_versions_total.load(Ordering::Relaxed), + operation_stall_timeouts_total: self.operation_stall_timeouts_total.load(Ordering::Relaxed), + operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed), } } @@ -321,6 +375,9 @@ impl InternodeMetrics { self.dial_total_time_nanos.store(0, Ordering::Relaxed); self.dial_samples_total.store(0, Ordering::Relaxed); self.last_dial_unix_millis.store(0, Ordering::Relaxed); + self.operation_http_versions_total.store(0, Ordering::Relaxed); + self.operation_stall_timeouts_total.store(0, Ordering::Relaxed); + self.operation_write_shutdown_errors_total.store(0, Ordering::Relaxed); } } @@ -393,14 +450,21 @@ mod tests { #[test] fn operation_metric_descriptors_include_backend_and_operation_labels() { - assert_eq!(INTERNODE_OPERATION_METRICS.len(), 10); + assert_eq!(INTERNODE_OPERATION_METRICS.len(), 13); for metric in &INTERNODE_OPERATION_METRICS[..6] { assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]); } for metric in &INTERNODE_OPERATION_METRICS[6..9] { assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]); } - assert_eq!(INTERNODE_OPERATION_METRICS[9].labels, &[STAGE_LABEL, DOMINANT_ERROR_LABEL]); + assert_eq!( + INTERNODE_OPERATION_METRICS[9].labels, + &[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL] + ); + for metric in &INTERNODE_OPERATION_METRICS[10..12] { + assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]); + } + assert_eq!(INTERNODE_OPERATION_METRICS[12].labels, &[STAGE_LABEL, DOMINANT_ERROR_LABEL]); } #[test] @@ -433,6 +497,18 @@ mod tests { ); assert_eq!( INTERNODE_OPERATION_METRICS[9].name, + "rustfs_system_network_internode_operation_http_versions_total" + ); + assert_eq!( + INTERNODE_OPERATION_METRICS[10].name, + "rustfs_system_network_internode_operation_stall_timeouts_total" + ); + assert_eq!( + INTERNODE_OPERATION_METRICS[11].name, + "rustfs_system_network_internode_operation_write_shutdown_errors_total" + ); + assert_eq!( + INTERNODE_OPERATION_METRICS[12].name, "rustfs_system_storage_erasure_write_quorum_failures_total" ); } @@ -456,6 +532,19 @@ mod tests { INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, "connection_reset", ); + metrics.record_http_version_for_operation_and_backend( + INTERNODE_OPERATION_PUT_FILE_STREAM, + INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + "http/1.1", + ); + metrics.record_stall_timeout_for_operation_and_backend( + INTERNODE_OPERATION_READ_FILE_STREAM, + INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + ); + metrics.record_write_shutdown_error_for_operation_and_backend( + INTERNODE_OPERATION_PUT_FILE_STREAM, + INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + ); metrics.record_erasure_write_quorum_failure("write", "connection_reset"); let snapshot = metrics.snapshot(); @@ -463,5 +552,8 @@ mod tests { assert_eq!(snapshot.recv_bytes_total, 0); assert_eq!(snapshot.outgoing_requests_total, 0); assert_eq!(snapshot.incoming_requests_total, 0); + assert_eq!(snapshot.operation_http_versions_total, 1); + assert_eq!(snapshot.operation_stall_timeouts_total, 1); + assert_eq!(snapshot.operation_write_shutdown_errors_total, 1); } } diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index f9af5d945..b7c2d51ed 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -1098,6 +1098,19 @@ pub fn record_get_object_shard_locality_policy_disabled(path: &'static str) { } counter!("rustfs_io_get_object_shard_locality_policy_disabled_total", "path" => path).increment(1); } + +/// Record observe-only shard-locality potential while scheduling remains disabled. +#[inline(always)] +pub fn record_get_object_shard_locality_observe_only(path: &'static str, remote_scheduled: usize, remote_avoid_potential: usize) { + if !get_stage_metrics_enabled() { + return; + } + histogram!("rustfs_io_get_object_shard_remote_scheduled_observe_only", "path" => path) + .record(shard_read_fanout_to_f64(remote_scheduled)); + histogram!("rustfs_io_get_object_shard_remote_avoid_potential", "path" => path) + .record(shard_read_fanout_to_f64(remote_avoid_potential)); +} + /// Record per-stripe shard-read fanout shape for GetObject read-path attribution. #[inline(always)] pub fn record_get_object_shard_read_fanout( @@ -1116,6 +1129,70 @@ pub fn record_get_object_shard_read_fanout( histogram!("rustfs_io_get_object_shard_read_failed", "path" => path).record(shard_read_fanout_to_f64(failed)); } +fn batch_processor_count_to_f64(value: usize) -> f64 { + u64::try_from(value).unwrap_or(u64::MAX) as f64 +} + +fn batch_processor_count_to_u64(value: usize) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} + +/// Observe-only batch processor shape and adaptive-concurrency advice. +#[derive(Clone, Copy, Debug)] +pub struct BatchProcessorObservation { + pub operation: &'static str, + pub batch_size: usize, + pub configured_concurrency: usize, + pub max_queue_wait_secs: f64, + pub execution_latency_secs: f64, + pub successes: usize, + pub errors: usize, + pub timeouts: usize, + pub suggested_concurrency: usize, + pub suggestion_reason: &'static str, +} + +/// Record observe-only batch processor shape and adaptive-concurrency advice. +#[inline(always)] +pub fn record_batch_processor_observation(observation: BatchProcessorObservation) { + if !get_stage_metrics_enabled() { + return; + } + + histogram!("rustfs_ecstore_batch_processor_batch_size", "operation" => observation.operation) + .record(batch_processor_count_to_f64(observation.batch_size)); + histogram!("rustfs_ecstore_batch_processor_configured_concurrency", "operation" => observation.operation) + .record(batch_processor_count_to_f64(observation.configured_concurrency)); + histogram!("rustfs_ecstore_batch_processor_queue_wait_seconds", "operation" => observation.operation) + .record(observation.max_queue_wait_secs); + histogram!("rustfs_ecstore_batch_processor_execution_latency_seconds", "operation" => observation.operation) + .record(observation.execution_latency_secs); + counter!( + "rustfs_ecstore_batch_processor_results_total", + "operation" => observation.operation, + "outcome" => "success" + ) + .increment(batch_processor_count_to_u64(observation.successes)); + counter!( + "rustfs_ecstore_batch_processor_results_total", + "operation" => observation.operation, + "outcome" => "error" + ) + .increment(batch_processor_count_to_u64(observation.errors)); + counter!( + "rustfs_ecstore_batch_processor_results_total", + "operation" => observation.operation, + "outcome" => "timeout" + ) + .increment(batch_processor_count_to_u64(observation.timeouts)); + histogram!( + "rustfs_ecstore_batch_processor_suggested_concurrency", + "operation" => observation.operation, + "reason" => observation.suggestion_reason + ) + .record(batch_processor_count_to_f64(observation.suggested_concurrency)); +} + /// Record the bitrot reader setup scheduling strategy selected for a GET read. #[inline(always)] pub fn record_get_object_reader_setup_strategy(strategy: &'static str, mode: &'static str) { @@ -2033,6 +2110,26 @@ mod tests { record_bytespool_return("xlarge", "dropped"); } + #[test] + fn test_record_batch_processor_observation() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + set_get_stage_metrics_enabled(true); + record_batch_processor_observation(BatchProcessorObservation { + operation: "read", + batch_size: 16, + configured_concurrency: 8, + max_queue_wait_secs: 0.001, + execution_latency_secs: 0.025, + successes: 15, + errors: 1, + timeouts: 0, + suggested_concurrency: 10, + suggestion_reason: "improving", + }); + assert!(get_stage_metrics_enabled()); + set_get_stage_metrics_enabled(false); + } + #[test] fn test_record_zero_copy_write() { record_zero_copy_write(1024, 10.5); @@ -2094,6 +2191,7 @@ mod tests { record_get_object_pipeline_failure_for_path("codec_streaming", "decode", "read_quorum"); record_get_object_shard_read_observation("codec_streaming", 0, "data", "local", "success", "none", 1024, 0.004, 0.001); record_get_object_shard_read_cost_summary("codec_streaming", 3, 1, 2, 0, 4, 4, 4, true); + record_get_object_shard_locality_observe_only("codec_streaming", 2, 1); record_get_object_reader_setup_strategy("data_blocks_first", "read_quorum"); record_get_object_reader_setup_strategy_by_size( "codec_streaming", @@ -2115,10 +2213,32 @@ mod tests { 0, 2, ); + record_batch_processor_observation(BatchProcessorObservation { + operation: "read", + batch_size: 16, + configured_concurrency: 8, + max_queue_wait_secs: 0.001, + execution_latency_secs: 0.025, + successes: 15, + errors: 1, + timeouts: 0, + suggested_concurrency: 10, + suggestion_reason: "improving", + }); assert!(0.005_f64.is_sign_positive()); } + #[test] + fn test_get_object_shard_locality_observe_only_metrics_smoke() { + let remote_scheduled = 2; + let remote_avoid_potential = 1; + + record_get_object_shard_locality_observe_only("codec_streaming", remote_scheduled, remote_avoid_potential); + + assert!(remote_scheduled >= remote_avoid_potential); + } + #[test] fn test_record_get_object_fill_metrics() { record_get_object_fill_queued("codec_streaming", "single_inflight", 1); diff --git a/crates/protocols/src/sftp/server.rs b/crates/protocols/src/sftp/server.rs index dfbf8a1b3..8f3953912 100644 --- a/crates/protocols/src/sftp/server.rs +++ b/crates/protocols/src/sftp/server.rs @@ -36,8 +36,8 @@ use super::wedge_watchdog; use crate::common::client::s3::StorageBackend; use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext}; use russh::keys::{self, PrivateKey, PublicKeyBase64}; -use russh::server::{Auth, Msg, Session}; -use russh::{Channel, ChannelId, MethodKind, MethodSet, Pty, Sig}; +use russh::server::{Auth, ChannelOpenHandle, Msg, Session}; +use russh::{Channel, ChannelId, ChannelOpenFailure, MethodKind, MethodSet, Pty, Sig}; use rustfs_config::{ DEFAULT_SFTP_HOST_KEY_RELOAD_ENABLE, DEFAULT_SFTP_HOST_KEY_RELOAD_INTERVAL, ENV_SFTP_HOST_KEY_RELOAD_ENABLE, ENV_SFTP_HOST_KEY_RELOAD_INTERVAL, @@ -1135,15 +1135,19 @@ impl russh::server::Handler for SshSe } } - #[tracing::instrument(level = "debug", skip(self, channel, _session), fields(peer = %self.peer_addr))] + #[tracing::instrument(level = "debug", skip(self, channel, reply, _session), fields(peer = %self.peer_addr))] fn channel_open_session( &mut self, channel: Channel, + reply: ChannelOpenHandle, _session: &mut Session, - ) -> impl std::future::Future> + Send { + ) -> impl std::future::Future> + Send { let id = channel.id(); self.channels.insert(id, channel); - async { Ok(true) } + async move { + reply.accept().await; + Ok(()) + } } #[tracing::instrument(level = "debug", skip(self, _session), fields(peer = %self.peer_addr, channel = ?channel))] @@ -1404,13 +1408,13 @@ impl russh::server::Handler for SshSe async { Ok(false) } } - // Channel-open rejections. russh 0.60 defaults all of these to - // Ok(false), but we override them explicitly with a warn log so + // Channel-open rejections. russh defaults to rejecting dropped channel-open + // handles, but we override them explicitly with a warn log so // (a) probe attempts are visible in operator logs and // (b) a future russh default flip cannot silently allow these // channel types. - #[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))] + #[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))] fn channel_open_direct_tcpip( &mut self, _channel: Channel, @@ -1418,12 +1422,16 @@ impl russh::server::Handler for SshSe port_to_connect: u32, _originator_address: &str, _originator_port: u32, + reply: ChannelOpenHandle, _session: &mut Session, - ) -> impl std::future::Future> + Send { - async { Ok(false) } + ) -> impl std::future::Future> + Send { + async move { + reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await; + Ok(()) + } } - #[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))] + #[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))] fn channel_open_forwarded_tcpip( &mut self, _channel: Channel, @@ -1431,30 +1439,42 @@ impl russh::server::Handler for SshSe port_to_connect: u32, _originator_address: &str, _originator_port: u32, + reply: ChannelOpenHandle, _session: &mut Session, - ) -> impl std::future::Future> + Send { - async { Ok(false) } + ) -> impl std::future::Future> + Send { + async move { + reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await; + Ok(()) + } } - #[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr))] + #[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr))] fn channel_open_x11( &mut self, _channel: Channel, _originator_address: &str, _originator_port: u32, + reply: ChannelOpenHandle, _session: &mut Session, - ) -> impl std::future::Future> + Send { - async { Ok(false) } + ) -> impl std::future::Future> + Send { + async move { + reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await; + Ok(()) + } } - #[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, socket = %socket_path))] + #[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, socket = %socket_path))] fn channel_open_direct_streamlocal( &mut self, _channel: Channel, socket_path: &str, + reply: ChannelOpenHandle, _session: &mut Session, - ) -> impl std::future::Future> + Send { - async { Ok(false) } + ) -> impl std::future::Future> + Send { + async move { + reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await; + Ok(()) + } } } diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index d5e0021ae..773603369 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -531,6 +531,26 @@ pub struct ReadVersionResponse { pub file_info_bin: ::prost::bytes::Bytes, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct BatchReadVersionRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub batch_read_version_req: ::prost::alloc::string::String, + #[prost(bytes = "bytes", tag = "3")] + pub batch_read_version_req_bin: ::prost::bytes::Bytes, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct BatchReadVersionResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, repeated, tag = "2")] + pub batch_read_version_resps: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(message, optional, tag = "3")] + pub error: ::core::option::Option, + #[prost(bytes = "bytes", repeated, tag = "4")] + pub batch_read_version_resps_bin: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadXlRequest { #[prost(string, tag = "1")] pub disk: ::prost::alloc::string::String, @@ -1689,6 +1709,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "ReadVersion")); self.inner.unary(req, path, codec).await } + pub async fn batch_read_version( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/BatchReadVersion"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "BatchReadVersion")); + self.inner.unary(req, path, codec).await + } pub async fn read_xl( &mut self, request: impl tonic::IntoRequest, @@ -2610,6 +2645,10 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn batch_read_version( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn read_xl( &self, request: tonic::Request, @@ -3706,6 +3745,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/BatchReadVersion" => { + #[allow(non_camel_case_types)] + struct BatchReadVersionSvc(pub Arc); + impl tonic::server::UnaryService for BatchReadVersionSvc { + type Response = super::BatchReadVersionResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::batch_read_version(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = BatchReadVersionSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/ReadXL" => { #[allow(non_camel_case_types)] struct ReadXLSvc(pub Arc); diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index 83a46d048..b7f59cce3 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -371,6 +371,19 @@ message ReadVersionResponse { bytes file_info_bin = 4; } +message BatchReadVersionRequest { + string disk = 1; + string batch_read_version_req = 2; + bytes batch_read_version_req_bin = 3; +} + +message BatchReadVersionResponse { + bool success = 1; + repeated string batch_read_version_resps = 2; + optional Error error = 3; + repeated bytes batch_read_version_resps_bin = 4; +} + message ReadXLRequest { string disk = 1; string volume = 2; @@ -868,6 +881,7 @@ service NodeService { rpc ReadMetadata(ReadMetadataRequest) returns (ReadMetadataResponse) {}; rpc WriteMetadata(WriteMetadataRequest) returns (WriteMetadataResponse) {}; rpc ReadVersion(ReadVersionRequest) returns (ReadVersionResponse) {}; + rpc BatchReadVersion(BatchReadVersionRequest) returns (BatchReadVersionResponse) {}; rpc ReadXL(ReadXLRequest) returns (ReadXLResponse) {}; rpc DeleteVersion(DeleteVersionRequest) returns (DeleteVersionResponse) {}; rpc DeleteVersions(DeleteVersionsRequest) returns (DeleteVersionsResponse) {}; diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index a58734c45..a5378c205 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -15,14 +15,14 @@ use crate::{EtagResolvable, HashReaderDetector, HashReaderMut}; use bytes::{Bytes, BytesMut}; use futures::{Stream, TryStreamExt as _}; -use http::HeaderMap; +use http::{HeaderMap, Version}; use pin_project_lite::pin_project; use reqwest::{Certificate, Client, Identity, Method, RequestBuilder}; use rustfs_io_metrics::internode_metrics::{ INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR, }; use rustfs_tls_runtime::load_cert_bundle_der_bytes; -use rustfs_utils::get_env_opt_str; +use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_opt_u64, get_env_opt_usize}; use rustls_pki_types::pem::PemObject; use std::io::IoSlice; use std::io::{self, Error}; @@ -31,7 +31,7 @@ use std::ops::Not as _; use std::pin::Pin; use std::sync::LazyLock; use std::task::{Context, Poll}; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio::sync::{Mutex, mpsc}; use tokio::time::{self, Sleep}; @@ -42,6 +42,11 @@ use tracing::{error, warn}; const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream"; const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream"; const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir"; +const HTTP_VERSION_09_LABEL: &str = "http/0.9"; +const HTTP_VERSION_10_LABEL: &str = "http/1.0"; +const HTTP_VERSION_11_LABEL: &str = "http/1.1"; +const HTTP_VERSION_2_LABEL: &str = "h2"; +const HTTP_VERSION_UNKNOWN_LABEL: &str = "unknown"; #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum InternodeHttpErrorKind { @@ -213,18 +218,192 @@ fn add_root_certificates_from_der(builder: reqwest::ClientBuilder, certs_der: &[ struct CachedClients { generation: u64, client: Client, - local_client: Client, + no_proxy_client: Client, } static CLIENT_CACHE: LazyLock>> = LazyLock::new(|| Mutex::new(None)); -async fn build_http_client(disable_proxy: bool, outbound_tls: &rustfs_tls_runtime::GlobalPublishedOutboundTlsState) -> Client { +const INTERNODE_HTTP_PROFILE_LEGACY: &str = "legacy"; +const INTERNODE_HTTP_PROFILE_BALANCED: &str = "balanced"; +const INTERNODE_HTTP_PROFILE_THROUGHPUT: &str = "throughput"; +const INTERNODE_HTTP_PROXY_LEGACY: &str = "legacy"; +const INTERNODE_HTTP_PROXY_OFF: &str = "off"; +const INTERNODE_HTTP_PROXY_SYSTEM: &str = "system"; +const INTERNODE_HTTP2_WINDOW_MIN: u32 = 65_535; +const INTERNODE_HTTP2_WINDOW_MAX: u32 = 64 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum InternodeHttpTuningProfile { + Legacy, + Balanced, + Throughput, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum InternodeHttpProxyMode { + Legacy, + NoProxy, + System, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct InternodeHttpClientTuning { + profile: InternodeHttpTuningProfile, + pool_max_idle_per_host: Option, + pool_idle_timeout_secs: Option, + http2_initial_stream_window_size: Option, + http2_initial_connection_window_size: Option, + http2_adaptive_window: bool, + proxy_mode: InternodeHttpProxyMode, +} + +fn internode_http_client_tuning() -> InternodeHttpClientTuning { + // The tuning env vars cannot change at runtime; parse them once instead of + // re-reading the environment on every internode request. + static TUNING: LazyLock = LazyLock::new(InternodeHttpClientTuning::from_env); + *TUNING +} + +impl InternodeHttpClientTuning { + fn from_env() -> Self { + let profile = + parse_internode_http_tuning_profile(get_env_opt_str(rustfs_config::ENV_INTERNODE_HTTP_TUNING_PROFILE).as_deref()); + Self::from_values( + profile, + get_env_opt_usize(rustfs_config::ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST), + get_env_opt_u64(rustfs_config::ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS), + get_env_opt_u64(rustfs_config::ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE), + get_env_opt_u64(rustfs_config::ENV_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE), + get_env_bool( + rustfs_config::ENV_INTERNODE_HTTP2_ADAPTIVE_WINDOW, + profile.default_http2_adaptive_window(), + ), + get_env_opt_str(rustfs_config::ENV_INTERNODE_HTTP_PROXY).as_deref(), + ) + } + + fn from_values( + profile: InternodeHttpTuningProfile, + pool_max_idle_per_host: Option, + pool_idle_timeout_secs: Option, + stream_window_size: Option, + connection_window_size: Option, + http2_adaptive_window: bool, + proxy_mode: Option<&str>, + ) -> Self { + Self { + profile, + pool_max_idle_per_host: pool_max_idle_per_host.or_else(|| profile.default_pool_max_idle_per_host()), + pool_idle_timeout_secs: pool_idle_timeout_secs.or_else(|| profile.default_pool_idle_timeout_secs()), + http2_initial_stream_window_size: clamp_http2_window( + stream_window_size.or_else(|| profile.default_stream_window_size()), + ), + http2_initial_connection_window_size: clamp_http2_window( + connection_window_size.or_else(|| profile.default_connection_window_size()), + ), + http2_adaptive_window, + proxy_mode: parse_internode_http_proxy_mode(proxy_mode, profile), + } + } +} + +impl InternodeHttpTuningProfile { + fn default_pool_max_idle_per_host(self) -> Option { + match self { + Self::Legacy => None, + Self::Balanced => Some(64), + Self::Throughput => Some(256), + } + } + + fn default_pool_idle_timeout_secs(self) -> Option { + match self { + Self::Legacy => None, + Self::Balanced => Some(120), + Self::Throughput => Some(300), + } + } + + fn default_stream_window_size(self) -> Option { + match self { + Self::Legacy => None, + Self::Balanced => Some(1024 * 1024), + Self::Throughput => Some(4 * 1024 * 1024), + } + } + + fn default_connection_window_size(self) -> Option { + match self { + Self::Legacy => None, + Self::Balanced => Some(4 * 1024 * 1024), + Self::Throughput => Some(16 * 1024 * 1024), + } + } + + fn default_http2_adaptive_window(self) -> bool { + matches!(self, Self::Throughput) + } +} + +fn parse_internode_http_tuning_profile(value: Option<&str>) -> InternodeHttpTuningProfile { + match value.map(|value| value.trim().to_ascii_lowercase()) { + Some(value) if value == INTERNODE_HTTP_PROFILE_LEGACY => InternodeHttpTuningProfile::Legacy, + Some(value) if value == INTERNODE_HTTP_PROFILE_BALANCED => InternodeHttpTuningProfile::Balanced, + Some(value) if value == INTERNODE_HTTP_PROFILE_THROUGHPUT => InternodeHttpTuningProfile::Throughput, + _ => InternodeHttpTuningProfile::Legacy, + } +} + +fn parse_internode_http_proxy_mode(value: Option<&str>, profile: InternodeHttpTuningProfile) -> InternodeHttpProxyMode { + match value.map(|value| value.trim().to_ascii_lowercase()) { + Some(value) if value == INTERNODE_HTTP_PROXY_SYSTEM => InternodeHttpProxyMode::System, + Some(value) if value == INTERNODE_HTTP_PROXY_LEGACY => InternodeHttpProxyMode::Legacy, + Some(value) if matches!(value.as_str(), INTERNODE_HTTP_PROXY_OFF | "none" | "no_proxy" | "disabled") => { + InternodeHttpProxyMode::NoProxy + } + Some(_) | None if matches!(profile, InternodeHttpTuningProfile::Legacy) => InternodeHttpProxyMode::Legacy, + Some(_) | None => InternodeHttpProxyMode::NoProxy, + } +} + +fn clamp_http2_window(value: Option) -> Option { + let value = value?; + let value = value.clamp(u64::from(INTERNODE_HTTP2_WINDOW_MIN), u64::from(INTERNODE_HTTP2_WINDOW_MAX)); + u32::try_from(value).ok() +} + +fn apply_http_client_tuning(mut builder: reqwest::ClientBuilder, tuning: InternodeHttpClientTuning) -> reqwest::ClientBuilder { + if let Some(pool_max_idle_per_host) = tuning.pool_max_idle_per_host { + builder = builder.pool_max_idle_per_host(pool_max_idle_per_host); + } + if let Some(pool_idle_timeout_secs) = tuning.pool_idle_timeout_secs { + builder = builder.pool_idle_timeout(std::time::Duration::from_secs(pool_idle_timeout_secs)); + } + if tuning.http2_adaptive_window { + builder = builder.http2_adaptive_window(true); + } else { + if let Some(stream_window_size) = tuning.http2_initial_stream_window_size { + builder = builder.http2_initial_stream_window_size(stream_window_size); + } + if let Some(connection_window_size) = tuning.http2_initial_connection_window_size { + builder = builder.http2_initial_connection_window_size(connection_window_size); + } + } + builder +} + +async fn build_http_client( + disable_proxy: bool, + tuning: InternodeHttpClientTuning, + outbound_tls: &rustfs_tls_runtime::GlobalPublishedOutboundTlsState, +) -> Client { let mut builder = Client::builder() .connect_timeout(std::time::Duration::from_secs(5)) .tcp_keepalive(std::time::Duration::from_secs(10)) .http2_keep_alive_interval(std::time::Duration::from_secs(5)) .http2_keep_alive_timeout(std::time::Duration::from_secs(3)) .http2_keep_alive_while_idle(true); + builder = apply_http_client_tuning(builder, tuning); if disable_proxy { builder = builder.no_proxy(); @@ -292,10 +471,19 @@ fn should_bypass_proxy_for_url(url: &str) -> bool { host.eq_ignore_ascii_case("localhost") || host.parse::().is_ok_and(|addr| addr.is_loopback()) } +fn should_disable_proxy_for_url(url: &str, tuning: InternodeHttpClientTuning) -> bool { + match tuning.proxy_mode { + InternodeHttpProxyMode::System => false, + InternodeHttpProxyMode::NoProxy => true, + InternodeHttpProxyMode::Legacy => should_bypass_proxy_for_url(url), + } +} + async fn get_http_client(url: &str) -> Client { - // Reuse HTTP connection pools while keeping loopback traffic away from - // system proxies so local RPC/tests do not leak to proxy listeners. - let disable_proxy = should_bypass_proxy_for_url(url); + let tuning = internode_http_client_tuning(); + // Reuse HTTP connection pools while honoring the configured internode proxy + // policy. The legacy profile only bypasses loopback URLs to preserve defaults. + let disable_proxy = should_disable_proxy_for_url(url, tuning); // Fast path: check generation first (cheap atomic read) to avoid cloning // the full PEM + identity bytes when the TLS state hasn't changed. @@ -305,7 +493,7 @@ async fn get_http_client(url: &str) -> Client { if let Some(cached) = guard.as_ref() { if cached.generation == generation { return if disable_proxy { - cached.local_client.clone() + cached.no_proxy_client.clone() } else { cached.client.clone() }; @@ -317,16 +505,16 @@ async fn get_http_client(url: &str) -> Client { // Cache miss or stale generation — load full outbound TLS state. let outbound_tls = crate::http_runtime_sources::outbound_tls_state().await; - let client = build_http_client(false, &outbound_tls).await; - let local_client = build_http_client(true, &outbound_tls).await; + let client = build_http_client(false, tuning, &outbound_tls).await; + let no_proxy_client = build_http_client(true, tuning, &outbound_tls).await; let cached = CachedClients { generation, client, - local_client, + no_proxy_client, }; let return_client = if disable_proxy { - cached.local_client.clone() + cached.no_proxy_client.clone() } else { cached.client.clone() }; @@ -419,6 +607,8 @@ pin_project! { internode_operation: Option<&'static str>, stall_timeout: Option, stall_timer: Option>>, + request_started: Instant, + duration_recorded: bool, #[pin] inner: StreamReader>+Send+Sync>>, Bytes>, } @@ -467,13 +657,17 @@ impl HttpReader { request = request.body(body); } + let request_started = Instant::now(); let resp = request.send().await.map_err(|e| { + record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed()); record_internode_error(track_internode_metrics, internode_operation); record_internode_classified_error(track_internode_metrics, internode_operation, classify_reqwest_error(&e)); internode_reqwest_error(&method, &url, internode_operation, e) })?; + record_internode_http_version(track_internode_metrics, internode_operation, http_version_metric_label(resp.version())); if resp.status().is_success().not() { + record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed()); record_internode_error(track_internode_metrics, internode_operation); record_internode_classified_error(track_internode_metrics, internode_operation, classify_http_status(resp.status())); return Err(internode_status_error(&method, &url, internode_operation, resp.status())); @@ -498,6 +692,8 @@ impl HttpReader { internode_operation, stall_timer: None, stall_timeout, + request_started, + duration_recorded: false, }) } pub fn url(&self) -> &str { @@ -521,6 +717,13 @@ impl AsyncRead for HttpReader { let bytes_read = buf.filled().len().saturating_sub(filled_before); if bytes_read > 0 { record_internode_recv_bytes(*this.track_internode_metrics, *this.internode_operation, bytes_read); + } else { + record_internode_operation_duration_once( + *this.track_internode_metrics, + *this.internode_operation, + *this.request_started, + this.duration_recorded, + ); } *this.stall_timer = None; Poll::Ready(Ok(())) @@ -531,6 +734,13 @@ impl AsyncRead for HttpReader { }; let timer = this.stall_timer.get_or_insert_with(|| Box::pin(time::sleep(stall_timeout))); if timer.as_mut().poll(cx).is_ready() { + record_internode_operation_duration_once( + *this.track_internode_metrics, + *this.internode_operation, + *this.request_started, + this.duration_recorded, + ); + record_internode_stall_timeout(*this.track_internode_metrics, *this.internode_operation); record_internode_error(*this.track_internode_metrics, *this.internode_operation); Poll::Ready(Err(Error::new( io::ErrorKind::TimedOut, @@ -540,7 +750,15 @@ impl AsyncRead for HttpReader { Poll::Pending } } - other => other, + Poll::Ready(Err(err)) => { + record_internode_operation_duration_once( + *this.track_internode_metrics, + *this.internode_operation, + *this.request_started, + this.duration_recorded, + ); + Poll::Ready(Err(err)) + } } } } @@ -610,6 +828,8 @@ pin_project! { handle: tokio::task::JoinHandle>, pending_chunk: BytesMut, finish:bool, + track_internode_metrics: bool, + internode_operation: Option<&'static str>, } } @@ -648,10 +868,17 @@ impl HttpWriter { .body(body); // Hold the request until the shutdown signal is received + let request_started = Instant::now(); let response = request.send().await; match response { Ok(resp) => { + record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed()); + record_internode_http_version( + track_internode_metrics, + internode_operation, + http_version_metric_label(resp.version()), + ); // http_log!("[HttpWriter::spawn] got response: status={}", resp.status()); if !resp.status().is_success() { record_internode_error(track_internode_metrics, internode_operation); @@ -667,6 +894,7 @@ impl HttpWriter { } } Err(e) => { + record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed()); record_internode_error(track_internode_metrics, internode_operation); let classified = classify_reqwest_error(&e); record_internode_classified_error(track_internode_metrics, internode_operation, classified); @@ -691,6 +919,8 @@ impl HttpWriter { handle, pending_chunk: BytesMut::with_capacity(HTTP_WRITER_BUFFER_SIZE), finish: false, + track_internode_metrics, + internode_operation, }) } @@ -721,6 +951,20 @@ fn internode_rpc_operation(url: &str) -> Option<&'static str> { } } +fn http_version_metric_label(version: Version) -> &'static str { + if version == Version::HTTP_09 { + HTTP_VERSION_09_LABEL + } else if version == Version::HTTP_10 { + HTTP_VERSION_10_LABEL + } else if version == Version::HTTP_11 { + HTTP_VERSION_11_LABEL + } else if version == Version::HTTP_2 { + HTTP_VERSION_2_LABEL + } else { + HTTP_VERSION_UNKNOWN_LABEL + } +} + fn record_internode_outgoing_request(track: bool, operation: Option<&'static str>) { if !track { return; @@ -763,6 +1007,60 @@ fn record_internode_classified_error(track: bool, operation: Option<&'static str } } +fn record_internode_operation_duration(track: bool, operation: Option<&'static str>, duration: Duration) { + if !track { + return; + } + + if let Some(operation) = operation { + crate::http_runtime_sources::record_duration(operation, duration); + } +} + +fn record_internode_operation_duration_once( + track: bool, + operation: Option<&'static str>, + request_started: Instant, + duration_recorded: &mut bool, +) { + if *duration_recorded { + return; + } + + *duration_recorded = true; + record_internode_operation_duration(track, operation, request_started.elapsed()); +} + +fn record_internode_http_version(track: bool, operation: Option<&'static str>, http_version: &'static str) { + if !track { + return; + } + + if let Some(operation) = operation { + crate::http_runtime_sources::record_http_version(operation, http_version); + } +} + +fn record_internode_stall_timeout(track: bool, operation: Option<&'static str>) { + if !track { + return; + } + + if let Some(operation) = operation { + crate::http_runtime_sources::record_stall_timeout(operation); + } +} + +fn record_internode_write_shutdown_error(track: bool, operation: Option<&'static str>) { + if !track { + return; + } + + if let Some(operation) = operation { + crate::http_runtime_sources::record_write_shutdown_error(operation); + } +} + fn poll_send_error_to_io(err: tokio_util::sync::PollSendError, context: &str) -> io::Error { Error::other(format!("{context}: {err}")) } @@ -772,6 +1070,13 @@ fn send_error_to_io(err: tokio_util::sync::PollSendError, context: &str) - } impl HttpWriter { + fn take_background_error(&mut self) -> io::Result<()> { + match self.err_rx.try_recv() { + Ok(err) => Err(err), + Err(tokio::sync::oneshot::error::TryRecvError::Empty | tokio::sync::oneshot::error::TryRecvError::Closed) => Ok(()), + } + } + fn poll_send_pending_chunk(&mut self, cx: &mut Context<'_>) -> Poll> { if self.pending_chunk.is_empty() { return Poll::Ready(Ok(())); @@ -799,11 +1104,10 @@ impl AsyncWrite for HttpWriter { // self.method, // buf.len() // ); - if let Ok(e) = Pin::new(&mut self.err_rx).try_recv() { - return Poll::Ready(Err(e)); - } - let this = self.as_mut().get_mut(); + if let Err(err) = this.take_background_error() { + return Poll::Ready(Err(err)); + } if this.pending_chunk.len() >= HTTP_WRITER_BUFFER_SIZE { match this.poll_send_pending_chunk(cx) { @@ -832,15 +1136,19 @@ impl AsyncWrite for HttpWriter { } fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.as_mut().get_mut().poll_send_pending_chunk(cx) + let this = self.as_mut().get_mut(); + if let Err(err) = this.take_background_error() { + return Poll::Ready(Err(err)); + } + + this.poll_send_pending_chunk(cx) } fn poll_write_vectored(mut self: Pin<&mut Self>, cx: &mut Context<'_>, bufs: &[IoSlice<'_>]) -> Poll> { - if let Ok(e) = Pin::new(&mut self.err_rx).try_recv() { - return Poll::Ready(Err(e)); - } - let this = self.as_mut().get_mut(); + if let Err(err) = this.take_background_error() { + return Poll::Ready(Err(err)); + } if this.pending_chunk.len() >= HTTP_WRITER_BUFFER_SIZE { match this.poll_send_pending_chunk(cx) { @@ -883,9 +1191,17 @@ impl AsyncWrite for HttpWriter { // let url = self.url.clone(); // let method = self.method.clone(); + if let Err(err) = self.as_mut().get_mut().take_background_error() { + record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation); + return Poll::Ready(Err(err)); + } + match self.as_mut().get_mut().poll_send_pending_chunk(cx) { Poll::Ready(Ok(())) => {} - Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Ready(Err(err)) => { + record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation); + return Poll::Ready(Err(err)); + } Poll::Pending => return Poll::Pending, } @@ -894,11 +1210,15 @@ impl AsyncWrite for HttpWriter { let this = self.as_mut().get_mut(); match this.sender.poll_reserve(cx) { Poll::Ready(Ok(())) => { - this.sender - .send_item(None) - .map_err(|e| send_error_to_io(e, "HttpWriter shutdown error"))?; + this.sender.send_item(None).map_err(|e| { + record_internode_write_shutdown_error(this.track_internode_metrics, this.internode_operation); + send_error_to_io(e, "HttpWriter shutdown error") + })?; + } + Poll::Ready(Err(err)) => { + record_internode_write_shutdown_error(this.track_internode_metrics, this.internode_operation); + return Poll::Ready(Err(poll_send_error_to_io(err, "HttpWriter shutdown error"))); } - Poll::Ready(Err(err)) => return Poll::Ready(Err(poll_send_error_to_io(err, "HttpWriter shutdown error"))), Poll::Pending => return Poll::Pending, } // http_log!( @@ -911,15 +1231,20 @@ impl AsyncWrite for HttpWriter { } // Wait for the HTTP request to complete use futures::FutureExt; - match Pin::new(&mut self.get_mut().handle).poll_unpin(cx) { - Poll::Ready(Ok(_)) => { + match Pin::new(&mut self.as_mut().get_mut().handle).poll_unpin(cx) { + Poll::Ready(Ok(Ok(()))) => { // http_log!( // "[HttpWriter::poll_shutdown] HTTP request finished successfully, url: {}, method: {:?}", // url, // method // ); } + Poll::Ready(Ok(Err(err))) => { + record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation); + return Poll::Ready(Err(err)); + } Poll::Ready(Err(e)) => { + record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation); // http_log!("[HttpWriter::poll_shutdown] HTTP request failed: {e}, url: {}, method: {:?}", url, method); return Poll::Ready(Err(Error::other(format!("HTTP request failed: {e}")))); } @@ -992,6 +1317,13 @@ mod tests { StatusCode::OK } + async fn reject_put(State(state): State, body: Body) -> impl IntoResponse { + state.put_count.fetch_add(1, Ordering::SeqCst); + let bytes = body.collect().await.unwrap().to_bytes(); + state.put_bodies.lock().await.push(bytes.to_vec()); + StatusCode::INTERNAL_SERVER_ERROR + } + async fn start_test_server(state: TestState) -> Option<(String, tokio::task::JoinHandle<()>)> { let listener = match TcpListener::bind("127.0.0.1:0").await { Ok(listener) => listener, @@ -1001,6 +1333,7 @@ mod tests { let addr = listener.local_addr().expect("listener local address should be available"); let app = Router::new() .route("/stream", get(get_stream).head(reject_head).put(accept_put)) + .route("/reject-put", get(get_stream).put(reject_put)) .route("/stall", get(get_stalling_stream)) .route("/delayed-first", get(get_delayed_first_chunk)) .with_state(state); @@ -1033,6 +1366,14 @@ mod tests { ); } + #[test] + fn http_version_metrics_labels_are_low_cardinality() { + assert_eq!(http_version_metric_label(Version::HTTP_09), HTTP_VERSION_09_LABEL); + assert_eq!(http_version_metric_label(Version::HTTP_10), HTTP_VERSION_10_LABEL); + assert_eq!(http_version_metric_label(Version::HTTP_11), HTTP_VERSION_11_LABEL); + assert_eq!(http_version_metric_label(Version::HTTP_2), HTTP_VERSION_2_LABEL); + } + #[tokio::test] async fn http_reader_does_not_send_preflight_head() { let state = TestState::default(); @@ -1174,6 +1515,37 @@ mod tests { handle.abort(); } + #[tokio::test] + async fn http_writer_shutdown_reports_http_status_error() { + let state = TestState::default(); + let Some((base_url, handle)) = start_test_server(state.clone()).await else { + return; + }; + let url = base_url.replace("/stream", "/reject-put"); + + let mut writer = HttpWriter::new(url, Method::PUT, HeaderMap::new()).await.unwrap(); + writer.write_all(b"payload").await.unwrap(); + let err = writer + .shutdown() + .await + .expect_err("shutdown should report the HTTP response failure"); + + let source = err + .get_ref() + .and_then(|source| source.downcast_ref::()) + .expect("expected shutdown error to carry InternodeHttpError source"); + assert_eq!( + source.kind(), + InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR) + ); + assert_eq!(source.context().method(), "PUT"); + assert!(source.context().target().contains("/reject-put")); + assert_eq!(state.put_count.load(Ordering::SeqCst), 1); + assert_eq!(state.put_bodies.lock().await.as_slice(), &[b"payload".to_vec()]); + + handle.abort(); + } + #[tokio::test] async fn http_reader_request_error_includes_method_and_url() { let listener = match TcpListener::bind("127.0.0.1:0").await { @@ -1267,4 +1639,126 @@ mod tests { assert!(!should_bypass_proxy_for_url("http://example.com/stream")); assert!(!should_bypass_proxy_for_url("not-a-url")); } + + #[test] + fn internode_http_tuning_profile_parses_known_values_and_falls_back_to_legacy() { + assert_eq!(parse_internode_http_tuning_profile(None), InternodeHttpTuningProfile::Legacy); + assert_eq!( + parse_internode_http_tuning_profile(Some("balanced")), + InternodeHttpTuningProfile::Balanced + ); + assert_eq!( + parse_internode_http_tuning_profile(Some(" Throughput ")), + InternodeHttpTuningProfile::Throughput + ); + assert_eq!( + parse_internode_http_tuning_profile(Some("aggressive")), + InternodeHttpTuningProfile::Legacy + ); + } + + #[test] + fn legacy_internode_http_tuning_keeps_existing_defaults() { + let tuning = InternodeHttpClientTuning::from_values( + InternodeHttpTuningProfile::Legacy, + None, + None, + None, + None, + InternodeHttpTuningProfile::Legacy.default_http2_adaptive_window(), + None, + ); + + assert_eq!(tuning.pool_max_idle_per_host, None); + assert_eq!(tuning.pool_idle_timeout_secs, None); + assert_eq!(tuning.http2_initial_stream_window_size, None); + assert_eq!(tuning.http2_initial_connection_window_size, None); + assert!(!tuning.http2_adaptive_window); + assert_eq!(tuning.proxy_mode, InternodeHttpProxyMode::Legacy); + } + + #[test] + fn balanced_and_throughput_profiles_apply_conservative_defaults() { + let balanced = InternodeHttpClientTuning::from_values( + InternodeHttpTuningProfile::Balanced, + None, + None, + None, + None, + InternodeHttpTuningProfile::Balanced.default_http2_adaptive_window(), + None, + ); + let throughput = InternodeHttpClientTuning::from_values( + InternodeHttpTuningProfile::Throughput, + None, + None, + None, + None, + InternodeHttpTuningProfile::Throughput.default_http2_adaptive_window(), + None, + ); + + assert_eq!(balanced.pool_max_idle_per_host, Some(64)); + assert_eq!(balanced.pool_idle_timeout_secs, Some(120)); + assert_eq!(balanced.http2_initial_stream_window_size, Some(1024 * 1024)); + assert_eq!(balanced.http2_initial_connection_window_size, Some(4 * 1024 * 1024)); + assert!(!balanced.http2_adaptive_window); + assert_eq!(balanced.proxy_mode, InternodeHttpProxyMode::NoProxy); + + assert_eq!(throughput.pool_max_idle_per_host, Some(256)); + assert_eq!(throughput.pool_idle_timeout_secs, Some(300)); + assert!(throughput.http2_adaptive_window); + assert_eq!(throughput.proxy_mode, InternodeHttpProxyMode::NoProxy); + } + + #[test] + fn internode_http_tuning_overrides_are_clamped() { + let tuning = InternodeHttpClientTuning::from_values( + InternodeHttpTuningProfile::Balanced, + Some(8), + Some(30), + Some(1), + Some(u64::MAX), + false, + Some("system"), + ); + + assert_eq!(tuning.pool_max_idle_per_host, Some(8)); + assert_eq!(tuning.pool_idle_timeout_secs, Some(30)); + assert_eq!(tuning.http2_initial_stream_window_size, Some(INTERNODE_HTTP2_WINDOW_MIN)); + assert_eq!(tuning.http2_initial_connection_window_size, Some(INTERNODE_HTTP2_WINDOW_MAX)); + assert_eq!(tuning.proxy_mode, InternodeHttpProxyMode::System); + } + + #[test] + fn internode_http_proxy_policy_matches_profile_and_overrides() { + let legacy = + InternodeHttpClientTuning::from_values(InternodeHttpTuningProfile::Legacy, None, None, None, None, false, None); + let balanced = + InternodeHttpClientTuning::from_values(InternodeHttpTuningProfile::Balanced, None, None, None, None, false, None); + let system_proxy = InternodeHttpClientTuning::from_values( + InternodeHttpTuningProfile::Throughput, + None, + None, + None, + None, + true, + Some("system"), + ); + let no_proxy = InternodeHttpClientTuning::from_values( + InternodeHttpTuningProfile::Legacy, + None, + None, + None, + None, + false, + Some("off"), + ); + + assert!(should_disable_proxy_for_url("http://127.0.0.1:9000/stream", legacy)); + assert!(!should_disable_proxy_for_url("http://192.168.1.10:9000/stream", legacy)); + assert!(should_disable_proxy_for_url("http://192.168.1.10:9000/stream", balanced)); + assert!(!should_disable_proxy_for_url("http://127.0.0.1:9000/stream", system_proxy)); + assert!(should_disable_proxy_for_url("http://example.com/stream", no_proxy)); + } } diff --git a/crates/rio/src/http_runtime_sources.rs b/crates/rio/src/http_runtime_sources.rs index b8b6b6635..12837e0b3 100644 --- a/crates/rio/src/http_runtime_sources.rs +++ b/crates/rio/src/http_runtime_sources.rs @@ -17,6 +17,7 @@ use rustfs_tls_runtime::{ GlobalPublishedOutboundTlsState, load_global_outbound_tls_generation, load_global_outbound_tls_state, record_tls_consumer_stale_generation, }; +use std::time::Duration; pub(crate) fn outbound_tls_generation() -> u64 { load_global_outbound_tls_generation().0 @@ -76,3 +77,28 @@ pub(crate) fn record_classified_error(operation: &'static str, classification: & classification, ); } + +pub(crate) fn record_duration(operation: &'static str, duration: Duration) { + global_internode_metrics().record_duration_for_operation_and_backend( + operation, + INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + duration, + ); +} + +pub(crate) fn record_http_version(operation: &'static str, http_version: &'static str) { + global_internode_metrics().record_http_version_for_operation_and_backend( + operation, + INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + http_version, + ); +} + +pub(crate) fn record_stall_timeout(operation: &'static str) { + global_internode_metrics().record_stall_timeout_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP); +} + +pub(crate) fn record_write_shutdown_error(operation: &'static str) { + global_internode_metrics() + .record_write_shutdown_error_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP); +} diff --git a/rustfs/src/storage/concurrency/manager.rs b/rustfs/src/storage/concurrency/manager.rs index 71f7fa0df..543b05cf8 100644 --- a/rustfs/src/storage/concurrency/manager.rs +++ b/rustfs/src/storage/concurrency/manager.rs @@ -679,7 +679,6 @@ mod integration_tests { use super::ConcurrencyManager; use crate::storage::storage_api::concurrency_consumer::PutObjectGuard; use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass}; - use rustfs_config::MI_B; use rustfs_io_core::io_profile::{AccessPattern, StorageMedia}; use serial_test::serial; use std::time::Duration; @@ -989,13 +988,10 @@ mod integration_tests { StorageMedia::Hdd => config.hdd_buffer_cap, StorageMedia::Unknown => config.ssd_buffer_cap, }; - let expected_max = media_cap.min(MI_B); + let expected_max = media_cap; - // Large base buffer should be constrained by storage cap first, then global clamp. - assert_eq!( - strategy.buffer_size, expected_max, - "Buffer should be capped by media profile and global clamp" - ); + // Large base buffer should be constrained by the active storage media cap. + assert_eq!(strategy.buffer_size, expected_max, "Buffer should be capped by the active media profile"); } #[tokio::test] diff --git a/rustfs/src/storage/rpc/disk.rs b/rustfs/src/storage/rpc/disk.rs index fe94b93b8..b6da6e29b 100644 --- a/rustfs/src/storage/rpc/disk.rs +++ b/rustfs/src/storage/rpc/disk.rs @@ -14,8 +14,8 @@ use super::NodeService; use crate::storage::storage_api::rpc_consumer::node_service::{ - DeleteOptions, DiskError, DiskInfoOptions, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, - StorageDiskRpcExt as _, UpdateMetadataOpts, + BatchReadVersionReq, BatchReadVersionResp, DeleteOptions, DiskError, DiskInfoOptions, FileInfoVersions, ReadMultipleReq, + ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts, validate_batch_read_version_item_count, }; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; use bytes::Bytes; @@ -55,6 +55,40 @@ fn encode_msgpack_named(value: &T, value_name: &str) -> std Ok(serializer.into_inner()) } +fn encode_read_multiple_response_payloads( + read_multiple_resps: &[ReadMultipleResp], +) -> std::result::Result<(Vec, Vec), DiskError> { + let mut read_multiple_resps_json = Vec::with_capacity(read_multiple_resps.len()); + let mut read_multiple_resps_bin = Vec::with_capacity(read_multiple_resps.len()); + + for read_multiple_resp in read_multiple_resps { + read_multiple_resps_json.push( + serde_json::to_string(read_multiple_resp) + .map_err(|err| DiskError::other(format!("encode ReadMultipleResp json failed: {err}")))?, + ); + read_multiple_resps_bin.push(Bytes::from(encode_msgpack(read_multiple_resp, "ReadMultipleResp")?)); + } + + Ok((read_multiple_resps_json, read_multiple_resps_bin)) +} + +fn encode_batch_read_version_response_payloads( + batch_read_version_resps: &[BatchReadVersionResp], +) -> std::result::Result<(Vec, Vec), DiskError> { + let mut batch_read_version_resps_json = Vec::with_capacity(batch_read_version_resps.len()); + let mut batch_read_version_resps_bin = Vec::with_capacity(batch_read_version_resps.len()); + + for batch_read_version_resp in batch_read_version_resps { + batch_read_version_resps_json.push( + serde_json::to_string(batch_read_version_resp) + .map_err(|err| DiskError::other(format!("encode BatchReadVersionResp json failed: {err}")))?, + ); + batch_read_version_resps_bin.push(Bytes::from(encode_msgpack(batch_read_version_resp, "BatchReadVersionResp")?)); + } + + Ok((batch_read_version_resps_json, batch_read_version_resps_bin)) +} + impl NodeService { pub(super) async fn handle_disk_info(&self, request: Request) -> Result, Status> { let request = request.into_inner(); @@ -92,7 +126,7 @@ impl NodeService { Ok(Response::new(DiskInfoResponse { success: false, disk_info: "".to_string(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -116,7 +150,7 @@ impl NodeService { } else { Ok(Response::new(DeleteVolumeResponse { success: false, - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -144,16 +178,18 @@ impl NodeService { }; match disk.read_multiple(read_multiple_req).await { Ok(read_multiple_resps) => { - let read_multiple_resps: Vec = read_multiple_resps - .into_iter() - .filter_map(|read_multiple_resp| serde_json::to_string(&read_multiple_resp).ok()) - .collect(); - let read_multiple_resps_bin = read_multiple_resps - .iter() - .filter_map(|json_str| serde_json::from_str::(json_str).ok()) - .filter_map(|resp| encode_msgpack(&resp, "ReadMultipleResp").ok()) - .map(Into::into) - .collect(); + let (read_multiple_resps, read_multiple_resps_bin) = + match encode_read_multiple_response_payloads(&read_multiple_resps) { + Ok(payloads) => payloads, + Err(err) => { + return Ok(Response::new(ReadMultipleResponse { + success: false, + read_multiple_resps: Vec::new(), + read_multiple_resps_bin: Vec::new(), + error: Some(err.into()), + })); + } + }; Ok(Response::new(ReadMultipleResponse { success: true, @@ -174,7 +210,77 @@ impl NodeService { success: false, read_multiple_resps: Vec::new(), read_multiple_resps_bin: Vec::new(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), + })) + } + } + + pub(super) async fn handle_batch_read_version( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let batch_read_version_req: BatchReadVersionReq = match decode_msgpack_or_json( + &request.batch_read_version_req_bin, + &request.batch_read_version_req, + "BatchReadVersionReq", + ) { + Ok(batch_read_version_req) => batch_read_version_req, + Err(err) => { + return Ok(Response::new(BatchReadVersionResponse { + success: false, + batch_read_version_resps: Vec::new(), + batch_read_version_resps_bin: Vec::new(), + error: Some(DiskError::other(format!("decode BatchReadVersionReq failed: {err}")).into()), + })); + } + }; + + if let Err(err) = validate_batch_read_version_item_count(batch_read_version_req.items.len()) { + return Ok(Response::new(BatchReadVersionResponse { + success: false, + batch_read_version_resps: Vec::new(), + batch_read_version_resps_bin: Vec::new(), + error: Some(err.into()), + })); + } + + match disk.batch_read_version(batch_read_version_req).await { + Ok(batch_read_version_resps) => { + let (batch_read_version_resps, batch_read_version_resps_bin) = + match encode_batch_read_version_response_payloads(&batch_read_version_resps) { + Ok(payloads) => payloads, + Err(err) => { + return Ok(Response::new(BatchReadVersionResponse { + success: false, + batch_read_version_resps: Vec::new(), + batch_read_version_resps_bin: Vec::new(), + error: Some(err.into()), + })); + } + }; + + Ok(Response::new(BatchReadVersionResponse { + success: true, + batch_read_version_resps, + batch_read_version_resps_bin, + error: None, + })) + } + Err(err) => Ok(Response::new(BatchReadVersionResponse { + success: false, + batch_read_version_resps: Vec::new(), + batch_read_version_resps_bin: Vec::new(), + error: Some(err.into()), + })), + } + } else { + Ok(Response::new(BatchReadVersionResponse { + success: false, + batch_read_version_resps: Vec::new(), + batch_read_version_resps_bin: Vec::new(), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -228,7 +334,7 @@ impl NodeService { Ok(Response::new(DeleteVersionsResponse { success: false, errors: Vec::new(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -285,7 +391,7 @@ impl NodeService { Ok(Response::new(DeleteVersionResponse { success: false, raw_file_info: "".to_string(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -330,7 +436,7 @@ impl NodeService { success: false, raw_file_info: String::new(), raw_file_info_bin: Vec::new().into(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -392,7 +498,7 @@ impl NodeService { success: false, file_info: String::new(), file_info_bin: Vec::new().into(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -425,7 +531,7 @@ impl NodeService { } else { Ok(Response::new(WriteMetadataResponse { success: false, - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -469,7 +575,7 @@ impl NodeService { } else { Ok(Response::new(UpdateMetadataResponse { success: false, - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -496,7 +602,7 @@ impl NodeService { Ok(Response::new(ReadMetadataResponse { success: false, data: Bytes::new(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -520,7 +626,7 @@ impl NodeService { } else { Ok(Response::new(DeletePathsResponse { success: false, - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -554,7 +660,7 @@ impl NodeService { Ok(Response::new(StatVolumeResponse { success: false, volume_info: String::new(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -587,7 +693,7 @@ impl NodeService { Ok(Response::new(ListVolumesResponse { success: false, volume_infos: Vec::new(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -611,7 +717,7 @@ impl NodeService { } else { Ok(Response::new(MakeVolumeResponse { success: false, - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -635,7 +741,7 @@ impl NodeService { } else { Ok(Response::new(MakeVolumesResponse { success: false, - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -697,7 +803,7 @@ impl NodeService { success: false, rename_data_resp: String::new(), rename_data_resp_bin: Vec::new().into(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -721,7 +827,7 @@ impl NodeService { Ok(Response::new(ListDirResponse { success: false, volumes: Vec::new(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -752,7 +858,7 @@ impl NodeService { } else { Ok(Response::new(RenameFileResponse { success: false, - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -785,7 +891,7 @@ impl NodeService { } else { Ok(Response::new(RenamePartResponse { success: false, - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -834,7 +940,7 @@ impl NodeService { Ok(Response::new(CheckPartsResponse { success: false, check_parts_resp: "".to_string(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -874,7 +980,7 @@ impl NodeService { Ok(Response::new(ReadPartsResponse { success: false, object_part_infos: Bytes::new(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -923,7 +1029,7 @@ impl NodeService { Ok(Response::new(VerifyFileResponse { success: false, check_parts_resp: "".to_string(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -953,7 +1059,7 @@ impl NodeService { } else { Ok(Response::new(DeleteResponse { success: false, - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -992,7 +1098,7 @@ impl NodeService { metrics.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC); Ok(Response::new(WriteAllResponse { success: false, - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -1037,7 +1143,7 @@ impl NodeService { Ok(Response::new(ReadAllResponse { success: false, data: Bytes::new(), - error: Some(DiskError::other("can not find disk".to_string()).into()), + error: Some(DiskError::other("cannot find disk".to_string()).into()), })) } } @@ -1045,7 +1151,12 @@ impl NodeService { #[cfg(test)] mod tests { - use super::{decode_msgpack_or_json, encode_msgpack}; + use super::{ + decode_msgpack_or_json, encode_batch_read_version_response_payloads, encode_msgpack, + encode_read_multiple_response_payloads, + }; + use crate::storage::storage_api::ReadMultipleResp; + use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp; use serde::{Deserialize, Serialize}; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -1080,4 +1191,68 @@ mod tests { } ); } + + #[test] + fn encode_read_multiple_response_payloads_keeps_json_and_msgpack_in_sync() { + let responses = vec![ + ReadMultipleResp { + bucket: "bucket".to_string(), + prefix: "prefix".to_string(), + file: "a".to_string(), + exists: true, + data: b"alpha".to_vec(), + ..Default::default() + }, + ReadMultipleResp { + bucket: "bucket".to_string(), + prefix: "prefix".to_string(), + file: "b".to_string(), + exists: true, + data: b"beta".to_vec(), + ..Default::default() + }, + ]; + + let (json_payloads, msgpack_payloads) = + encode_read_multiple_response_payloads(&responses).expect("read multiple responses should encode"); + + assert_eq!(json_payloads.len(), responses.len()); + assert_eq!(msgpack_payloads.len(), responses.len()); + + let json_decoded: ReadMultipleResp = + serde_json::from_str(&json_payloads[0]).expect("json read multiple response should decode"); + let msgpack_decoded = decode_msgpack_or_json::(&msgpack_payloads[0], "", "ReadMultipleResp") + .expect("msgpack read multiple response should decode"); + + assert_eq!(json_decoded.file, responses[0].file); + assert_eq!(msgpack_decoded.file, responses[0].file); + assert_eq!(msgpack_decoded.data, responses[0].data); + } + + #[test] + fn encode_batch_read_version_response_payloads_keeps_json_and_msgpack_in_sync() { + let responses = vec![BatchReadVersionResp { + index: 3, + path: "object-a".to_string(), + version_id: "version-a".to_string(), + success: false, + error: "file version not found".to_string(), + ..Default::default() + }]; + + let (json_payloads, msgpack_payloads) = + encode_batch_read_version_response_payloads(&responses).expect("batch read version responses should encode"); + + assert_eq!(json_payloads.len(), responses.len()); + assert_eq!(msgpack_payloads.len(), responses.len()); + + let json_decoded: BatchReadVersionResp = + serde_json::from_str(&json_payloads[0]).expect("json batch read version response should decode"); + let msgpack_decoded = decode_msgpack_or_json::(&msgpack_payloads[0], "", "BatchReadVersionResp") + .expect("msgpack batch read version response should decode"); + + assert_eq!(json_decoded.index, responses[0].index); + assert_eq!(msgpack_decoded.path, responses[0].path); + assert_eq!(msgpack_decoded.error, responses[0].error); + } } diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index f0a038ff7..86fabaf6e 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -495,6 +495,13 @@ impl Node for NodeService { self.handle_read_version(request).await } + async fn batch_read_version( + &self, + request: Request, + ) -> Result, Status> { + self.handle_batch_read_version(request).await + } + async fn read_xl(&self, request: Request) -> Result, Status> { self.handle_read_xl(request).await } diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 2ee696da3..378de5c7c 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -186,11 +186,12 @@ pub(crate) mod rpc_consumer { pub(crate) mod node_service { pub(crate) use super::super::{ - CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, ECStore, Error, FileInfoVersions, - LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, - SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt, StoragePeerS3ClientExt, - UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_local_server_property, - load_bucket_metadata, reload_transition_tier_config, set_bucket_metadata, + BatchReadVersionReq, BatchReadVersionResp, CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, + ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, + ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt, + StoragePeerS3ClientExt, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, + get_local_server_property, load_bucket_metadata, reload_transition_tier_config, set_bucket_metadata, + validate_batch_read_version_item_count, }; pub(crate) type StorageResult = super::super::Result; @@ -358,9 +359,10 @@ pub(crate) mod ecstore_data_usage { #[allow(unused_imports)] pub(crate) mod ecstore_disk { pub(crate) use rustfs_ecstore::api::disk::{ - CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter, - RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, - WalkDirOptions, get_object_disk_read_timeout, + BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore, + FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, + RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout, + validate_batch_read_version_item_count, }; pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce}; } @@ -510,6 +512,8 @@ pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints; pub(crate) type WorkloadAdmissionSnapshotProviderRef = rustfs_ecstore::WorkloadAdmissionSnapshotProviderRef; pub(crate) type QuotaError = ecstore_bucket::quota::QuotaError; pub(crate) type RawFileInfo = rustfs_filemeta::RawFileInfo; +pub(crate) type BatchReadVersionReq = ecstore_disk::BatchReadVersionReq; +pub(crate) type BatchReadVersionResp = ecstore_disk::BatchReadVersionResp; pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq; pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp; pub(crate) type ReadOptions = ecstore_disk::ReadOptions; @@ -518,6 +522,7 @@ pub(crate) type ReplicationStats = StorageReplicationStatsHandle; pub(crate) type SetupType = ecstore_layout::SetupType; pub(crate) type StorageError = ecstore_error::StorageError; pub(crate) type TierConfigMgr = ecstore_tier::TierConfigMgr; +pub(crate) use ecstore_disk::validate_batch_read_version_item_count; pub(crate) type TransitionState = ecstore_bucket::lifecycle::bucket_lifecycle_ops::TransitionState; pub(crate) type Error = ecstore_error::Error; pub(crate) type Result = ecstore_error::Result; @@ -787,6 +792,7 @@ pub(crate) trait StorageDiskRpcExt { async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult; async fn delete_volume(&self, volume: &str) -> DiskResult<()>; async fn read_multiple(&self, req: ReadMultipleReq) -> DiskResult>; + async fn batch_read_version(&self, req: BatchReadVersionReq) -> DiskResult>; async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) -> Vec>; async fn delete_version( @@ -872,6 +878,10 @@ where ecstore_disk::DiskAPI::read_multiple(self, req).await } + async fn batch_read_version(&self, req: BatchReadVersionReq) -> DiskResult> { + ecstore_disk::DiskAPI::batch_read_version(self, req).await + } + async fn delete_versions( &self, volume: &str, diff --git a/scripts/run_issue797_local_4node_16disk_ab.sh b/scripts/run_issue797_local_4node_16disk_ab.sh new file mode 100755 index 000000000..71b990631 --- /dev/null +++ b/scripts/run_issue797_local_4node_16disk_ab.sh @@ -0,0 +1,582 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Local 4-node / 16-disk RustFS runner for rustfs/backlog#797. +# It starts four local RustFS processes, runs warp workloads, and captures +# health, logs, benchmark summaries, and optional signed admin metrics. + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +RUSTFS_BIN="${RUSTFS_BIN:-${PROJECT_ROOT}/target/debug/rustfs}" +BUILD_BIN="${BUILD_BIN:-true}" +BASE_PORT="${BASE_PORT:-19100}" +NODE_COUNT="${NODE_COUNT:-4}" +DISKS_PER_NODE="${DISKS_PER_NODE:-4}" +ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin}" +SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}" +REGION="${REGION:-us-east-1}" +SIZES="${SIZES:-4KiB,1MiB}" +CONCURRENCY="${CONCURRENCY:-8}" +DURATION="${DURATION:-60s}" +WARP_BIN="${WARP_BIN:-warp}" +WARP_MODE="${WARP_MODE:-mixed}" +WARP_EXTRA_ARGS="${WARP_EXTRA_ARGS:---noclear}" +PROFILES="${PROFILES:-baseline,metrics_logging}" +OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/bench/issue-797-local-4node-16disk-ab-$(date -u +%Y%m%dT%H%M%SZ)}" +DATA_ROOT="${DATA_ROOT:-/tmp/issue797-local-4node-16disk-ab-$(date -u +%Y%m%dT%H%M%SZ)}" +KEEP_DATA="${KEEP_DATA:-false}" +WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}" +HEALTH_POLL_SECS="${HEALTH_POLL_SECS:-2}" +CAPTURE_ADMIN_METRICS="${CAPTURE_ADMIN_METRICS:-true}" +AWSCURL_BIN="${AWSCURL_BIN:-awscurl}" +CURL_BIN="${CURL_BIN:-curl}" +RG_BIN="${RG_BIN:-rg}" +DRY_RUN=false + +PIDS=() + +usage() { + cat <<'USAGE' +Usage: + scripts/run_issue797_local_4node_16disk_ab.sh [options] + +Options: + --rustfs-bin RustFS binary (default: target/debug/rustfs) + --skip-build Do not build rustfs before running + --base-port First node port; uses port..port+3 + --sizes Object sizes for warp (default: 4KiB,1MiB) + --concurrency Warp concurrency (default: 8) + --duration Warp duration per size/profile (default: 60s) + --profiles baseline,metrics_logging,locality_on + --out-dir Output directory + --data-root Temporary disk root + --keep-data Keep data root after exit + --warp-extra-args Extra args appended to warp (default: --noclear) + --skip-admin-metrics Do not attempt signed admin metrics capture + --dry-run Print planned layout without running + -h, --help Show help + +Profiles: + baseline Metrics exports, file logging, shard locality, and batch + processor observation disabled. + metrics_logging Enables metrics export gate, bounded warn-level file + logging, shard locality observe mode, and batch processor + observe mode. + locality_on Same as metrics_logging, but uses shard locality on mode. + +Notes: + Admin metrics are captured from /rustfs/admin/v3/metrics when awscurl or + curl --aws-sigv4 is available. If neither is available, benchmark still runs + and records a skipped metrics status file. +USAGE +} + +log() { + printf '[INFO] %s\n' "$*" +} + +warn() { + printf '[WARN] %s\n' "$*" >&2 +} + +die() { + printf '[ERROR] %s\n' "$*" >&2 + exit 1 +} + +arg_value() { + local flag="$1" + local value="${2:-}" + if [[ -z "$value" || "$value" == --* ]]; then + die "missing value for ${flag}" + fi + printf '%s\n' "$value" +} + +# Like arg_value, but accepts values that themselves start with `--` +# (e.g. `--warp-extra-args --noclear`). +arg_value_allow_dashes() { + local flag="$1" + local value="${2:-}" + if [[ -z "$value" ]]; then + die "missing value for ${flag}" + fi + printf '%s\n' "$value" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --rustfs-bin) RUSTFS_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --skip-build) BUILD_BIN=false; shift ;; + --base-port) BASE_PORT="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --sizes) SIZES="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --concurrency) CONCURRENCY="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --duration) DURATION="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --profiles) PROFILES="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --data-root) DATA_ROOT="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --keep-data) KEEP_DATA=true; shift ;; + --warp-extra-args) WARP_EXTRA_ARGS="$(arg_value_allow_dashes "$1" "${2:-}")"; shift 2 ;; + --skip-admin-metrics) CAPTURE_ADMIN_METRICS=false; shift ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown arg: $1" ;; + esac + done +} + +is_positive_integer() { + [[ "$1" =~ ^[1-9][0-9]*$ ]] +} + +validate_args() { + is_positive_integer "$BASE_PORT" || die "--base-port must be a positive integer" + is_positive_integer "$CONCURRENCY" || die "--concurrency must be a positive integer" + [[ "$NODE_COUNT" == "4" ]] || die "NODE_COUNT is fixed to 4 for this runner" + [[ "$DISKS_PER_NODE" == "4" ]] || die "DISKS_PER_NODE is fixed to 4 for this runner" +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + die "command not found: $1" + fi +} + +print_redacted_command() { + local redact_next=false + local arg + + printf 'Command:' + for arg in "$@"; do + if [[ "$redact_next" == "true" ]]; then + printf ' %q' "REDACTED" + redact_next=false + continue + fi + + case "$arg" in + --access-key|--secret-key) + printf ' %q' "$arg" + redact_next=true + ;; + -accessKey=*|-secretKey=*) + printf ' %q' "${arg%%=*}=REDACTED" + ;; + *) + printf ' %q' "$arg" + ;; + esac + done + printf '\n' +} + +endpoint_for_node() { + local endpoint_node_id="$1" + printf 'http://127.0.0.1:%s' "$((BASE_PORT + endpoint_node_id - 1))" +} + +endpoint_label() { + local endpoint="$1" + local label + label="${endpoint#*://}" + label="${label%%/*}" + printf '%s' "$label" | tr -c 'A-Za-z0-9._-' '_' +} + +all_endpoints_csv() { + local endpoint_loop_id + local endpoints=() + for ((endpoint_loop_id = 1; endpoint_loop_id <= NODE_COUNT; endpoint_loop_id++)); do + endpoints+=("$(endpoint_for_node "$endpoint_loop_id")") + done + local IFS=',' + printf '%s' "${endpoints[*]}" +} + +metrics_url() { + # SCANNER(1) + NET(32) + RPC(256): enough for readiness context plus + # internode aggregate traffic without collecting the heavier all-metrics set. + printf '%s/rustfs/admin/v3/metrics?types=289&by-host=true&n=1\n' "${1%/}" +} + +curl_supports_aws_sigv4() { + "$CURL_BIN" --help all 2>/dev/null | grep -q -- '--aws-sigv4' +} + +profile_is_supported() { + case "$1" in + baseline|metrics_logging|locality_on) return 0 ;; + *) return 1 ;; + esac +} + +profile_env_file() { + local profile="$1" + local env_file="$2" + case "$profile" in + baseline) + cat >"$env_file" <<'EOF' +RUSTFS_OBS_LOGGER_LEVEL=off +RUSTFS_OBS_TRACES_EXPORT_ENABLED=false +RUSTFS_OBS_METRICS_EXPORT_ENABLED=false +RUSTFS_OBS_LOGS_EXPORT_ENABLED=false +RUSTFS_OBS_PROFILING_EXPORT_ENABLED=false +RUSTFS_OBS_USE_STDOUT=false +RUSTFS_OBS_LOG_STDOUT_ENABLED=false +RUSTFS_BATCH_PROCESSOR_ADAPTIVE=off +RUSTFS_SHARD_LOCALITY_SCHEDULING=off +EOF + ;; + metrics_logging) + cat >"$env_file" <<'EOF' +RUSTFS_OBS_LOGGER_LEVEL=warn +RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318 +RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS=500 +RUSTFS_OBS_TRACES_EXPORT_ENABLED=false +RUSTFS_OBS_METRICS_EXPORT_ENABLED=true +RUSTFS_OBS_LOGS_EXPORT_ENABLED=false +RUSTFS_OBS_PROFILING_EXPORT_ENABLED=false +RUSTFS_OBS_USE_STDOUT=false +RUSTFS_OBS_LOG_STDOUT_ENABLED=false +RUSTFS_OBS_METER_INTERVAL=5 +RUSTFS_OBS_LOG_KEEP_FILES=2 +RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES=268435456 +RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES=134217728 +RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS=0 +RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS=30 +RUSTFS_BATCH_PROCESSOR_ADAPTIVE=observe +RUSTFS_SHARD_LOCALITY_SCHEDULING=observe +EOF + ;; + locality_on) + cat >"$env_file" <<'EOF' +RUSTFS_OBS_LOGGER_LEVEL=warn +RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318 +RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS=500 +RUSTFS_OBS_TRACES_EXPORT_ENABLED=false +RUSTFS_OBS_METRICS_EXPORT_ENABLED=true +RUSTFS_OBS_LOGS_EXPORT_ENABLED=false +RUSTFS_OBS_PROFILING_EXPORT_ENABLED=false +RUSTFS_OBS_USE_STDOUT=false +RUSTFS_OBS_LOG_STDOUT_ENABLED=false +RUSTFS_OBS_METER_INTERVAL=5 +RUSTFS_OBS_LOG_KEEP_FILES=2 +RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES=268435456 +RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES=134217728 +RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS=0 +RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS=30 +RUSTFS_BATCH_PROCESSOR_ADAPTIVE=observe +RUSTFS_SHARD_LOCALITY_SCHEDULING=on +EOF + ;; + esac +} + +load_profile_env() { + local env_file="$1" + set -a + # shellcheck disable=SC1090 + source "$env_file" + set +a +} + +prepare_profile_layout() { + local profile="$1" + local profile_dir="$OUT_DIR/$profile" + mkdir -p "$profile_dir"/{bench,health,logs,metrics,pids} + profile_env_file "$profile" "$profile_dir/profile.env" +} + +build_volumes() { + local profile="$1" + local node_index disk_index + local volumes=() + for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do + for ((disk_index = 1; disk_index <= DISKS_PER_NODE; disk_index++)); do + volumes+=("http://127.0.0.1:$((BASE_PORT + node_index - 1))${DATA_ROOT}/${profile}/node${node_index}/disk${disk_index}") + done + done + printf '%s ' "${volumes[@]}" +} + +prepare_data_dirs() { + local profile="$1" + local node_index disk_index + for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do + for ((disk_index = 1; disk_index <= DISKS_PER_NODE; disk_index++)); do + mkdir -p "${DATA_ROOT}/${profile}/node${node_index}/disk${disk_index}" + done + done +} + +stop_nodes() { + local pid + if [[ ${#PIDS[@]} -eq 0 ]]; then + return + fi + for pid in "${PIDS[@]}"; do + if kill -0 "$pid" >/dev/null 2>&1; then + kill "$pid" >/dev/null 2>&1 || true + fi + done + for ((_ = 1; _ <= 5; _++)); do + local remaining=0 + for pid in "${PIDS[@]}"; do + if kill -0 "$pid" >/dev/null 2>&1; then + remaining=$((remaining + 1)) + fi + done + [[ "$remaining" -eq 0 ]] && break + sleep 1 + done + for pid in "${PIDS[@]}"; do + if kill -0 "$pid" >/dev/null 2>&1; then + kill -9 "$pid" >/dev/null 2>&1 || true + fi + wait "$pid" >/dev/null 2>&1 || true + done + PIDS=() +} + +cleanup() { + stop_nodes + if [[ "$KEEP_DATA" != "true" && "$DRY_RUN" != "true" && -n "$DATA_ROOT" && -d "$DATA_ROOT" ]]; then + rm -rf "$DATA_ROOT" + fi +} + +trap cleanup EXIT + +start_nodes() { + local profile="$1" + local profile_dir="$OUT_DIR/$profile" + local volumes + volumes="$(build_volumes "$profile")" + + prepare_data_dirs "$profile" + load_profile_env "$profile_dir/profile.env" + + local node_index endpoint log_file pid_file + for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do + endpoint="$(endpoint_for_node "$node_index")" + log_file="$profile_dir/logs/node${node_index}.log" + pid_file="$profile_dir/pids/node${node_index}.pid" + + ( + export RUSTFS_ACCESS_KEY="$ACCESS_KEY" + export RUSTFS_SECRET_KEY="$SECRET_KEY" + export RUSTFS_ADDRESS="127.0.0.1:$((BASE_PORT + node_index - 1))" + export RUSTFS_CONSOLE_ENABLE=false + export RUSTFS_SCANNER_ENABLED=false + export RUSTFS_SCANNER_START_DELAY_SECS=3600 + export RUSTFS_SCANNER_CYCLE=3600 + export RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true + export RUSTFS_VOLUMES="$volumes" + export RUSTFS_OBS_SERVICE_NAME="rustfs-issue797-${profile}-node${node_index}" + export RUSTFS_OBS_LOG_DIRECTORY="$profile_dir/logs/node${node_index}" + mkdir -p "$RUSTFS_OBS_LOG_DIRECTORY" + # Keep both env and positional volumes: current CLI still requires the + # positional server volumes, while env makes the layout visible in logs. + # shellcheck disable=SC2086 + exec "$RUSTFS_BIN" server $volumes + ) >"$log_file" 2>&1 & + PIDS+=("$!") + printf '%s\n' "$!" >"$pid_file" + log "started ${profile} node${node_index} ${endpoint} pid=$!" + done +} + +wait_for_health() { + local profile="$1" + local profile_dir="$OUT_DIR/$profile" + local deadline node_index endpoint status_file + deadline=$((SECONDS + WAIT_TIMEOUT_SECS)) + while (( SECONDS < deadline )); do + local ready=0 + for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do + endpoint="$(endpoint_for_node "$node_index")" + status_file="$profile_dir/health/node${node_index}.live" + if "$CURL_BIN" -fsS --max-time 2 "${endpoint}/health/live" >"$status_file" 2>"$status_file.err"; then + ready=$((ready + 1)) + fi + done + if [[ "$ready" -eq "$NODE_COUNT" ]]; then + log "${profile}: all nodes are live" + return 0 + fi + sleep "$HEALTH_POLL_SECS" + done + + for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do + warn "${profile} node${node_index} log tail:" + tail -n 40 "$profile_dir/logs/node${node_index}.log" >&2 || true + done + die "${profile}: timed out waiting for health" +} + +capture_admin_metrics() { + local profile="$1" + local phase="$2" + local profile_dir="$OUT_DIR/$profile" + local status_file="$profile_dir/metrics/${phase}.status" + local endpoint label metrics_file endpoints + + if [[ "$CAPTURE_ADMIN_METRICS" != "true" ]]; then + echo "skipped: disabled" >"$status_file" + return 0 + fi + local capture_method="" + if command -v "$AWSCURL_BIN" >/dev/null 2>&1; then + capture_method="awscurl" + elif curl_supports_aws_sigv4; then + capture_method="curl-sigv4" + else + echo "skipped: awscurl not found and curl lacks --aws-sigv4" >"$status_file" + return 0 + fi + + echo "capturing: ${capture_method}" >"$status_file" + IFS=',' read -r -a endpoints <<<"$(all_endpoints_csv)" + for endpoint in "${endpoints[@]}"; do + label="$(endpoint_label "$endpoint")" + metrics_file="$profile_dir/metrics/${phase}.${label}.ndjson" + if [[ "$capture_method" == "awscurl" ]]; then + AWS_ACCESS_KEY_ID="$ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$SECRET_KEY" \ + AWS_DEFAULT_REGION="$REGION" \ + "$AWSCURL_BIN" \ + --service s3 \ + --region "$REGION" \ + --request GET \ + "$(metrics_url "$endpoint")" \ + >"$metrics_file" 2>"$metrics_file.err" || true + else + local curl_config="$profile_dir/metrics/.curl-sigv4-${phase}.${label}.conf" + { + printf 'aws-sigv4 = "aws:amz:%s:s3"\n' "$REGION" + printf 'user = "%s:%s"\n' "$ACCESS_KEY" "$SECRET_KEY" + printf 'request = "GET"\n' + printf 'max-time = 10\n' + printf 'fail\n' + printf 'silent\n' + printf 'show-error\n' + } >"$curl_config" + chmod 600 "$curl_config" + "$CURL_BIN" --config "$curl_config" "$(metrics_url "$endpoint")" >"$metrics_file" 2>"$metrics_file.err" || true + rm -f "$curl_config" + fi + done + echo "done: ${capture_method}" >"$status_file" +} + +run_bench() { + local profile="$1" + local profile_dir="$OUT_DIR/$profile" + local bucket_profile="${profile//_/-}" + local -a cmd=( + "${PROJECT_ROOT}/scripts/run_object_batch_bench.sh" + --tool warp + --endpoint "$(all_endpoints_csv)" + --access-key "$ACCESS_KEY" + --secret-key "$SECRET_KEY" + --region "$REGION" + --auto-new-bucket + --bucket-prefix "issue797-${bucket_profile}" + --sizes "$SIZES" + --concurrency "$CONCURRENCY" + --duration "$DURATION" + --warp-bin "$WARP_BIN" + --warp-mode "$WARP_MODE" + --out-dir "$profile_dir/bench" + ) + + if [[ -n "$WARP_EXTRA_ARGS" ]]; then + cmd+=(--extra-args "$WARP_EXTRA_ARGS") + fi + + print_redacted_command "${cmd[@]}" >"$profile_dir/bench/command.txt" + log "${profile}: running warp benchmark" + "${cmd[@]}" 2>&1 | tee "$profile_dir/bench/run.log" +} + +extract_tail_summary() { + local profile="$1" + local profile_dir="$OUT_DIR/$profile" + local output="$profile_dir/bench/tail_latency_summary.txt" + : >"$output" + if ! command -v "$RG_BIN" >/dev/null 2>&1; then + echo "rg not found; skipped" >"$output" + return 0 + fi + "$RG_BIN" -n 'Average:|Median:|90th:|99th:|Fastest:|Slowest:|StdDev:|Total:|Throughput by host|warp: ' \ + "$profile_dir/bench"/*.log >"$output" || true +} + +write_run_meta() { + local meta="$OUT_DIR/run-meta.txt" + mkdir -p "$OUT_DIR" + { + echo "created_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "rustfs_bin=$RUSTFS_BIN" + echo "base_port=$BASE_PORT" + echo "node_count=$NODE_COUNT" + echo "disks_per_node=$DISKS_PER_NODE" + echo "sizes=$SIZES" + echo "concurrency=$CONCURRENCY" + echo "duration=$DURATION" + echo "profiles=$PROFILES" + echo "warp_extra_args=$WARP_EXTRA_ARGS" + echo "endpoints=$(all_endpoints_csv)" + echo "data_root=$DATA_ROOT" + echo "keep_data=$KEEP_DATA" + echo "capture_admin_metrics=$CAPTURE_ADMIN_METRICS" + } >"$meta" +} + +run_profile() { + local profile="$1" + profile_is_supported "$profile" || die "unsupported profile: $profile" + prepare_profile_layout "$profile" + start_nodes "$profile" + wait_for_health "$profile" + capture_admin_metrics "$profile" before + run_bench "$profile" + capture_admin_metrics "$profile" after + extract_tail_summary "$profile" + stop_nodes +} + +main() { + parse_args "$@" + validate_args + + write_run_meta + + if [[ "$DRY_RUN" == "true" ]]; then + log "dry run only" + cat "$OUT_DIR/run-meta.txt" + return 0 + fi + + require_cmd "$CURL_BIN" + require_cmd "$WARP_BIN" + + if [[ "$BUILD_BIN" == "true" ]]; then + log "building rustfs binary" + cargo build -p rustfs --bin rustfs + fi + [[ -x "$RUSTFS_BIN" ]] || die "rustfs binary is not executable: $RUSTFS_BIN" + + local profile + IFS=',' read -r -a profile_arr <<<"$PROFILES" + for profile in "${profile_arr[@]}"; do + profile="${profile//[[:space:]]/}" + [[ -z "$profile" ]] && continue + run_profile "$profile" + done + + log "done. Output dir: $OUT_DIR" +} + +main "$@"