gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(storage): harden internode data-path controls (#4224)

Browse Source 
* fix(rio): propagate http writer shutdown errors

* fix(ecstore): unify remote lock rpc deadlines

* fix(storage): reject corrupt read multiple payloads

* feat(rio): add internode http tuning profiles

* feat(metrics): add internode baseline signals

* feat(ecstore): observe shard locality topology

* feat(ecstore): gate shard locality scheduling

* feat(ecstore): gate batch read version rpc

* feat(ecstore): observe batch processor adaptation

* feat(ecstore): gate batch processor observation

* docs: add get benchmark regression analysis

* docs: add issue 797 execution plan status

* fix(ecstore): require explicit batch rpc support

* fix(ecstore): honor documented batch read gate

* fix(ecstore): keep batch read gate stable per call

* chore: update workspace dependencies

* feat(ecstore): log batch read gate decisions

* feat(ecstore): count batch read gate decisions

* test(issue-797): add local internode A/B runner

* test(rio): fix tuning profile spelling fixture

* fix(protocols): adapt sftp channel open callbacks

* fix(metrics): wrap batch processor observation args

* chore(docs): keep issue notes local only

* fix(storage): address internode review feedback

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

Co-Authored-By: heihutu<heihutu@gmail.com>

* fix(storage): align buffer clamp test with media cap

* fix(ecstore): release optimized read locks before streaming

---------

Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
houseme
2026-07-03 17:08:15 +08:00
committed by GitHub
parent a9ac5f578d
commit 25d80d7c60
26 changed files with 3627 additions and 483 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+204 -118
View File
@@ -332,9 +332,9 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
[[package]]
name = "arrayvec"
version = "0.7.7"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe"
checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
[[package]]
name = "arrow"
@@ -706,7 +706,7 @@ dependencies = [
"nuid",
"pin-project",
"portable-atomic",
"rand 0.10.1",
"rand 0.10.2",
"regex",
"ring",
"rustls-native-certs",
@@ -2447,9 +2447,9 @@ dependencies = [
[[package]]
name = "curve25519-dalek"
version = "5.0.0-rc.0"
version = "5.0.0-rc.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f359e08ca85e7bd759e1fd933ff2bccd81864c60a8fba0e259c7f822b0924bf"
checksum = "c906a87e53a36ff795d72e06e8162a83c5436e3ea89e942a9cb9fc083f0a384f"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
@@ -3604,7 +3604,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3667,7 +3667,7 @@ dependencies = [
"http 1.4.2",
"local-ip-address",
"md5",
"rand 0.10.1",
"rand 0.10.2",
"rcgen",
"reqwest",
"rmp-serde",
@@ -3718,14 +3718,14 @@ dependencies = [
[[package]]
name = "ecdsa"
version = "0.17.0-rc.18"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54fb064faabbee66e1fc8e5c5a9458d4269dc2d8b638fe86a425adb2510d1a96"
checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0"
dependencies = [
"der 0.8.0",
"digest 0.11.3",
"elliptic-curve 0.14.0-rc.33",
"rfc6979 0.5.0",
"elliptic-curve 0.14.1",
"rfc6979 0.6.0",
"signature 3.0.0",
"spki 0.8.0",
"zeroize",
@@ -3768,11 +3768,11 @@ dependencies = [
[[package]]
name = "ed25519-dalek"
version = "3.0.0-rc.0"
version = "3.0.0-rc.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b011170fe4f04665565b4110afef66774fe9ffff278f3eb5b81cc73d26e27d60"
checksum = "1685663e23882cd8517dcbcb1c23a6ebff4433c22dfb681d760219b62cd1b849"
dependencies = [
"curve25519-dalek 5.0.0-rc.0",
"curve25519-dalek 5.0.0-rc.1",
"ed25519 3.0.0",
"rand_core 0.10.1",
"serde",
@@ -3811,9 +3811,9 @@ dependencies = [
[[package]]
name = "elliptic-curve"
version = "0.14.0-rc.33"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "102d3643d30dd8b559613c5cced68317199597fffb278cdc88daa2ef7fafc935"
checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65"
dependencies = [
"base16ct 1.0.0",
"crypto-bigint 0.7.5",
@@ -3823,7 +3823,6 @@ dependencies = [
"group 0.14.0",
"hkdf 0.13.0",
"hybrid-array",
"once_cell",
"pem-rfc7468 1.0.0",
"pkcs8 0.11.0",
"rand_core 0.10.1",
@@ -3928,7 +3927,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4390,7 +4389,7 @@ dependencies = [
"google-cloud-wkt",
"http 1.4.2",
"pin-project",
"rand 0.10.1",
"rand 0.10.2",
"serde",
"serde_json",
"thiserror 2.0.18",
@@ -4770,7 +4769,7 @@ dependencies = [
"idna",
"ipnet",
"jni",
"rand 0.10.1",
"rand 0.10.2",
"thiserror 2.0.18",
"tinyvec",
"tokio",
@@ -4790,7 +4789,7 @@ dependencies = [
"jni",
"once_cell",
"prefix-trie",
"rand 0.10.1",
"rand 0.10.2",
"ring",
"thiserror 2.0.18",
"tinyvec",
@@ -4815,7 +4814,7 @@ dependencies = [
"ndk-context",
"once_cell",
"parking_lot",
"rand 0.10.1",
"rand 0.10.2",
"resolv-conf",
"smallvec",
"system-configuration",
@@ -4952,9 +4951,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "humantime"
version = "2.3.0"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
[[package]]
name = "hybrid-array"
@@ -5243,7 +5242,7 @@ checksum = "ae8e22120c32fb4d19ec55fba35015f57095cd95a2e3b732e44457f5915b2ee8"
dependencies = [
"num-integer",
"num-traits",
"rand 0.10.1",
"rand 0.10.2",
"rand_core 0.10.1",
]
@@ -5363,9 +5362,9 @@ dependencies = [
[[package]]
name = "jiff-tzdb"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076"
checksum = "6142247df1a93c2b3587402a19710be3e6e942f1581a1702e76408f2c21d6590"
[[package]]
name = "jiff-tzdb-platform"
@@ -6280,7 +6279,7 @@ dependencies = [
"lru 0.18.0",
"mysql_common",
"percent-encoding",
"rand 0.10.1",
"rand 0.10.2",
"rustls",
"serde",
"socket2",
@@ -6481,7 +6480,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -6495,9 +6494,9 @@ dependencies = [
[[package]]
name = "num-bigint"
version = "0.4.6"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6"
dependencies = [
"num-integer",
"num-traits",
@@ -6619,7 +6618,7 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [
"base64 0.21.7",
"base64 0.22.1",
"chrono",
"getrandom 0.2.17",
"http 1.4.2",
@@ -6960,7 +6959,7 @@ dependencies = [
"hmac 0.12.1",
"pkcs12",
"pkcs5 0.7.1",
"rand 0.10.1",
"rand 0.10.2",
"rc2",
"sha1 0.10.6",
"sha2 0.10.9",
@@ -6982,14 +6981,14 @@ dependencies = [
[[package]]
name = "p256"
version = "0.14.0-rc.10"
version = "0.14.0-rc.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41adc63effe99d48837a8cc0e6d7a77e32ae6a07f6000df466178dbc2193093e"
checksum = "b6bb40a5099e2c38a09dd29321a7a7f045f165a54317679c7cdfb0cbaf8f6b1e"
dependencies = [
"ecdsa 0.17.0-rc.18",
"elliptic-curve 0.14.0-rc.33",
"ecdsa 0.17.0",
"elliptic-curve 0.14.1",
"primefield",
"primeorder 0.14.0-rc.10",
"primeorder 0.14.0",
"sha2 0.11.0",
]
@@ -7007,29 +7006,29 @@ dependencies = [
[[package]]
name = "p384"
version = "0.14.0-rc.10"
version = "0.14.0-rc.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bd5333afa5ae0347f39e6a0f2c9c155da431583fd71fe5555bd0521b4ccaf02"
checksum = "492f329d7eb11d22dadc988626b9ea1f503b4ab043a8b1e4e2cc4ae45dabdd70"
dependencies = [
"ecdsa 0.17.0-rc.18",
"elliptic-curve 0.14.0-rc.33",
"ecdsa 0.17.0",
"elliptic-curve 0.14.1",
"fiat-crypto 0.3.0",
"primefield",
"primeorder 0.14.0-rc.10",
"primeorder 0.14.0",
"sha2 0.11.0",
]
[[package]]
name = "p521"
version = "0.14.0-rc.10"
version = "0.14.0-rc.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3a5297f53dc16d35909060ba3032cff7867e8809f01e273ff325579d5f0ceae"
checksum = "ff42e4ace5424e3b6d7cb82514be89866b85af87015f80341e37dcc21a66ce6e"
dependencies = [
"base16ct 1.0.0",
"ecdsa 0.17.0-rc.18",
"elliptic-curve 0.14.0-rc.33",
"ecdsa 0.17.0",
"elliptic-curve 0.14.1",
"primefield",
"primeorder 0.14.0-rc.10",
"primeorder 0.14.0",
"sha2 0.11.0",
]
@@ -7055,7 +7054,7 @@ dependencies = [
"delegate",
"futures",
"log",
"rand 0.10.1",
"rand 0.10.2",
"sha2 0.11.0",
"thiserror 2.0.18",
"tokio",
@@ -7527,7 +7526,7 @@ dependencies = [
"hmac 0.13.0",
"md-5 0.11.0",
"memchr",
"rand 0.10.1",
"rand 0.10.2",
"sha2 0.11.0",
"stringprep",
]
@@ -7631,11 +7630,15 @@ dependencies = [
[[package]]
name = "primeorder"
version = "0.14.0-rc.10"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d2793f22b9b6fd11ef3ac1d59bf003c2573593e4968702341605c2748fd90bf"
checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06"
dependencies = [
"elliptic-curve 0.14.0-rc.33",
"elliptic-curve 0.14.1",
"once_cell",
"primefield",
"serdect",
"wnaf",
]
[[package]]
@@ -7749,7 +7752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
dependencies = [
"heck",
"itertools 0.10.5",
"itertools 0.14.0",
"log",
"multimap",
"once_cell",
@@ -7769,7 +7772,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
dependencies = [
"heck",
"itertools 0.10.5",
"itertools 0.14.0",
"log",
"multimap",
"petgraph 0.8.3",
@@ -7790,7 +7793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
dependencies = [
"anyhow",
"itertools 0.10.5",
"itertools 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.118",
@@ -7803,7 +7806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [
"anyhow",
"itertools 0.10.5",
"itertools 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.118",
@@ -8020,7 +8023,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -8077,9 +8080,9 @@ dependencies = [
[[package]]
name = "rand"
version = "0.10.1"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
dependencies = [
"chacha20",
"getrandom 0.4.3",
@@ -8441,12 +8444,12 @@ dependencies = [
[[package]]
name = "rfc6979"
version = "0.5.0"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5236ce872cac07e0fb3969b0cbf468c7d2f37d432f1b627dcb7b8d34563fb0c3"
checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b"
dependencies = [
"crypto-bigint 0.7.5",
"hmac 0.13.0",
"subtle",
]
[[package]]
@@ -8581,9 +8584,9 @@ dependencies = [
[[package]]
name = "russh"
version = "0.61.2"
version = "0.62.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbf893f64684e58da8a68d56a5e84d1cf0440226274c515770fe267707a7d0b0"
checksum = "20d2039c3e50abb1ec3bdadea27098679da12a33befa9ab4e62adad9a13ebcb2"
dependencies = [
"aes 0.9.1",
"aws-lc-rs",
@@ -8595,14 +8598,14 @@ dependencies = [
"cipher 0.5.2",
"crypto-bigint 0.7.5",
"ctr",
"curve25519-dalek 5.0.0-rc.0",
"curve25519-dalek 5.0.0-rc.1",
"data-encoding",
"delegate",
"der 0.8.0",
"digest 0.11.3",
"ecdsa 0.17.0-rc.18",
"ed25519-dalek 3.0.0-rc.0",
"elliptic-curve 0.14.0-rc.33",
"ecdsa 0.17.0",
"ed25519-dalek 3.0.0-rc.1",
"elliptic-curve 0.14.1",
"enum_dispatch",
"flate2",
"futures",
@@ -8619,8 +8622,8 @@ dependencies = [
"ml-kem",
"module-lattice",
"num-bigint",
"p256 0.14.0-rc.10",
"p384 0.14.0-rc.10",
"p256 0.14.0-rc.15",
"p384 0.14.0-rc.15",
"p521",
"pageant",
"pbkdf2 0.13.0",
@@ -8628,7 +8631,7 @@ dependencies = [
"pkcs5 0.8.1",
"pkcs8 0.11.0",
"polyval",
"rand 0.10.1",
"rand 0.10.2",
"rand_core 0.10.1",
"rsa 0.10.0-rc.18",
"russh-cryptovec",
@@ -8653,9 +8656,9 @@ dependencies = [
[[package]]
name = "russh-cryptovec"
version = "0.61.0"
version = "0.62.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "443f6bbcfacb34a1aab2b12b99bf08e0c63abdc5a0db261901365df9d57fff51"
checksum = "3aec6cb630dbe85d72ffd7bcd95f07e1bd69f9f270ee8adfa1afe443a6331438"
dependencies = [
"log",
"nix 0.31.3",
@@ -8738,9 +8741,9 @@ checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
[[package]]
name = "rustc-hash"
version = "2.1.2"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
[[package]]
name = "rustc_version"
@@ -8802,7 +8805,7 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"proptest",
"rand 0.10.1",
"rand 0.10.2",
"rcgen",
"reqwest",
"rmp-serde",
@@ -8961,7 +8964,7 @@ version = "1.0.0-beta.8"
dependencies = [
"base64-simd",
"hmac 0.13.0",
"rand 0.10.1",
"rand 0.10.2",
"serde",
"serde_json",
"sha2 0.11.0",
@@ -8978,7 +8981,7 @@ dependencies = [
"chacha20poly1305",
"jsonwebtoken",
"pbkdf2 0.13.0",
"rand 0.10.1",
"rand 0.10.2",
"rsa 0.10.0-rc.18",
"serde",
"serde_json",
@@ -9052,7 +9055,7 @@ dependencies = [
"pin-project-lite",
"proptest",
"quick-xml",
"rand 0.10.1",
"rand 0.10.2",
"ratelimit",
"rcgen",
"reed-solomon-simd",
@@ -9266,7 +9269,7 @@ dependencies = [
"kafka-protocol",
"metrics",
"pbkdf2 0.13.0",
"rand 0.10.1",
"rand 0.10.2",
"rustls",
"rustls-native-certs",
"sha2 0.11.0",
@@ -9289,7 +9292,7 @@ dependencies = [
"kafka-protocol",
"metrics",
"pbkdf2 0.13.0",
"rand 0.10.1",
"rand 0.10.2",
"rustfs-kafka",
"rustls",
"rustls-native-certs",
@@ -9340,7 +9343,7 @@ dependencies = [
"jiff",
"md5",
"moka",
"rand 0.10.1",
"rand 0.10.2",
"reqwest",
"rustfs-security-governance",
"rustfs-utils",
@@ -9630,7 +9633,7 @@ dependencies = [
"http-body-util",
"md-5 0.11.0",
"pin-project-lite",
"rand 0.10.1",
"rand 0.10.2",
"reqwest",
"rustfs-config",
"rustfs-io-metrics",
@@ -9660,7 +9663,7 @@ dependencies = [
"hmac 0.13.0",
"minlz",
"pin-project-lite",
"rand 0.10.1",
"rand 0.10.2",
"rustfs-filemeta",
"rustfs-rio",
"rustfs-utils",
@@ -9739,7 +9742,7 @@ dependencies = [
"futures",
"http 1.4.2",
"metrics",
"rand 0.10.1",
"rand 0.10.2",
"rmp-serde",
"rustfs-common",
"rustfs-config",
@@ -10010,7 +10013,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.12.1",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -10084,7 +10087,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -10141,7 +10144,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "s3s"
version = "0.14.0"
source = "git+https://github.com/rustfs/s3s?rev=921e9c5911951ae9efb91dc237e11be9f2f10a14#921e9c5911951ae9efb91dc237e11be9f2f10a14"
source = "git+https://github.com/rustfs/s3s?rev=b365107038a633ce0f7a199f507507a8c5e64665#b365107038a633ce0f7a199f507507a8c5e64665"
dependencies = [
"arc-swap",
"arrayvec",
@@ -10912,17 +10915,15 @@ dependencies = [
[[package]]
name = "ssh-cipher"
version = "0.3.0-rc.9"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10db6f219196a8528f9ec904d9d45cdad692d65b0e57e72be4dedd1c5fddce36"
checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597"
dependencies = [
"aead",
"aes 0.9.1",
"aes-gcm",
"cbc 0.2.1",
"chacha20",
"cipher 0.5.2",
"ctr",
"ctutils",
"des 0.9.0",
"poly1305",
@@ -10932,9 +10933,9 @@ dependencies = [
[[package]]
name = "ssh-encoding"
version = "0.3.0-rc.9"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7abf34aa716da5d5b4c496936d042ea282ab392092cd68a72ef6a8863ff8c96a"
checksum = "7b54d0ed0498daf3f78d82e00e28c8eec9d75a067c4cfbcc7a0f7d0f4077749e"
dependencies = [
"base64ct",
"bytes",
@@ -10947,18 +10948,18 @@ dependencies = [
[[package]]
name = "ssh-key"
version = "0.7.0-rc.10"
version = "0.7.0-rc.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45735ce3dea95690e4a9e414c4cfde7f79835063c3dcd35881df85a84118e74b"
checksum = "f9a32fae177b74a22aa9c5b01bf7e68b33545be32d9e381e248058d2adc15ce3"
dependencies = [
"argon2",
"bcrypt-pbkdf",
"ctutils",
"ed25519-dalek 3.0.0-rc.0",
"ed25519-dalek 3.0.0-rc.1",
"hex",
"hmac 0.13.0",
"p256 0.14.0-rc.10",
"p384 0.14.0-rc.10",
"p256 0.14.0-rc.15",
"p384 0.14.0-rc.15",
"p521",
"rand_core 0.10.1",
"rsa 0.10.0-rc.18",
@@ -11114,9 +11115,9 @@ dependencies = [
[[package]]
name = "symbolic-common"
version = "13.7.0"
version = "13.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82bfbe03a011e4d3e2a94f03e054c33b101767f6905ec2a608dd85aa0cd71c97"
checksum = "dddd4e6e0693d88f584394027d97d4c4b7748c2fa86be4441703db5fa8b78a95"
dependencies = [
"debugid",
"memmap2",
@@ -11126,9 +11127,9 @@ dependencies = [
[[package]]
name = "symbolic-demangle"
version = "13.7.0"
version = "13.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9d2ed98c0fd513dd4788554e1a62d9736f3642f8aaf7923644ec647158c1ded"
checksum = "6cad904b0202151337e3dca9187593aadf6b70ca1e495a2ff420b5bf287f1b3b"
dependencies = [
"rustc-demangle",
"symbolic-common",
@@ -11270,7 +11271,7 @@ dependencies = [
"getrandom 0.4.3",
"once_cell",
"rustix 1.1.4",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -11511,7 +11512,7 @@ dependencies = [
"pin-project-lite",
"postgres-protocol",
"postgres-types",
"rand 0.10.1",
"rand 0.10.2",
"socket2",
"tokio",
"tokio-util",
@@ -12085,7 +12086,7 @@ checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53"
dependencies = [
"getrandom 0.4.3",
"js-sys",
"rand 0.10.1",
"rand 0.10.2",
"serde_core",
"wasm-bindgen",
]
@@ -12353,7 +12354,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -12480,7 +12481,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
@@ -12489,7 +12490,16 @@ version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.5",
]
[[package]]
@@ -12507,14 +12517,31 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
]
[[package]]
@@ -12532,48 +12559,96 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "1.0.3"
@@ -12589,6 +12664,17 @@ version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
[[package]]
name = "wnaf"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1"
dependencies = [
"ff 0.14.0",
"group 0.14.0",
"hybrid-array",
]
[[package]]
name = "wrapcenum-derive"
version = "0.4.1"
Cargo.toml
+5 -5
View File
@@ -202,7 +202,7 @@ zeroize = { version = "1.9.0", features = ["derive"] }
# Time and Date
chrono = { version = "0.4.45", features = ["serde"] }
humantime = "2.3.0"
humantime = "2.4.0"
jiff = { version = "0.2.31", features = ["serde"] }
time = { version = "0.3.53", features = ["std", "parsing", "formatting", "macros", "serde"] }
@@ -267,7 +267,7 @@ path-clean = "1.0.1"
percent-encoding = "2.3.2"
pin-project-lite = "0.2.17"
pretty_assertions = "1.4.1"
rand = { version = "0.10.1", features = ["serde"] }
rand = { version = "0.10.2", features = ["serde"] }
ratelimit = "0.10.1"
rayon = "1.12.0"
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "7.0.1", features = ["simd-accel"] }
@@ -278,8 +278,8 @@ rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket
redis = { version = "1.3.0", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
rustix = { version = "1.1.4", features = ["fs"] }
rust-embed = { version = "8.11.0" }
rustc-hash = { version = "2.1.2" }
s3s = { git = "https://github.com/rustfs/s3s", rev = "921e9c5911951ae9efb91dc237e11be9f2f10a14", features = ["minio"] }
rustc-hash = { version = "2.1.3" }
s3s = { git = "https://github.com/rustfs/s3s", rev = "b365107038a633ce0f7a199f507507a8c5e64665", features = ["minio"] }
serial_test = "3.5.0"
shadow-rs = { version = "2.0.0", default-features = false }
siphasher = "1.0.3"
@@ -327,7 +327,7 @@ libunftp = { version = "0.23.0", features = ["experimental"] }
unftp-core = "0.1.0"
suppaftp = { version = "10.0.0", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
rcgen = "0.14.8"
russh = { version = "0.61.2", features = ["serde"] }
russh = { version = "0.62.1", features = ["serde"] }
russh-sftp = "2.3.0"
# WebDAV
crates/config/src/constants/internode.rs
+36
View File
@@ -32,6 +32,28 @@ pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3;
pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10;
/// Profile selector for conservative internode HTTP data-plane client tuning.
pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy";
/// Internode HTTP connection pool maximum idle connections per host.
pub const ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST: &str = "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST";
/// Internode HTTP connection pool idle timeout in seconds.
pub const ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS";
/// Internode HTTP/2 initial stream window size in bytes.
pub const ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE";
/// Internode HTTP/2 initial connection window size in bytes.
pub const ENV_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE";
/// Whether internode HTTP/2 adaptive window sizing is enabled.
pub const ENV_INTERNODE_HTTP2_ADAPTIVE_WINDOW: &str = "RUSTFS_INTERNODE_HTTP2_ADAPTIVE_WINDOW";
/// Internode HTTP proxy mode: legacy, off, or system.
pub const ENV_INTERNODE_HTTP_PROXY: &str = "RUSTFS_INTERNODE_HTTP_PROXY";
/// Environment variable for selecting the internode data-plane transport backend.
pub const ENV_RUSTFS_INTERNODE_DATA_TRANSPORT: &str = "RUSTFS_INTERNODE_DATA_TRANSPORT";
pub const DEFAULT_INTERNODE_DATA_TRANSPORT: &str = "tcp-http";
@@ -53,6 +75,7 @@ mod tests {
assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 3);
assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 10);
assert_eq!(DEFAULT_INTERNODE_HTTP_TUNING_PROFILE, "legacy");
}
#[test]
@@ -68,6 +91,19 @@ mod tests {
"RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"
);
assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
assert_eq!(ENV_INTERNODE_HTTP_TUNING_PROFILE, "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE");
assert_eq!(ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST, "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST");
assert_eq!(ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS, "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS");
assert_eq!(
ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE,
"RUSTFS_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE"
);
assert_eq!(
ENV_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE,
"RUSTFS_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE"
);
assert_eq!(ENV_INTERNODE_HTTP2_ADAPTIVE_WINDOW, "RUSTFS_INTERNODE_HTTP2_ADAPTIVE_WINDOW");
assert_eq!(ENV_INTERNODE_HTTP_PROXY, "RUSTFS_INTERNODE_HTTP_PROXY");
assert_eq!(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, "RUSTFS_INTERNODE_DATA_TRANSPORT");
assert_eq!(DEFAULT_INTERNODE_DATA_TRANSPORT, "tcp-http");
assert_eq!(INTERNODE_DATA_TRANSPORT_TCP, "tcp");
crates/config/src/constants/object.rs
+3 -2
View File
@@ -191,8 +191,9 @@ pub const DEFAULT_OBJECT_IO_BUFFER_SIZE: usize = 128 * 1024;
/// Environment variable to enable/disable lock optimization.
///
/// When enabled, fully materialized reads may release read locks before the
/// reader is returned. Streaming reads keep the lock until EOF or drop.
/// When enabled, read locks may be released before the reader is returned.
/// Disable this only when streaming readers must keep the object namespace
/// locked until EOF or drop.
///
/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE`).
pub const ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE: &str = "RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE";
crates/e2e_test/src/reliant/grpc_lock_server.rs
+10 -2
View File
@@ -21,8 +21,9 @@ use rustfs_lock::{LockClient, LockRequest};
use rustfs_protos::{
models::PingBodyBuilder,
proto_gen::node_service::{
BatchGenerallyLockRequest, BatchGenerallyLockResponse, GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult,
PingRequest, PingResponse, node_service_server::NodeService,
BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
node_service_server::NodeService,
},
};
use std::pin::Pin;
@@ -96,6 +97,13 @@ impl NodeService for MinimalLockNodeService {
}))
}
async fn batch_read_version(
&self,
_request: Request<BatchReadVersionRequest>,
) -> Result<Response<BatchReadVersionResponse>, Status> {
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
}
async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
let args: LockRequest = match serde_json::from_str(&request.args) {
crates/ecstore/src/api/mod.rs
+4 -2
View File
@@ -84,9 +84,11 @@ pub mod disk {
pub use crate::disk::error_reduce::is_all_buckets_not_found;
pub use crate::disk::local::ScanGuard;
pub use crate::disk::{
BUCKET_META_PREFIX, CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
DiskStore, FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
validate_batch_read_version_item_count,
};
pub use crate::disk::{endpoint, error, error_reduce};
pub use bytes::Bytes;
crates/ecstore/src/cluster/rpc/remote_disk.rs
+441 -21
View File
@@ -20,8 +20,9 @@ use crate::cluster::rpc::internode_data_transport::{
};
use crate::disk::error::{Error, Result};
use crate::disk::{
CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader,
FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
DiskOption, FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
disk_store::{
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
@@ -30,6 +31,7 @@ use crate::disk::{
},
endpoint::Endpoint,
health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state},
validate_batch_read_version_item_count,
};
use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
use crate::set_disk::DEFAULT_READ_BUFFER_SIZE;
@@ -40,11 +42,11 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_protos::evict_failed_connection;
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
use rustfs_protos::proto_gen::node_service::{
CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest,
DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest,
ReadMetadataRequest, ReadMultipleRequest, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
RenameFileRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
node_service_client::NodeServiceClient,
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, StatVolumeRequest,
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
};
use serde::{Serialize, de::DeserializeOwned};
use std::{
@@ -63,7 +65,7 @@ use tokio::{
time::timeout,
};
use tokio_util::sync::CancellationToken;
use tonic::{Request, service::interceptor::InterceptedService, transport::Channel};
use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
use tracing::{Instrument, debug, warn};
use uuid::Uuid;
@@ -75,11 +77,78 @@ enum FailureHealthAction {
const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2;
const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20);
const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
const BATCH_METADATA_RPC_OFF: &str = "off";
const BATCH_METADATA_RPC_AUTO: &str = "auto";
const BATCH_METADATA_RPC_ON: &str = "on";
const BATCH_READ_VERSION_GATE_ATTEMPT: &str = "attempt";
const BATCH_READ_VERSION_GATE_OFF_UNARY: &str = "off_unary";
const BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED: &str = "fallback_unimplemented";
const BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK: &str = "unsupported_no_fallback";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum BatchMetadataRpcMode {
Off,
Auto,
On,
}
impl BatchMetadataRpcMode {
fn as_str(self) -> &'static str {
match self {
Self::Off => BATCH_METADATA_RPC_OFF,
Self::Auto => BATCH_METADATA_RPC_AUTO,
Self::On => BATCH_METADATA_RPC_ON,
}
}
fn should_attempt(self) -> bool {
matches!(self, Self::Auto | Self::On)
}
fn should_fallback_on_unimplemented(self) -> bool {
matches!(self, Self::Auto)
}
}
fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
match raw.trim() {
value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_AUTO) => BatchMetadataRpcMode::Auto,
value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_ON) => BatchMetadataRpcMode::On,
value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_OFF) => BatchMetadataRpcMode::Off,
_ => BatchMetadataRpcMode::Off,
}
}
fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
.or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
.as_deref()
.map(parse_batch_metadata_rpc_mode)
.unwrap_or(BatchMetadataRpcMode::Off)
}
fn batch_metadata_rpc_mode() -> BatchMetadataRpcMode {
// The gate cannot change at runtime; parse it once instead of re-reading
// the environment on every batch RPC.
static MODE: std::sync::LazyLock<BatchMetadataRpcMode> = std::sync::LazyLock::new(batch_metadata_rpc_mode_from_env);
*MODE
}
fn record_batch_read_version_gate_decision(mode: BatchMetadataRpcMode, decision: &'static str) {
counter!(
"rustfs_remote_disk_batch_read_version_gate_total",
"mode" => mode.as_str(),
"decision" => decision
)
.increment(1);
}
async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64>
where
R: AsyncRead + Unpin,
@@ -769,6 +838,86 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str) -> Res
serde_json::from_str(json).map_err(Error::from)
}
fn decode_read_multiple_response_items(response: ReadMultipleResponse, endpoint: &Endpoint) -> Result<Vec<ReadMultipleResp>> {
if !response.read_multiple_resps_bin.is_empty() {
if !response.read_multiple_resps.is_empty()
&& response.read_multiple_resps.len() != response.read_multiple_resps_bin.len()
{
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
json_count = response.read_multiple_resps.len(),
msgpack_count = response.read_multiple_resps_bin.len(),
op = "read_multiple",
state = "response_count_mismatch",
"Remote disk ReadMultiple compatibility payload counts differ"
);
}
let mut read_multiple_resps = Vec::with_capacity(response.read_multiple_resps_bin.len());
for (index, buf) in response.read_multiple_resps_bin.iter().enumerate() {
let resp = decode_msgpack_or_json::<ReadMultipleResp>(buf, "").map_err(|err| {
Error::other(format!("decode ReadMultipleResp msgpack item {index} from {endpoint} failed: {err}"))
})?;
read_multiple_resps.push(resp);
}
return Ok(read_multiple_resps);
}
let mut read_multiple_resps = Vec::with_capacity(response.read_multiple_resps.len());
for (index, json_str) in response.read_multiple_resps.iter().enumerate() {
let resp = serde_json::from_str::<ReadMultipleResp>(json_str)
.map_err(|err| Error::other(format!("decode ReadMultipleResp json item {index} from {endpoint} failed: {err}")))?;
read_multiple_resps.push(resp);
}
Ok(read_multiple_resps)
}
fn decode_batch_read_version_response_items(
response: BatchReadVersionResponse,
endpoint: &Endpoint,
) -> Result<Vec<BatchReadVersionResp>> {
if !response.batch_read_version_resps_bin.is_empty() {
if !response.batch_read_version_resps.is_empty()
&& response.batch_read_version_resps.len() != response.batch_read_version_resps_bin.len()
{
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
json_count = response.batch_read_version_resps.len(),
msgpack_count = response.batch_read_version_resps_bin.len(),
op = "batch_read_version",
state = "response_count_mismatch",
"Remote disk BatchReadVersion compatibility payload counts differ"
);
}
let mut batch_read_version_resps = Vec::with_capacity(response.batch_read_version_resps_bin.len());
for (index, buf) in response.batch_read_version_resps_bin.iter().enumerate() {
let resp = decode_msgpack_or_json::<BatchReadVersionResp>(buf, "").map_err(|err| {
Error::other(format!("decode BatchReadVersionResp msgpack item {index} from {endpoint} failed: {err}"))
})?;
batch_read_version_resps.push(resp);
}
return Ok(batch_read_version_resps);
}
let mut batch_read_version_resps = Vec::with_capacity(response.batch_read_version_resps.len());
for (index, json_str) in response.batch_read_version_resps.iter().enumerate() {
let resp = serde_json::from_str::<BatchReadVersionResp>(json_str).map_err(|err| {
Error::other(format!("decode BatchReadVersionResp json item {index} from {endpoint} failed: {err}"))
})?;
batch_read_version_resps.push(resp);
}
Ok(batch_read_version_resps)
}
// TODO: all api need to handle errors
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
@@ -1462,6 +1611,99 @@ impl DiskAPI for RemoteDisk {
.await
}
#[tracing::instrument(skip(self, req))]
async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
validate_batch_read_version_item_count(req.items.len())?;
let mode = batch_metadata_rpc_mode();
if !mode.should_attempt() {
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_OFF_UNARY);
return batch_read_version_one_by_one(self, req).await;
}
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_ATTEMPT);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
item_count = req.items.len(),
batch_metadata_rpc_mode = mode.as_str(),
op = "batch_read_version",
state = "started",
"Remote disk RPC started"
);
let batch_read_version_req = serde_json::to_string(&req)?;
let batch_read_version_req_bin = encode_msgpack(&req)?;
let batch_result = self
.execute_with_timeout_for_op(
"batch_read_version",
move || async move {
let disk = self.disk_ref().await;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(BatchReadVersionRequest {
disk,
batch_read_version_req,
batch_read_version_req_bin: batch_read_version_req_bin.into(),
});
let response = match client.batch_read_version(request).await {
Ok(response) => response.into_inner(),
Err(status) if status.code() == Code::Unimplemented => {
if mode.should_fallback_on_unimplemented() {
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
batch_metadata_rpc_mode = mode.as_str(),
op = "batch_read_version",
state = "fallback_unimplemented",
"Remote disk BatchReadVersion unsupported; falling back to unary read_version"
);
return Ok(None);
}
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
batch_metadata_rpc_mode = mode.as_str(),
op = "batch_read_version",
state = "unsupported_no_fallback",
"Remote disk BatchReadVersion unsupported and explicit batch RPC mode forbids fallback"
);
return Err(Error::from(status));
}
Err(status) => return Err(Error::from(status)),
};
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
},
get_max_timeout_duration(),
)
.await?;
match batch_result {
Some(batch_read_version_resps) => Ok(batch_read_version_resps),
// Run the unary fallback outside the batch RPC deadline so each
// read_version keeps its own per-op timeout and health accounting
// instead of racing the whole batch against one drive timeout.
None => batch_read_version_one_by_one(self, req).await,
}
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
debug!(
@@ -2069,19 +2311,7 @@ impl DiskAPI for RemoteDisk {
return Err(response.error.unwrap_or_default().into());
}
let read_multiple_resps = if !response.read_multiple_resps_bin.is_empty() {
response
.read_multiple_resps_bin
.into_iter()
.filter_map(|buf| decode_msgpack_or_json::<ReadMultipleResp>(&buf, "").ok())
.collect()
} else {
response
.read_multiple_resps
.into_iter()
.filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
.collect()
};
let read_multiple_resps = decode_read_multiple_response_items(response, &self.endpoint)?;
Ok(read_multiple_resps)
},
@@ -2413,6 +2643,196 @@ mod tests {
}
}
fn sample_read_multiple_resp(file: &str, data: &[u8]) -> ReadMultipleResp {
ReadMultipleResp {
bucket: "bucket".to_string(),
prefix: "prefix".to_string(),
file: file.to_string(),
exists: true,
data: data.to_vec(),
..Default::default()
}
}
fn sample_remote_endpoint() -> Endpoint {
Endpoint {
url: url::Url::parse("http://server:9000/disk-a").expect("endpoint URL should parse"),
is_local: false,
pool_idx: 0,
set_idx: 0,
disk_idx: 0,
}
}
#[test]
fn read_multiple_response_decode_prefers_msgpack_payloads() {
let endpoint = sample_remote_endpoint();
let msgpack_resp = sample_read_multiple_resp("msgpack", b"binary");
let json_resp = sample_read_multiple_resp("json", b"fallback");
let response = ReadMultipleResponse {
success: true,
read_multiple_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")],
read_multiple_resps_bin: vec![encode_msgpack(&msgpack_resp).expect("msgpack response should encode").into()],
error: None,
};
let decoded = decode_read_multiple_response_items(response, &endpoint).expect("msgpack response should decode");
assert_eq!(decoded.len(), 1);
assert_eq!(decoded[0].file, "msgpack");
assert_eq!(decoded[0].data, b"binary");
}
#[test]
fn read_multiple_response_decode_falls_back_to_json_payloads() {
let endpoint = sample_remote_endpoint();
let json_resp = sample_read_multiple_resp("json", b"fallback");
let response = ReadMultipleResponse {
success: true,
read_multiple_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")],
read_multiple_resps_bin: Vec::new(),
error: None,
};
let decoded = decode_read_multiple_response_items(response, &endpoint).expect("json response should decode");
assert_eq!(decoded.len(), 1);
assert_eq!(decoded[0].file, "json");
assert_eq!(decoded[0].data, b"fallback");
}
#[test]
fn read_multiple_response_decode_reports_corrupt_msgpack_item() {
let endpoint = sample_remote_endpoint();
let response = ReadMultipleResponse {
success: true,
read_multiple_resps: Vec::new(),
read_multiple_resps_bin: vec![
encode_msgpack(&sample_read_multiple_resp("ok", b"data"))
.expect("msgpack response should encode")
.into(),
bytes::Bytes::from_static(b"not-msgpack"),
],
error: None,
};
let err = decode_read_multiple_response_items(response, &endpoint).expect_err("corrupt msgpack item should fail");
let err = err.to_string();
assert!(err.contains("ReadMultipleResp msgpack item 1"), "unexpected error: {err}");
assert!(err.contains("server:9000"), "unexpected error: {err}");
}
fn sample_batch_read_version_resp(index: usize, path: &str, success: bool) -> BatchReadVersionResp {
BatchReadVersionResp {
index,
path: path.to_string(),
version_id: "version-a".to_string(),
success,
error: if success {
String::new()
} else {
"file version not found".to_string()
},
..Default::default()
}
}
#[test]
fn batch_read_version_response_decode_prefers_msgpack_payloads() {
let endpoint = sample_remote_endpoint();
let msgpack_resp = sample_batch_read_version_resp(7, "msgpack-object", true);
let json_resp = sample_batch_read_version_resp(1, "json-object", false);
let response = BatchReadVersionResponse {
success: true,
batch_read_version_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")],
batch_read_version_resps_bin: vec![encode_msgpack(&msgpack_resp).expect("msgpack response should encode").into()],
error: None,
};
let decoded = decode_batch_read_version_response_items(response, &endpoint).expect("msgpack response should decode");
assert_eq!(decoded.len(), 1);
assert_eq!(decoded[0].index, 7);
assert_eq!(decoded[0].path, "msgpack-object");
assert!(decoded[0].success);
}
#[test]
fn batch_read_version_response_decode_reports_corrupt_msgpack_item() {
let endpoint = sample_remote_endpoint();
let response = BatchReadVersionResponse {
success: true,
batch_read_version_resps: Vec::new(),
batch_read_version_resps_bin: vec![
encode_msgpack(&sample_batch_read_version_resp(0, "ok", true))
.expect("msgpack response should encode")
.into(),
bytes::Bytes::from_static(b"not-msgpack"),
],
error: None,
};
let err = decode_batch_read_version_response_items(response, &endpoint)
.expect_err("corrupt msgpack item should fail")
.to_string();
assert!(err.contains("BatchReadVersionResp msgpack item 1"), "unexpected error: {err}");
assert!(err.contains("server:9000"), "unexpected error: {err}");
}
#[test]
fn batch_metadata_rpc_mode_defaults_to_off_and_parses_supported_values() {
assert_eq!(parse_batch_metadata_rpc_mode(""), BatchMetadataRpcMode::Off);
assert_eq!(parse_batch_metadata_rpc_mode("off"), BatchMetadataRpcMode::Off);
assert_eq!(parse_batch_metadata_rpc_mode("auto"), BatchMetadataRpcMode::Auto);
assert_eq!(parse_batch_metadata_rpc_mode("on"), BatchMetadataRpcMode::On);
assert_eq!(parse_batch_metadata_rpc_mode("unknown"), BatchMetadataRpcMode::Off);
assert_eq!(BatchMetadataRpcMode::Off.as_str(), "off");
assert_eq!(BatchMetadataRpcMode::Auto.as_str(), "auto");
assert_eq!(BatchMetadataRpcMode::On.as_str(), "on");
assert!(!BatchMetadataRpcMode::Off.should_attempt());
assert!(BatchMetadataRpcMode::Auto.should_attempt());
assert!(BatchMetadataRpcMode::On.should_attempt());
assert_eq!(BATCH_READ_VERSION_GATE_ATTEMPT, "attempt");
assert_eq!(BATCH_READ_VERSION_GATE_OFF_UNARY, "off_unary");
assert_eq!(BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED, "fallback_unimplemented");
assert_eq!(BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK, "unsupported_no_fallback");
}
#[test]
fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
|| {
assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::Auto);
},
);
}
#[test]
fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
|| {
assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
},
);
}
#[test]
fn batch_metadata_rpc_mode_only_auto_falls_back_on_unimplemented() {
assert!(!BatchMetadataRpcMode::Off.should_fallback_on_unimplemented());
assert!(BatchMetadataRpcMode::Auto.should_fallback_on_unimplemented());
assert!(!BatchMetadataRpcMode::On.should_fallback_on_unimplemented());
}
#[test]
fn rename_data_file_info_named_msgpack_is_smaller_than_json() {
let file_info = sample_rename_data_file_info();
crates/ecstore/src/cluster/rpc/remote_locker.rs
+178 -68
View File
@@ -249,6 +249,22 @@ impl RemoteClient {
}
}
}
fn unknown_lock_info(lock_id: &LockId) -> LockInfo {
LockInfo {
id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive,
status: LockStatus::Acquired,
owner: "unknown".to_string(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
last_refreshed: std::time::SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
}
}
}
#[async_trait]
@@ -338,11 +354,11 @@ impl LockClient for RemoteClient {
let request_string = serde_json::to_string(&unlock_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
let mut client = self.get_client().await?;
let resource_summary = unlock_request.resource.to_string();
let req = Request::new(GenerallyLockRequest { args: request_string });
let resp = client
.un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
let resp = self
.execute_rpc("release", &resource_summary, client.un_lock(req))
.await?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
@@ -351,21 +367,25 @@ impl LockClient for RemoteClient {
}
async fn release_locks_batch(&self, lock_ids: &[LockId]) -> Result<Vec<bool>> {
if lock_ids.is_empty() {
return Ok(Vec::new());
}
let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
let mut client = self.get_client().await?;
let resource_summary = Self::summarize_resources(&unlock_requests);
let req = Request::new(BatchGenerallyLockRequest {
args: lock_ids
args: unlock_requests
.iter()
.map(|lock_id| {
serde_json::to_string(&Self::create_unlock_request(lock_id))
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
.map(|request| {
serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
})
.collect::<Result<Vec<_>>>()?,
});
let resp = client
.un_lock_batch(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
let resp = self
.execute_rpc("release_batch", &resource_summary, client.un_lock_batch(req))
.await?
.into_inner();
Ok(lock_ids
@@ -379,14 +399,14 @@ impl LockClient for RemoteClient {
info!("remote refresh for {}", lock_id);
let refresh_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
let resource_summary = refresh_request.resource.to_string();
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&refresh_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.refresh(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
let resp = self
.execute_rpc("refresh", &resource_summary, client.refresh(req))
.await?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
@@ -398,14 +418,14 @@ impl LockClient for RemoteClient {
info!("remote force_release for {}", lock_id);
let force_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
let resource_summary = force_request.resource.to_string();
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&force_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.force_un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
let resp = self
.execute_rpc("force_release", &resource_summary, client.force_un_lock(req))
.await?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
@@ -419,6 +439,7 @@ impl LockClient for RemoteClient {
// Since there's no direct status query in the gRPC service,
// we attempt a non-blocking lock acquisition to check if the resource is available
let status_request = Self::create_unlock_request(lock_id);
let resource_summary = status_request.resource.to_string();
let mut client = self.get_client().await?;
// Try to acquire a very short-lived lock to test availability
@@ -428,56 +449,27 @@ impl LockClient for RemoteClient {
});
// Try exclusive lock first with very short timeout
let resp = client.lock(req).await;
let resp = match self.execute_rpc("check_status", &resource_summary, client.lock(req)).await {
Ok(response) => response.into_inner(),
Err(_) => return Ok(Some(Self::unknown_lock_info(lock_id))),
};
match resp {
Ok(response) => {
let resp = response.into_inner();
if resp.success {
// If we successfully acquired the lock, the resource was free
// Immediately release it
let release_req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&status_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let _ = client.un_lock(release_req).await; // Best effort release
if resp.success {
// If we successfully acquired the lock, the resource was free.
// Immediately release it on a best-effort basis.
let release_req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&status_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let _ = self
.execute_rpc("check_status_release", &resource_summary, client.un_lock(release_req))
.await;
// Return None since no one was holding the lock
Ok(None)
} else {
// Lock acquisition failed, meaning someone is holding it
// We can't determine the exact details remotely, so return a generic status
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive, // We can't know the exact type
status: LockStatus::Acquired,
owner: "unknown".to_string(), // Remote client can't determine owner
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
last_refreshed: std::time::SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
}))
}
}
Err(_) => {
// Communication error or lock is held
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive,
status: LockStatus::Acquired,
owner: "unknown".to_string(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
last_refreshed: std::time::SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
}))
}
Ok(None)
} else {
// Lock acquisition failed, meaning someone is holding it.
// We can't determine the exact details remotely, so return a generic status.
Ok(Some(Self::unknown_lock_info(lock_id)))
}
}
@@ -558,6 +550,17 @@ mod tests {
Some((addr, task))
}
async fn closed_listener_addr() -> Option<String> {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
Err(err) => panic!("test listener should bind: {err}"),
};
let addr = format!("http://{}", listener.local_addr().expect("listener local address should be available"));
drop(listener);
Some(addr)
}
async fn cache_lazy_channel(addr: &str) {
let channel = TonicEndpoint::from_shared(addr.to_string()).unwrap().connect_lazy();
runtime_sources::cache_test_node_channel(addr.to_string(), channel).await;
@@ -664,6 +667,113 @@ mod tests {
accept_task.abort();
}
#[tokio::test]
#[serial_test::serial]
async fn test_remote_client_release_uses_rpc_timeout_and_evicts_connection() {
ensure_test_rpc_secret();
let Some((addr, accept_task)) = spawn_hanging_listener().await else {
return;
};
cache_lazy_channel(&addr).await;
assert!(runtime_sources::test_node_channel_is_cached(&addr).await);
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("50"))], async {
let client = RemoteClient::new(addr.clone());
let request = test_lock_request(Duration::from_millis(5));
let started_at = tokio::time::Instant::now();
let err = client.release(&request.lock_id).await.expect_err("release should time out");
let elapsed = started_at.elapsed();
assert!(
elapsed >= Duration::from_millis(40),
"remote release RPC should use configured transport timeout, got {elapsed:?}"
);
assert!(
elapsed < Duration::from_secs(1),
"test RPC timeout should keep the test fast, got {elapsed:?}"
);
assert!(matches!(err, LockError::Timeout { .. }), "expected remote release timeout, got {err:?}");
assert!(
!runtime_sources::test_node_channel_is_cached(&addr).await,
"release timeout should evict cached connection"
);
})
.await;
accept_task.abort();
}
#[tokio::test]
#[serial_test::serial]
async fn test_remote_client_refresh_tonic_error_evicts_connection() {
ensure_test_rpc_secret();
let Some(addr) = closed_listener_addr().await else {
return;
};
cache_lazy_channel(&addr).await;
assert!(runtime_sources::test_node_channel_is_cached(&addr).await);
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("500"))], async {
let client = RemoteClient::new(addr.clone());
let request = test_lock_request(Duration::from_millis(5));
let err = client
.refresh(&request.lock_id)
.await
.expect_err("refresh should report tonic failure");
assert!(
err.to_string().contains("refresh RPC failed"),
"expected refresh RPC failure marker, got {err}"
);
assert!(
!runtime_sources::test_node_channel_is_cached(&addr).await,
"refresh tonic error should evict cached connection"
);
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_remote_client_check_status_timeout_evicts_connection_and_preserves_status_shape() {
ensure_test_rpc_secret();
let Some((addr, accept_task)) = spawn_hanging_listener().await else {
return;
};
cache_lazy_channel(&addr).await;
assert!(runtime_sources::test_node_channel_is_cached(&addr).await);
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("50"))], async {
let client = RemoteClient::new(addr.clone());
let request = test_lock_request(Duration::from_millis(5));
let started_at = tokio::time::Instant::now();
let status = client.check_status(&request.lock_id).await.unwrap();
let elapsed = started_at.elapsed();
assert!(
elapsed >= Duration::from_millis(40),
"remote check_status RPC should use configured transport timeout, got {elapsed:?}"
);
assert!(
elapsed < Duration::from_secs(1),
"test RPC timeout should keep the test fast, got {elapsed:?}"
);
let info = status.expect("communication failure should preserve unknown lock status shape");
assert_eq!(info.id, request.lock_id);
assert_eq!(info.owner, "unknown");
assert!(
!runtime_sources::test_node_channel_is_cached(&addr).await,
"check_status timeout should evict cached connection"
);
})
.await;
accept_task.abort();
}
#[test]
#[serial_test::serial]
fn test_remote_client_rpc_timeout_honors_configured_deadline() {
crates/ecstore/src/disk/mod.rs
+73
View File
@@ -577,6 +577,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo>;
async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
batch_read_version_one_by_one(self, req).await
}
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo>;
async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes>;
async fn rename_data(
@@ -637,6 +640,41 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
fn start_scan(&self) -> ScanGuard;
}
pub async fn batch_read_version_one_by_one<D>(disk: &D, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>>
where
D: DiskAPI + ?Sized,
{
validate_batch_read_version_item_count(req.items.len())?;
let mut responses = Vec::with_capacity(req.items.len());
for (index, item) in req.items.iter().enumerate() {
let response = match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: true,
file_info,
error: String::new(),
},
Err(err) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
},
};
responses.push(response);
}
Ok(responses)
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct CheckPartsResp {
pub results: Vec<usize>,
@@ -807,6 +845,41 @@ pub struct ReadMultipleResp {
pub mod_time: Option<OffsetDateTime>,
}
pub const BATCH_READ_VERSION_MAX_ITEMS: usize = 128;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BatchReadVersionItem {
pub org_volume: String,
pub volume: String,
pub path: String,
pub version_id: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BatchReadVersionReq {
pub items: Vec<BatchReadVersionItem>,
pub opts: ReadOptions,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct BatchReadVersionResp {
pub index: usize,
pub path: String,
pub version_id: String,
pub success: bool,
pub file_info: FileInfo,
pub error: String,
}
pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
if item_count > BATCH_READ_VERSION_MAX_ITEMS {
return Err(DiskError::other(format!(
"batch read version item count {item_count} exceeds limit {BATCH_READ_VERSION_MAX_ITEMS}"
)));
}
Ok(())
}
#[derive(Debug, Deserialize, Serialize)]
pub struct VolumeInfo {
pub name: String,
crates/ecstore/src/erasure/coding/decode.rs
+317 -115
View File
@@ -40,18 +40,54 @@ use tracing::{error, warn};
type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, bool)> + Send + 'a>>;
const ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING: &str = "RUSTFS_SHARD_LOCALITY_SCHEDULING";
const ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: &str = "RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE";
const DEFAULT_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: bool = false;
const SHARD_LOCALITY_SCHEDULING_OFF: &str = "off";
const SHARD_LOCALITY_SCHEDULING_OBSERVE: &str = "observe";
const SHARD_LOCALITY_SCHEDULING_ON: &str = "on";
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum ShardLocalitySchedulingMode {
Off,
Observe,
On,
}
impl ShardLocalitySchedulingMode {
fn is_on(self) -> bool {
matches!(self, ShardLocalitySchedulingMode::On)
}
}
fn parse_shard_locality_scheduling_mode(value: &str) -> ShardLocalitySchedulingMode {
match value.trim() {
value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_OFF) => ShardLocalitySchedulingMode::Off,
value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_OBSERVE) => ShardLocalitySchedulingMode::Observe,
value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_ON) => ShardLocalitySchedulingMode::On,
_ => ShardLocalitySchedulingMode::Off,
}
}
fn get_shard_locality_scheduling_mode() -> ShardLocalitySchedulingMode {
if let Some(value) = rustfs_utils::get_env_opt_str(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING) {
return parse_shard_locality_scheduling_mode(&value);
}
if rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, false) {
return ShardLocalitySchedulingMode::On;
}
ShardLocalitySchedulingMode::Off
}
fn get_shard_locality_preference_enabled() -> bool {
rustfs_utils::get_env_bool(
ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE,
DEFAULT_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE,
)
get_shard_locality_scheduling_mode().is_on()
}
pub(crate) fn should_collect_shard_read_costs() -> bool {
rustfs_io_metrics::get_stage_metrics_enabled() || get_shard_locality_preference_enabled()
// `observe` mode only feeds the stage-metrics histograms, so skip the
// cost-collection overhead when those metrics cannot be reported anyway.
rustfs_io_metrics::get_stage_metrics_enabled() || get_shard_locality_scheduling_mode().is_on()
}
/// Number of stripes to prefetch in the legacy decode path.
@@ -108,9 +144,10 @@ impl ShardReadCostCounts {
}
fn shard_read_launch_rank(cost: ShardReadCost) -> u8 {
match cost {
ShardReadCost::Local | ShardReadCost::SameNode => 0,
ShardReadCost::Unknown => 1,
ShardReadCost::Local => 0,
ShardReadCost::SameNode => 1,
ShardReadCost::Remote => 2,
ShardReadCost::Unknown => 3,
}
}
@@ -533,6 +570,11 @@ fn shard_read_hedge_delay(read_timeout: Duration) -> Option<Duration> {
}
}
fn shard_locality_remote_avoid_potential(remote_scheduled: usize, low_cost_available: usize, data_shards: usize) -> usize {
let theoretical_remote_needed = data_shards.saturating_sub(low_cost_available);
remote_scheduled.saturating_sub(theoretical_remote_needed)
}
#[allow(clippy::too_many_arguments)]
fn record_scheduled_read_cost(
read_cost: ShardReadCost,
@@ -544,15 +586,13 @@ fn record_scheduled_read_cost(
remote_scheduled: &mut usize,
fallback_to_remote: &mut usize,
) {
if !locality_preference_enabled {
return;
}
if read_cost.is_low_cost() {
*local_preferred += 1;
if locality_preference_enabled {
*local_preferred += 1;
}
} else if read_cost.is_remote() {
*remote_scheduled += 1;
if count_remote_as_fallback || low_cost_available < data_shards {
if locality_preference_enabled && (count_remote_as_fallback || low_cost_available < data_shards) {
*fallback_to_remote += 1;
}
}
@@ -889,6 +929,13 @@ where
fallback_to_remote,
);
} else {
let remote_avoid_potential =
shard_locality_remote_avoid_potential(remote_scheduled, low_cost_available, self.data_shards);
rustfs_io_metrics::record_get_object_shard_locality_observe_only(
path,
remote_scheduled,
remote_avoid_potential,
);
rustfs_io_metrics::record_get_object_shard_locality_policy_disabled(path);
}
}
@@ -1598,14 +1645,138 @@ mod tests {
}
#[test]
#[serial_test::serial]
fn test_shard_locality_preference_gate_defaults_disabled() {
temp_env::with_var(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>, || {
assert!(!get_shard_locality_preference_enabled());
});
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Off);
assert!(!get_shard_locality_preference_enabled());
},
);
}
#[test]
fn test_shard_read_launch_order_is_gated() {
#[serial_test::serial]
fn test_shard_locality_scheduling_mode_parses_supported_values() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("observe")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Observe);
assert!(!get_shard_locality_preference_enabled());
},
);
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::On);
assert!(get_shard_locality_preference_enabled());
assert!(should_collect_shard_read_costs());
},
);
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("unexpected")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Off);
assert!(!get_shard_locality_preference_enabled());
},
);
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::On);
assert!(get_shard_locality_preference_enabled());
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_scheduling_off_does_not_collect_without_metrics() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("off")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
assert!(!should_collect_shard_read_costs());
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_scheduling_observe_collects_only_with_stage_metrics() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("observe")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
// Without stage metrics there is no reporting channel, so
// observe mode must not pay the cost-collection overhead.
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
assert!(!should_collect_shard_read_costs());
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
assert!(should_collect_shard_read_costs());
assert!(!get_shard_locality_preference_enabled());
let read_costs = [ShardReadCost::Remote, ShardReadCost::Local, ShardReadCost::SameNode];
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), false), vec![0, 1, 2]);
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_scheduling_on_enables_reordering() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert!(get_shard_locality_preference_enabled());
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_legacy_preference_gate_still_enables_on() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
],
|| {
assert!(get_shard_locality_preference_enabled());
},
);
}
#[test]
fn test_shard_locality_read_launch_order_is_gated() {
let read_costs = [
ShardReadCost::Remote,
ShardReadCost::Local,
@@ -1615,7 +1786,14 @@ mod tests {
];
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), false), vec![0, 1, 2, 3, 4]);
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), true), vec![1, 3, 2, 0, 4]);
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), true), vec![1, 3, 0, 4, 2]);
}
#[test]
fn test_shard_locality_remote_avoid_potential_is_observe_only() {
assert_eq!(shard_locality_remote_avoid_potential(2, 4, 4), 2);
assert_eq!(shard_locality_remote_avoid_potential(2, 2, 4), 0);
assert_eq!(shard_locality_remote_avoid_potential(3, 3, 4), 2);
}
#[test]
@@ -1639,90 +1817,107 @@ mod tests {
#[tokio::test]
#[serial_test::serial]
async fn test_parallel_reader_local_first_avoids_remote_when_local_quorum_exists() {
temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::HighwayHash256;
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[]).await;
let read_costs = vec![
ShardReadCost::Remote,
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::Local,
ShardReadCost::SameNode,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
read_costs,
);
let hash_algo = HashAlgorithm::HighwayHash256;
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[]).await;
let read_costs = vec![
ShardReadCost::Remote,
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::Local,
ShardReadCost::SameNode,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
read_costs,
);
let (bufs, errs) = parallel_reader.read().await;
let (bufs, errs) = parallel_reader.read().await;
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert!(bufs[0].is_none());
assert!(bufs[1].is_none());
for (index, buf) in bufs.iter().enumerate().take(DATA_SHARDS + PARITY_SHARDS).skip(2) {
assert_eq!(buf.as_deref(), Some(&[(index % 256) as u8; SHARD_SIZE][..]));
}
assert!(errs.iter().all(Option::is_none));
})
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert!(bufs[0].is_none());
assert!(bufs[1].is_none());
for (index, buf) in bufs.iter().enumerate().take(DATA_SHARDS + PARITY_SHARDS).skip(2) {
assert_eq!(buf.as_deref(), Some(&[(index % 256) as u8; SHARD_SIZE][..]));
}
assert!(errs.iter().all(Option::is_none));
},
)
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_parallel_reader_local_missing_falls_back_to_remote() {
temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::HighwayHash256;
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[1], &[]).await;
let read_costs = vec![
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Remote,
ShardReadCost::Remote,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
read_costs,
);
let hash_algo = HashAlgorithm::HighwayHash256;
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[1], &[]).await;
let read_costs = vec![
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Remote,
ShardReadCost::Remote,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
read_costs,
);
let (bufs, errs) = parallel_reader.read().await;
let (bufs, errs) = parallel_reader.read().await;
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert!(matches!(errs[1], Some(Error::FileNotFound)));
assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
assert!(bufs[5].is_none());
})
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert!(matches!(errs[1], Some(Error::FileNotFound)));
assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
assert!(bufs[5].is_none());
},
)
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_parallel_reader_local_corrupt_falls_back_to_remote() {
temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
@@ -1757,45 +1952,52 @@ mod tests {
);
assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
assert!(bufs[5].is_none());
})
},
)
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_erasure_decode_local_first_preserves_output_order() {
temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..BLOCK_SIZE as u32).map(|i| i as u8).collect();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let shard_bufs = encode_test_object(&erasure, &total_data, shard_size, &hash_algo).await;
let readers = shard_bufs
.iter()
.map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false)))
.collect();
let read_costs = vec![
ShardReadCost::Remote,
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::SameNode,
];
let total_data: Vec<u8> = (0..BLOCK_SIZE as u32).map(|i| i as u8).collect();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let shard_bufs = encode_test_object(&erasure, &total_data, shard_size, &hash_algo).await;
let readers = shard_bufs
.iter()
.map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false)))
.collect();
let read_costs = vec![
ShardReadCost::Remote,
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::SameNode,
];
let mut output = Vec::new();
let (written, err) = erasure
.decode_with_read_costs(&mut output, readers, 0, total_data.len(), total_data.len(), read_costs)
.await;
let mut output = Vec::new();
let (written, err) = erasure
.decode_with_read_costs(&mut output, readers, 0, total_data.len(), total_data.len(), read_costs)
.await;
assert!(err.is_none(), "unexpected decode error: {err:?}");
assert_eq!(written, total_data.len());
assert_eq!(output, total_data);
})
assert!(err.is_none(), "unexpected decode error: {err:?}");
assert_eq!(written, total_data.len());
assert_eq!(output, total_data);
},
)
.await;
}
crates/ecstore/src/services/batch_processor.rs
+545 -16
View File
@@ -19,17 +19,255 @@
use crate::disk::error::{Error, Result};
use std::future::Future;
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tokio::task::JoinSet;
const BATCH_PROCESSOR_OPERATION_CUSTOM: &str = "custom";
const BATCH_PROCESSOR_OPERATION_READ: &str = "read";
const BATCH_PROCESSOR_OPERATION_WRITE: &str = "write";
const BATCH_PROCESSOR_OPERATION_METADATA: &str = "metadata";
const ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE: &str = "RUSTFS_BATCH_PROCESSOR_ADAPTIVE";
const BATCH_PROCESSOR_ADAPTIVE_OFF: &str = "off";
const BATCH_PROCESSOR_ADAPTIVE_OBSERVE: &str = "observe";
const BATCH_PROCESSOR_ADAPTIVE_ON: &str = "on";
const BATCH_SUGGESTION_REASON_IMPROVING: &str = "improving";
const BATCH_SUGGESTION_REASON_DEGRADING: &str = "degrading";
const BATCH_SUGGESTION_REASON_STABLE: &str = "stable";
const BATCH_SUGGESTION_REASON_COOLDOWN: &str = "cooldown";
const BATCH_OBSERVATION_IMPROVING_LATENCY: Duration = Duration::from_millis(50);
const BATCH_OBSERVATION_DEGRADING_LATENCY: Duration = Duration::from_millis(250);
const BATCH_OBSERVATION_COOLDOWN: Duration = Duration::from_secs(30);
const BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR: usize = 4;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum BatchProcessorAdaptiveMode {
Off,
Observe,
On,
}
impl BatchProcessorAdaptiveMode {
fn should_observe(self) -> bool {
matches!(self, Self::Observe | Self::On)
}
fn should_apply(self) -> bool {
matches!(self, Self::On)
}
}
fn parse_batch_processor_adaptive_mode(raw: &str) -> BatchProcessorAdaptiveMode {
match raw.trim() {
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OBSERVE) => BatchProcessorAdaptiveMode::Observe,
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_ON) => BatchProcessorAdaptiveMode::On,
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OFF) => BatchProcessorAdaptiveMode::Off,
_ => BatchProcessorAdaptiveMode::Off,
}
}
fn batch_processor_adaptive_mode_from_env() -> BatchProcessorAdaptiveMode {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE)
.as_deref()
.map(parse_batch_processor_adaptive_mode)
.unwrap_or(BatchProcessorAdaptiveMode::Off)
}
fn batch_processor_adaptive_mode() -> BatchProcessorAdaptiveMode {
// The gate cannot change at runtime; parse it once instead of re-reading
// the environment on every batch.
static MODE: std::sync::LazyLock<BatchProcessorAdaptiveMode> =
std::sync::LazyLock::new(batch_processor_adaptive_mode_from_env);
*MODE
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct BatchConcurrencySuggestion {
concurrency: usize,
reason: &'static str,
}
#[derive(Debug, Clone, Copy)]
struct BatchObservation {
batch_size: usize,
success_count: usize,
error_count: usize,
timeout_count: usize,
max_queue_wait: Duration,
execution_latency: Duration,
}
#[derive(Debug)]
struct BatchObservationState {
last_suggestion_at: Option<Instant>,
last_suggested_concurrency: usize,
}
impl BatchObservationState {
fn new(configured_concurrency: usize) -> Self {
Self {
last_suggestion_at: None,
last_suggested_concurrency: configured_concurrency,
}
}
fn suggest(
&mut self,
current_concurrency: usize,
configured_concurrency: usize,
observation: BatchObservation,
now: Instant,
) -> BatchConcurrencySuggestion {
let raw = calculate_batch_concurrency_suggestion(current_concurrency, configured_concurrency, observation);
if raw.concurrency == current_concurrency {
self.last_suggested_concurrency = current_concurrency;
return raw;
}
if self
.last_suggestion_at
.is_some_and(|last_suggestion_at| now.duration_since(last_suggestion_at) < BATCH_OBSERVATION_COOLDOWN)
{
return BatchConcurrencySuggestion {
concurrency: self.last_suggested_concurrency,
reason: BATCH_SUGGESTION_REASON_COOLDOWN,
};
}
self.last_suggestion_at = Some(now);
self.last_suggested_concurrency = raw.concurrency;
raw
}
}
fn calculate_batch_concurrency_suggestion(
current_concurrency: usize,
configured_concurrency: usize,
observation: BatchObservation,
) -> BatchConcurrencySuggestion {
let current_concurrency = current_concurrency.max(1);
let configured_concurrency = configured_concurrency.max(1);
if observation.batch_size == 0 {
return BatchConcurrencySuggestion {
concurrency: current_concurrency,
reason: BATCH_SUGGESTION_REASON_STABLE,
};
}
if observation.timeout_count > 0
|| observation.error_count > 0
|| (observation.batch_size >= current_concurrency && observation.execution_latency >= BATCH_OBSERVATION_DEGRADING_LATENCY)
{
return BatchConcurrencySuggestion {
concurrency: decrease_batch_concurrency(current_concurrency),
reason: BATCH_SUGGESTION_REASON_DEGRADING,
};
}
if observation.success_count == observation.batch_size
&& observation.batch_size >= current_concurrency
&& observation.execution_latency <= BATCH_OBSERVATION_IMPROVING_LATENCY
{
return BatchConcurrencySuggestion {
concurrency: increase_batch_concurrency(current_concurrency, configured_concurrency, observation.batch_size),
reason: BATCH_SUGGESTION_REASON_IMPROVING,
};
}
BatchConcurrencySuggestion {
concurrency: current_concurrency,
reason: BATCH_SUGGESTION_REASON_STABLE,
}
}
fn decrease_batch_concurrency(current_concurrency: usize) -> usize {
(current_concurrency.saturating_mul(3) / 4).max(1)
}
fn increase_batch_concurrency(current_concurrency: usize, configured_concurrency: usize, batch_size: usize) -> usize {
let step = (current_concurrency / 4).max(1);
// Suggestions build on the previous suggestion, so bound the ratchet at a
// hard multiple of the configured baseline instead of the current value.
let upper_bound = configured_concurrency
.saturating_mul(BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR)
.max(current_concurrency);
current_concurrency
.saturating_add(step)
.min(upper_bound)
.min(batch_size.max(current_concurrency))
}
fn is_timeout_error(err: &Error) -> bool {
matches!(err, Error::Timeout | Error::SourceStalled)
|| matches!(err, Error::Io(io_err) if io_err.kind() == std::io::ErrorKind::TimedOut)
}
/// Batch processor that executes tasks concurrently with a semaphore
pub struct AsyncBatchProcessor {
max_concurrent: usize,
operation: &'static str,
observation_state: Mutex<BatchObservationState>,
}
impl AsyncBatchProcessor {
pub fn new(max_concurrent: usize) -> Self {
Self { max_concurrent }
Self::new_with_operation(max_concurrent, BATCH_PROCESSOR_OPERATION_CUSTOM)
}
fn new_with_operation(max_concurrent: usize, operation: &'static str) -> Self {
let max_concurrent = max_concurrent.max(1);
Self {
max_concurrent,
operation,
observation_state: Mutex::new(BatchObservationState::new(max_concurrent)),
}
}
fn execution_concurrency(&self, mode: BatchProcessorAdaptiveMode) -> usize {
if !mode.should_apply() {
return self.max_concurrent;
}
self.observation_state
.lock()
.map(|state| state.last_suggested_concurrency.max(1))
.unwrap_or(self.max_concurrent)
}
fn observe_batch_with_mode(
&self,
mode: BatchProcessorAdaptiveMode,
execution_concurrency: usize,
observation: BatchObservation,
) -> BatchConcurrencySuggestion {
let execution_concurrency = execution_concurrency.max(1);
if !mode.should_observe() {
return BatchConcurrencySuggestion {
concurrency: execution_concurrency,
reason: BATCH_SUGGESTION_REASON_STABLE,
};
}
let suggestion = match self.observation_state.lock() {
Ok(mut state) => state.suggest(execution_concurrency, self.max_concurrent, observation, Instant::now()),
Err(_) => calculate_batch_concurrency_suggestion(execution_concurrency, self.max_concurrent, observation),
};
rustfs_io_metrics::record_batch_processor_observation(rustfs_io_metrics::BatchProcessorObservation {
operation: self.operation,
batch_size: observation.batch_size,
configured_concurrency: execution_concurrency,
max_queue_wait_secs: observation.max_queue_wait.as_secs_f64(),
execution_latency_secs: observation.execution_latency.as_secs_f64(),
successes: observation.success_count,
errors: observation.error_count,
timeouts: observation.timeout_count,
suggested_concurrency: suggestion.concurrency,
suggestion_reason: suggestion.reason,
});
suggestion
}
/// Execute a batch of tasks concurrently with concurrency control
@@ -42,27 +280,35 @@ impl AsyncBatchProcessor {
return Vec::new();
}
let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
let batch_size = tasks.len();
let batch_started_at = Instant::now();
let adaptive_mode = batch_processor_adaptive_mode();
let execution_concurrency = self.execution_concurrency(adaptive_mode);
let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency));
let mut join_set = JoinSet::new();
let mut results = Vec::with_capacity(tasks.len());
for _ in 0..tasks.len() {
let mut results = Vec::with_capacity(batch_size);
for _ in 0..batch_size {
results.push(Err(Error::other("Not completed")));
}
// Spawn all tasks with semaphore control
for (i, task) in tasks.into_iter().enumerate() {
let sem = semaphore.clone();
let queued_at = Instant::now();
join_set.spawn(async move {
let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
let queue_wait = queued_at.elapsed();
let result = task.await;
Ok::<(usize, Result<T>), Error>((i, result))
Ok::<(usize, Result<T>, Duration), Error>((i, result, queue_wait))
});
}
// Collect results
let mut max_queue_wait = Duration::ZERO;
while let Some(join_result) = join_set.join_next().await {
match join_result {
Ok(Ok((index, task_result))) => {
Ok(Ok((index, task_result, queue_wait))) => {
max_queue_wait = max_queue_wait.max(queue_wait);
if index < results.len() {
results[index] = task_result;
}
@@ -78,6 +324,29 @@ impl AsyncBatchProcessor {
}
}
let mut success_count = 0;
let mut error_count = 0;
let mut timeout_count = 0;
for result in &results {
match result {
Ok(_) => success_count += 1,
Err(err) if is_timeout_error(err) => timeout_count += 1,
Err(_) => error_count += 1,
}
}
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count,
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
results
}
@@ -97,17 +366,29 @@ impl AsyncBatchProcessor {
)));
}
let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
let batch_size = tasks.len();
let batch_started_at = Instant::now();
let adaptive_mode = batch_processor_adaptive_mode();
let execution_concurrency = self.execution_concurrency(adaptive_mode);
let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency));
let mut join_set = JoinSet::new();
let mut successes = Vec::new();
let mut pending_tasks = tasks.len();
let mut pending_tasks = batch_size;
let mut first_error = None;
let mut error_count = 0;
let mut timeout_count = 0;
let mut max_queue_wait = Duration::ZERO;
for task in tasks {
let sem = semaphore.clone();
let queued_at = Instant::now();
join_set.spawn(async move {
let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
task.await
let _permit = sem
.acquire()
.await
.map_err(|_| (Error::other("Semaphore error"), queued_at.elapsed()))?;
let queue_wait = queued_at.elapsed();
task.await.map(|value| (value, queue_wait)).map_err(|err| (err, queue_wait))
});
}
@@ -115,18 +396,38 @@ impl AsyncBatchProcessor {
pending_tasks = pending_tasks.saturating_sub(1);
match join_result {
Ok(Ok(value)) => {
Ok(Ok((value, queue_wait))) => {
max_queue_wait = max_queue_wait.max(queue_wait);
successes.push(value);
if successes.len() >= required_successes {
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count: successes.len(),
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
return Ok(successes);
}
}
Ok(Err(err)) => {
Ok(Err((err, queue_wait))) => {
max_queue_wait = max_queue_wait.max(queue_wait);
if is_timeout_error(&err) {
timeout_count += 1;
} else {
error_count += 1;
}
if first_error.is_none() {
first_error = Some(err);
}
}
Err(join_error) => {
error_count += 1;
if first_error.is_none() {
first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}")));
}
@@ -134,6 +435,18 @@ impl AsyncBatchProcessor {
}
if successes.len() + pending_tasks < required_successes {
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count: successes.len(),
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
return Err(first_error.unwrap_or_else(|| {
Error::other(format!(
"Insufficient successful results: got {}, needed {}",
@@ -144,6 +457,18 @@ impl AsyncBatchProcessor {
}
}
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count: successes.len(),
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
Err(first_error.unwrap_or_else(|| {
Error::other(format!(
"Insufficient successful results: got {}, needed {}",
@@ -164,9 +489,9 @@ pub struct GlobalBatchProcessors {
impl GlobalBatchProcessors {
pub fn new() -> Self {
Self {
read_processor: AsyncBatchProcessor::new(16), // Higher concurrency for reads
write_processor: AsyncBatchProcessor::new(8), // Lower concurrency for writes
metadata_processor: AsyncBatchProcessor::new(12), // Medium concurrency for metadata
read_processor: AsyncBatchProcessor::new_with_operation(16, BATCH_PROCESSOR_OPERATION_READ),
write_processor: AsyncBatchProcessor::new_with_operation(8, BATCH_PROCESSOR_OPERATION_WRITE),
metadata_processor: AsyncBatchProcessor::new_with_operation(12, BATCH_PROCESSOR_OPERATION_METADATA),
}
}
@@ -321,4 +646,208 @@ mod tests {
assert!(err.to_string().contains("first failure"));
assert!(started.elapsed() < Duration::from_millis(120));
}
#[tokio::test]
async fn test_batch_processor_observe_only_keeps_configured_concurrency() {
let processor = AsyncBatchProcessor::new(2);
let tasks: Vec<_> = (0..4)
.map(|i| async move {
tokio::time::sleep(Duration::from_millis(5)).await;
Ok::<i32, Error>(i)
})
.collect();
let results = processor.execute_batch(tasks).await;
assert_eq!(results.len(), 4);
assert_eq!(processor.max_concurrent, 2);
}
#[test]
fn batch_processor_adaptive_mode_defaults_to_off_and_parses_supported_values() {
assert_eq!(parse_batch_processor_adaptive_mode(""), BatchProcessorAdaptiveMode::Off);
assert_eq!(parse_batch_processor_adaptive_mode("off"), BatchProcessorAdaptiveMode::Off);
assert_eq!(parse_batch_processor_adaptive_mode("observe"), BatchProcessorAdaptiveMode::Observe);
assert_eq!(parse_batch_processor_adaptive_mode("on"), BatchProcessorAdaptiveMode::On);
assert_eq!(parse_batch_processor_adaptive_mode("unknown"), BatchProcessorAdaptiveMode::Off);
assert!(!BatchProcessorAdaptiveMode::Off.should_observe());
assert!(BatchProcessorAdaptiveMode::Observe.should_observe());
assert!(BatchProcessorAdaptiveMode::On.should_observe());
assert!(!BatchProcessorAdaptiveMode::Observe.should_apply());
assert!(BatchProcessorAdaptiveMode::On.should_apply());
}
#[test]
fn batch_processor_adaptive_mode_env_gate_is_parsed_from_env() {
temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, None::<&str>, || {
assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Off);
});
temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, Some("observe"), || {
assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Observe);
});
}
#[test]
fn batch_processor_observation_gate_keeps_default_off_stable() {
let processor = AsyncBatchProcessor::new(8);
let suggestion = processor.observe_batch_with_mode(
BatchProcessorAdaptiveMode::Off,
processor.execution_concurrency(BatchProcessorAdaptiveMode::Off),
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_STABLE);
assert_eq!(suggestion.concurrency, 8);
}
#[test]
fn batch_processor_observation_gate_observe_records_suggestion() {
let processor = AsyncBatchProcessor::new(8);
let suggestion = processor.observe_batch_with_mode(
BatchProcessorAdaptiveMode::Observe,
processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe),
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert!(suggestion.concurrency > 8);
assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe), 8);
}
#[test]
fn batch_processor_adaptive_on_applies_last_suggestion_to_next_batch() {
let processor = AsyncBatchProcessor::new(8);
let suggestion = processor.observe_batch_with_mode(
BatchProcessorAdaptiveMode::On,
processor.execution_concurrency(BatchProcessorAdaptiveMode::On),
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert!(suggestion.concurrency > 8);
assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::On), suggestion.concurrency);
}
#[test]
fn batch_processor_suggestion_increases_for_fast_successful_batches() {
let suggestion = calculate_batch_concurrency_suggestion(
8,
8,
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert!(suggestion.concurrency > 8);
}
#[test]
fn batch_processor_suggestion_decreases_for_timeout_batches() {
let suggestion = calculate_batch_concurrency_suggestion(
8,
8,
BatchObservation {
batch_size: 16,
success_count: 14,
error_count: 0,
timeout_count: 2,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_DEGRADING);
assert!(suggestion.concurrency < 8);
assert!(suggestion.concurrency >= 1);
}
#[test]
fn batch_processor_suggestion_clamps_to_batch_size() {
let suggestion = calculate_batch_concurrency_suggestion(
64,
64,
BatchObservation {
batch_size: 65,
success_count: 65,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert_eq!(suggestion.concurrency, 65);
}
#[test]
fn batch_processor_suggestion_cooldown_prevents_rapid_oscillation() {
let mut state = BatchObservationState::new(8);
let now = Instant::now();
let observation = BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
};
let first = state.suggest(8, 8, observation, now);
let second = state.suggest(8, 8, observation, now + Duration::from_secs(1));
assert_eq!(first.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert_eq!(second.reason, BATCH_SUGGESTION_REASON_COOLDOWN);
assert_eq!(second.concurrency, first.concurrency);
}
#[test]
fn batch_processor_suggestion_growth_is_capped_relative_to_configured_concurrency() {
const CONFIGURED: usize = 8;
let mut state = BatchObservationState::new(CONFIGURED);
let mut current = CONFIGURED;
let mut now = Instant::now();
let observation = BatchObservation {
batch_size: 1024,
success_count: 1024,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
};
for _ in 0..64 {
current = state.suggest(current, CONFIGURED, observation, now).concurrency;
now += BATCH_OBSERVATION_COOLDOWN;
}
assert_eq!(current, CONFIGURED * BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR);
}
}
crates/ecstore/src/set_disk/mod.rs
+34 -8
View File
@@ -270,10 +270,19 @@ impl AsyncRead for SetDiskLockGuardedReader {
}
}
fn attach_set_disk_read_lock_guard(mut reader: GetObjectReader, read_lock_guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
if let Some(guard) = read_lock_guard
&& reader.buffered_body.is_none()
{
fn finish_set_disk_read_lock(
mut reader: GetObjectReader,
read_lock_guard: Option<ObjectLockDiagGuard>,
lock_optimization_enabled: bool,
bucket: &str,
object: &str,
) -> GetObjectReader {
if lock_optimization_enabled || reader.buffered_body.is_some() {
release_materialized_read_lock(bucket, object, read_lock_guard);
return reader;
}
if let Some(guard) = read_lock_guard {
reader.stream = Box::new(SetDiskLockGuardedReader {
inner: reader.stream,
guard: Some(guard),
@@ -2314,7 +2323,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
opts.part_number = Some(1);
}
let gr = get_transitioned_object_reader(bucket, object, &range, &h, &object_info, &opts).await?;
return Ok(attach_set_disk_read_lock_guard(gr, read_lock_guard.take()));
return Ok(finish_set_disk_read_lock(
gr,
read_lock_guard.take(),
lock_optimization_enabled,
bucket,
object,
));
}
if is_get_small_object_direct_memory_eligible(&range, &object_info, &fi, opts) {
@@ -2418,7 +2433,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
);
record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket);
let (reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?;
return Ok(attach_set_disk_read_lock_guard(reader, read_lock_guard.take()));
return Ok(finish_set_disk_read_lock(
reader,
read_lock_guard.take(),
lock_optimization_enabled,
bucket,
object,
));
}
read::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => {
record_get_codec_streaming_gate_decision(
@@ -2454,8 +2475,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
let set_index = self.set_index;
let pool_index = self.pool_index;
let skip_verify = opts.skip_verify_bitrot;
// Move the read-lock guard into the task so it lives for the duration of the read.
// Fully materialized paths release it before returning; streaming paths keep it.
if lock_optimization_enabled {
release_materialized_read_lock(&bucket, &object, read_lock_guard.take());
debug!(bucket, object, "Lock optimization: released read lock before streaming read");
}
// When lock optimization is disabled, keep the read-lock guard in the
// task so it lives for the duration of the streaming read.
tokio::spawn(async move {
let _guard = read_lock_guard;
let mut writer = wd;
crates/ecstore/src/set_disk/read.rs
+65 -14
View File
@@ -595,14 +595,64 @@ fn resolve_read_part_from_responses(
Err(DiskError::ErasureReadQuorum)
}
fn shard_read_cost_for_disk(disk: Option<&DiskStore>) -> ShardReadCost {
fn shard_read_costs_for_disks(disks: &[Option<DiskStore>]) -> Vec<ShardReadCost> {
let local_endpoint_hosts = local_endpoint_hosts_for_shard_costs();
disks
.iter()
.map(|disk| shard_read_cost_for_disk(disk.as_ref(), local_endpoint_hosts))
.collect()
}
fn shard_read_cost_for_disk(disk: Option<&DiskStore>, local_endpoint_hosts: &[String]) -> ShardReadCost {
match disk {
Some(disk) if disk.is_local() => ShardReadCost::Local,
Some(_) => ShardReadCost::Remote,
Some(disk) => shard_read_cost_for_endpoint(false, &disk.host_name(), local_endpoint_hosts),
None => ShardReadCost::Unknown,
}
}
fn shard_read_cost_for_endpoint(is_local: bool, host_name: &str, local_endpoint_hosts: &[String]) -> ShardReadCost {
if is_local {
return ShardReadCost::Local;
}
if !host_name.is_empty() && local_endpoint_hosts.iter().any(|host| host == host_name) {
return ShardReadCost::SameNode;
}
ShardReadCost::Remote
}
fn local_endpoint_hosts_for_shard_costs() -> &'static [String] {
// Endpoint pools are immutable after startup, so build the host list once
// instead of walking every pool on each read. Do not cache the empty
// pre-startup answer: only memoize once the pools are published.
static LOCAL_ENDPOINT_HOSTS: std::sync::OnceLock<Vec<String>> = std::sync::OnceLock::new();
if let Some(hosts) = LOCAL_ENDPOINT_HOSTS.get() {
return hosts;
}
let Some(endpoint_pools) = runtime_sources::endpoint_pools() else {
return &[];
};
let mut hosts = Vec::new();
for pool in endpoint_pools.as_ref() {
for endpoint in pool.endpoints.as_ref() {
if !endpoint.is_local {
continue;
}
let host = endpoint.host_port();
if !host.is_empty() && !hosts.contains(&host) {
hosts.push(host);
}
}
}
LOCAL_ENDPOINT_HOSTS.get_or_init(|| hosts)
}
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
struct ReadRepairHealCacheKey {
bucket: String,
@@ -2813,12 +2863,7 @@ impl SetDisks {
let use_mmap_read = object_mmap_read_enabled();
let reader_setup_stage_start = Instant::now();
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| {
disks
.iter()
.map(|disk| shard_read_cost_for_disk(disk.as_ref()))
.collect::<Vec<_>>()
});
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(&disks));
let reader_setup = create_bitrot_readers_until_quorum_with_preference(
&files,
&disks,
@@ -3244,12 +3289,7 @@ impl SetDisks {
bitrot_reader_init_stage: GET_STAGE_READER_TASK_BITROT_READER_INIT,
});
let reader_setup_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| {
disks
.iter()
.map(|disk| shard_read_cost_for_disk(disk.as_ref()))
.collect::<Vec<_>>()
});
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(disks));
let reader_setup = create_bitrot_readers_until_quorum_with_preference(
files,
disks,
@@ -3835,6 +3875,17 @@ mod tests {
const CODEC_STREAMING_TEST_BUCKET: &str = "bucket";
const CODEC_STREAMING_TEST_OBJECT: &str = "object";
#[test]
fn shard_read_cost_for_endpoint_maps_topology_classes() {
let local_hosts = vec!["node-a:9000".to_string()];
assert_eq!(shard_read_cost_for_endpoint(true, "node-a:9000", &local_hosts), ShardReadCost::Local);
assert_eq!(shard_read_cost_for_endpoint(false, "node-a:9000", &local_hosts), ShardReadCost::SameNode);
assert_eq!(shard_read_cost_for_endpoint(false, "node-b:9000", &local_hosts), ShardReadCost::Remote);
assert_eq!(shard_read_cost_for_endpoint(false, "", &local_hosts), ShardReadCost::Remote);
assert_eq!(shard_read_cost_for_disk(None, &local_hosts), ShardReadCost::Unknown);
}
fn metadata_fanout_test_fileinfo(object: &str) -> FileInfo {
let mut fi = FileInfo::new(object, 2, 2);
fi.volume = "bucket".to_string();
crates/ecstore/src/store/object.rs
+5 -8
View File
@@ -15,7 +15,7 @@
use super::*;
use crate::set_disk::{
get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
is_object_lock_diag_enabled,
is_lock_optimization_enabled, is_object_lock_diag_enabled,
};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use rustfs_io_metrics::{
@@ -442,7 +442,7 @@ impl ECStore {
}
fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
if reader.buffered_body.is_some() {
if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
return reader;
}
@@ -1978,7 +1978,7 @@ mod tests {
#[tokio::test]
#[serial_test::serial]
async fn reader_lock_is_held_for_stream_when_optimization_is_enabled() {
async fn reader_lock_is_not_held_for_stream_when_optimization_is_enabled() {
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
@@ -2004,13 +2004,10 @@ mod tests {
let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
lock.get_write_lock(key.clone(), "writer", Duration::from_millis(20))
.await
.expect_err("streaming reader should hold the read lock");
drop(reader);
lock.get_write_lock(key, "writer", Duration::from_secs(1))
.await
.expect("dropping the reader should release the read lock");
.expect("lock optimization should release the read lock before returning the stream");
drop(reader);
})
.await;
}
crates/io-metrics/src/internode_metrics.rs
+94 -2
View File
@@ -33,6 +33,7 @@ const BACKEND_LABEL: &str = "backend";
const CLASSIFICATION_LABEL: &str = "classification";
const STAGE_LABEL: &str = "stage";
const DOMINANT_ERROR_LABEL: &str = "dominant_error";
const HTTP_VERSION_LABEL: &str = "http_version";
const INTERNODE_OPERATION_SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_sent_bytes_total";
const INTERNODE_OPERATION_RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_recv_bytes_total";
const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_outgoing_total";
@@ -42,6 +43,10 @@ const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_o
const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total";
const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total";
const INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL: &str = "rustfs_system_network_internode_operation_retry_successes_total";
const INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL: &str = "rustfs_system_network_internode_operation_http_versions_total";
const INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL: &str = "rustfs_system_network_internode_operation_stall_timeouts_total";
const INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL: &str =
"rustfs_system_network_internode_operation_write_shutdown_errors_total";
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -52,6 +57,7 @@ pub struct InternodeOperationMetricDescriptor {
const OPERATION_BACKEND_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL];
const OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL];
const OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL];
const QUORUM_FAILURE_LABELS: &[&str] = &[STAGE_LABEL, DOMINANT_ERROR_LABEL];
pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &[
@@ -91,6 +97,18 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
name: INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL,
labels: OPERATION_BACKEND_CLASSIFICATION_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL,
labels: OPERATION_BACKEND_HTTP_VERSION_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL,
labels: OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL,
labels: OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
labels: QUORUM_FAILURE_LABELS,
@@ -107,6 +125,9 @@ pub struct InternodeMetricsSnapshot {
pub dial_errors_total: u64,
pub dial_avg_time_nanos: u64,
pub last_dial_unix_millis: u64,
pub operation_http_versions_total: u64,
pub operation_stall_timeouts_total: u64,
pub operation_write_shutdown_errors_total: u64,
}
#[derive(Debug, Default)]
@@ -120,6 +141,9 @@ pub struct InternodeMetrics {
dial_total_time_nanos: AtomicU64,
dial_samples_total: AtomicU64,
last_dial_unix_millis: AtomicU64,
operation_http_versions_total: AtomicU64,
operation_stall_timeouts_total: AtomicU64,
operation_write_shutdown_errors_total: AtomicU64,
}
impl InternodeMetrics {
@@ -264,6 +288,33 @@ impl InternodeMetrics {
.increment(1);
}
pub fn record_http_version_for_operation_and_backend(
&self,
operation: &'static str,
backend: &'static str,
http_version: &'static str,
) {
self.operation_http_versions_total.fetch_add(1, Ordering::Relaxed);
counter!(
INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL,
OPERATION_LABEL => operation,
BACKEND_LABEL => backend,
HTTP_VERSION_LABEL => http_version
)
.increment(1);
}
pub fn record_stall_timeout_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
self.operation_stall_timeouts_total.fetch_add(1, Ordering::Relaxed);
counter!(INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1);
}
pub fn record_write_shutdown_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
self.operation_write_shutdown_errors_total.fetch_add(1, Ordering::Relaxed);
counter!(INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend)
.increment(1);
}
pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) {
counter!(
ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
@@ -307,6 +358,9 @@ impl InternodeMetrics {
dial_errors_total: self.dial_errors_total.load(Ordering::Relaxed),
dial_avg_time_nanos,
last_dial_unix_millis: self.last_dial_unix_millis.load(Ordering::Relaxed),
operation_http_versions_total: self.operation_http_versions_total.load(Ordering::Relaxed),
operation_stall_timeouts_total: self.operation_stall_timeouts_total.load(Ordering::Relaxed),
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
}
}
@@ -321,6 +375,9 @@ impl InternodeMetrics {
self.dial_total_time_nanos.store(0, Ordering::Relaxed);
self.dial_samples_total.store(0, Ordering::Relaxed);
self.last_dial_unix_millis.store(0, Ordering::Relaxed);
self.operation_http_versions_total.store(0, Ordering::Relaxed);
self.operation_stall_timeouts_total.store(0, Ordering::Relaxed);
self.operation_write_shutdown_errors_total.store(0, Ordering::Relaxed);
}
}
@@ -393,14 +450,21 @@ mod tests {
#[test]
fn operation_metric_descriptors_include_backend_and_operation_labels() {
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 10);
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 13);
for metric in &INTERNODE_OPERATION_METRICS[..6] {
assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]);
}
for metric in &INTERNODE_OPERATION_METRICS[6..9] {
assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]);
}
assert_eq!(INTERNODE_OPERATION_METRICS[9].labels, &[STAGE_LABEL, DOMINANT_ERROR_LABEL]);
assert_eq!(
INTERNODE_OPERATION_METRICS[9].labels,
&[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL]
);
for metric in &INTERNODE_OPERATION_METRICS[10..12] {
assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]);
}
assert_eq!(INTERNODE_OPERATION_METRICS[12].labels, &[STAGE_LABEL, DOMINANT_ERROR_LABEL]);
}
#[test]
@@ -433,6 +497,18 @@ mod tests {
);
assert_eq!(
INTERNODE_OPERATION_METRICS[9].name,
"rustfs_system_network_internode_operation_http_versions_total"
);
assert_eq!(
INTERNODE_OPERATION_METRICS[10].name,
"rustfs_system_network_internode_operation_stall_timeouts_total"
);
assert_eq!(
INTERNODE_OPERATION_METRICS[11].name,
"rustfs_system_network_internode_operation_write_shutdown_errors_total"
);
assert_eq!(
INTERNODE_OPERATION_METRICS[12].name,
"rustfs_system_storage_erasure_write_quorum_failures_total"
);
}
@@ -456,6 +532,19 @@ mod tests {
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
"connection_reset",
);
metrics.record_http_version_for_operation_and_backend(
INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
"http/1.1",
);
metrics.record_stall_timeout_for_operation_and_backend(
INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
);
metrics.record_write_shutdown_error_for_operation_and_backend(
INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
);
metrics.record_erasure_write_quorum_failure("write", "connection_reset");
let snapshot = metrics.snapshot();
@@ -463,5 +552,8 @@ mod tests {
assert_eq!(snapshot.recv_bytes_total, 0);
assert_eq!(snapshot.outgoing_requests_total, 0);
assert_eq!(snapshot.incoming_requests_total, 0);
assert_eq!(snapshot.operation_http_versions_total, 1);
assert_eq!(snapshot.operation_stall_timeouts_total, 1);
assert_eq!(snapshot.operation_write_shutdown_errors_total, 1);
}
}
crates/io-metrics/src/lib.rs
+120
View File
@@ -1098,6 +1098,19 @@ pub fn record_get_object_shard_locality_policy_disabled(path: &'static str) {
}
counter!("rustfs_io_get_object_shard_locality_policy_disabled_total", "path" => path).increment(1);
}
/// Record observe-only shard-locality potential while scheduling remains disabled.
#[inline(always)]
pub fn record_get_object_shard_locality_observe_only(path: &'static str, remote_scheduled: usize, remote_avoid_potential: usize) {
if !get_stage_metrics_enabled() {
return;
}
histogram!("rustfs_io_get_object_shard_remote_scheduled_observe_only", "path" => path)
.record(shard_read_fanout_to_f64(remote_scheduled));
histogram!("rustfs_io_get_object_shard_remote_avoid_potential", "path" => path)
.record(shard_read_fanout_to_f64(remote_avoid_potential));
}
/// Record per-stripe shard-read fanout shape for GetObject read-path attribution.
#[inline(always)]
pub fn record_get_object_shard_read_fanout(
@@ -1116,6 +1129,70 @@ pub fn record_get_object_shard_read_fanout(
histogram!("rustfs_io_get_object_shard_read_failed", "path" => path).record(shard_read_fanout_to_f64(failed));
}
fn batch_processor_count_to_f64(value: usize) -> f64 {
u64::try_from(value).unwrap_or(u64::MAX) as f64
}
fn batch_processor_count_to_u64(value: usize) -> u64 {
u64::try_from(value).unwrap_or(u64::MAX)
}
/// Observe-only batch processor shape and adaptive-concurrency advice.
#[derive(Clone, Copy, Debug)]
pub struct BatchProcessorObservation {
pub operation: &'static str,
pub batch_size: usize,
pub configured_concurrency: usize,
pub max_queue_wait_secs: f64,
pub execution_latency_secs: f64,
pub successes: usize,
pub errors: usize,
pub timeouts: usize,
pub suggested_concurrency: usize,
pub suggestion_reason: &'static str,
}
/// Record observe-only batch processor shape and adaptive-concurrency advice.
#[inline(always)]
pub fn record_batch_processor_observation(observation: BatchProcessorObservation) {
if !get_stage_metrics_enabled() {
return;
}
histogram!("rustfs_ecstore_batch_processor_batch_size", "operation" => observation.operation)
.record(batch_processor_count_to_f64(observation.batch_size));
histogram!("rustfs_ecstore_batch_processor_configured_concurrency", "operation" => observation.operation)
.record(batch_processor_count_to_f64(observation.configured_concurrency));
histogram!("rustfs_ecstore_batch_processor_queue_wait_seconds", "operation" => observation.operation)
.record(observation.max_queue_wait_secs);
histogram!("rustfs_ecstore_batch_processor_execution_latency_seconds", "operation" => observation.operation)
.record(observation.execution_latency_secs);
counter!(
"rustfs_ecstore_batch_processor_results_total",
"operation" => observation.operation,
"outcome" => "success"
)
.increment(batch_processor_count_to_u64(observation.successes));
counter!(
"rustfs_ecstore_batch_processor_results_total",
"operation" => observation.operation,
"outcome" => "error"
)
.increment(batch_processor_count_to_u64(observation.errors));
counter!(
"rustfs_ecstore_batch_processor_results_total",
"operation" => observation.operation,
"outcome" => "timeout"
)
.increment(batch_processor_count_to_u64(observation.timeouts));
histogram!(
"rustfs_ecstore_batch_processor_suggested_concurrency",
"operation" => observation.operation,
"reason" => observation.suggestion_reason
)
.record(batch_processor_count_to_f64(observation.suggested_concurrency));
}
/// Record the bitrot reader setup scheduling strategy selected for a GET read.
#[inline(always)]
pub fn record_get_object_reader_setup_strategy(strategy: &'static str, mode: &'static str) {
@@ -2033,6 +2110,26 @@ mod tests {
record_bytespool_return("xlarge", "dropped");
}
#[test]
fn test_record_batch_processor_observation() {
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
set_get_stage_metrics_enabled(true);
record_batch_processor_observation(BatchProcessorObservation {
operation: "read",
batch_size: 16,
configured_concurrency: 8,
max_queue_wait_secs: 0.001,
execution_latency_secs: 0.025,
successes: 15,
errors: 1,
timeouts: 0,
suggested_concurrency: 10,
suggestion_reason: "improving",
});
assert!(get_stage_metrics_enabled());
set_get_stage_metrics_enabled(false);
}
#[test]
fn test_record_zero_copy_write() {
record_zero_copy_write(1024, 10.5);
@@ -2094,6 +2191,7 @@ mod tests {
record_get_object_pipeline_failure_for_path("codec_streaming", "decode", "read_quorum");
record_get_object_shard_read_observation("codec_streaming", 0, "data", "local", "success", "none", 1024, 0.004, 0.001);
record_get_object_shard_read_cost_summary("codec_streaming", 3, 1, 2, 0, 4, 4, 4, true);
record_get_object_shard_locality_observe_only("codec_streaming", 2, 1);
record_get_object_reader_setup_strategy("data_blocks_first", "read_quorum");
record_get_object_reader_setup_strategy_by_size(
"codec_streaming",
@@ -2115,10 +2213,32 @@ mod tests {
0,
2,
);
record_batch_processor_observation(BatchProcessorObservation {
operation: "read",
batch_size: 16,
configured_concurrency: 8,
max_queue_wait_secs: 0.001,
execution_latency_secs: 0.025,
successes: 15,
errors: 1,
timeouts: 0,
suggested_concurrency: 10,
suggestion_reason: "improving",
});
assert!(0.005_f64.is_sign_positive());
}
#[test]
fn test_get_object_shard_locality_observe_only_metrics_smoke() {
let remote_scheduled = 2;
let remote_avoid_potential = 1;
record_get_object_shard_locality_observe_only("codec_streaming", remote_scheduled, remote_avoid_potential);
assert!(remote_scheduled >= remote_avoid_potential);
}
#[test]
fn test_record_get_object_fill_metrics() {
record_get_object_fill_queued("codec_streaming", "single_inflight", 1);
crates/protocols/src/sftp/server.rs
+39 -19
View File
@@ -36,8 +36,8 @@ use super::wedge_watchdog;
use crate::common::client::s3::StorageBackend;
use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
use russh::keys::{self, PrivateKey, PublicKeyBase64};
use russh::server::{Auth, Msg, Session};
use russh::{Channel, ChannelId, MethodKind, MethodSet, Pty, Sig};
use russh::server::{Auth, ChannelOpenHandle, Msg, Session};
use russh::{Channel, ChannelId, ChannelOpenFailure, MethodKind, MethodSet, Pty, Sig};
use rustfs_config::{
DEFAULT_SFTP_HOST_KEY_RELOAD_ENABLE, DEFAULT_SFTP_HOST_KEY_RELOAD_INTERVAL, ENV_SFTP_HOST_KEY_RELOAD_ENABLE,
ENV_SFTP_HOST_KEY_RELOAD_INTERVAL,
@@ -1135,15 +1135,19 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
}
}
#[tracing::instrument(level = "debug", skip(self, channel, _session), fields(peer = %self.peer_addr))]
#[tracing::instrument(level = "debug", skip(self, channel, reply, _session), fields(peer = %self.peer_addr))]
fn channel_open_session(
&mut self,
channel: Channel<Msg>,
reply: ChannelOpenHandle,
_session: &mut Session,
) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
let id = channel.id();
self.channels.insert(id, channel);
async { Ok(true) }
async move {
reply.accept().await;
Ok(())
}
}
#[tracing::instrument(level = "debug", skip(self, _session), fields(peer = %self.peer_addr, channel = ?channel))]
@@ -1404,13 +1408,13 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
async { Ok(false) }
}
// Channel-open rejections. russh 0.60 defaults all of these to
// Ok(false), but we override them explicitly with a warn log so
// Channel-open rejections. russh defaults to rejecting dropped channel-open
// handles, but we override them explicitly with a warn log so
// (a) probe attempts are visible in operator logs and
// (b) a future russh default flip cannot silently allow these
// channel types.
#[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
#[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
fn channel_open_direct_tcpip(
&mut self,
_channel: Channel<Msg>,
@@ -1418,12 +1422,16 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
port_to_connect: u32,
_originator_address: &str,
_originator_port: u32,
reply: ChannelOpenHandle,
_session: &mut Session,
) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
async { Ok(false) }
) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
async move {
reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
Ok(())
}
}
#[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
#[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
fn channel_open_forwarded_tcpip(
&mut self,
_channel: Channel<Msg>,
@@ -1431,30 +1439,42 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
port_to_connect: u32,
_originator_address: &str,
_originator_port: u32,
reply: ChannelOpenHandle,
_session: &mut Session,
) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
async { Ok(false) }
) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
async move {
reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
Ok(())
}
}
#[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr))]
#[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr))]
fn channel_open_x11(
&mut self,
_channel: Channel<Msg>,
_originator_address: &str,
_originator_port: u32,
reply: ChannelOpenHandle,
_session: &mut Session,
) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
async { Ok(false) }
) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
async move {
reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
Ok(())
}
}
#[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, socket = %socket_path))]
#[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, socket = %socket_path))]
fn channel_open_direct_streamlocal(
&mut self,
_channel: Channel<Msg>,
socket_path: &str,
reply: ChannelOpenHandle,
_session: &mut Session,
) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
async { Ok(false) }
) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
async move {
reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
Ok(())
}
}
}
crates/protos/src/generated/proto_gen/node_service.rs
+67
View File
@@ -531,6 +531,26 @@ pub struct ReadVersionResponse {
pub file_info_bin: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BatchReadVersionRequest {
#[prost(string, tag = "1")]
pub disk: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub batch_read_version_req: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "3")]
pub batch_read_version_req_bin: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BatchReadVersionResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(string, repeated, tag = "2")]
pub batch_read_version_resps: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
#[prost(message, optional, tag = "3")]
pub error: ::core::option::Option<Error>,
#[prost(bytes = "bytes", repeated, tag = "4")]
pub batch_read_version_resps_bin: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ReadXlRequest {
#[prost(string, tag = "1")]
pub disk: ::prost::alloc::string::String,
@@ -1689,6 +1709,21 @@ pub mod node_service_client {
.insert(GrpcMethod::new("node_service.NodeService", "ReadVersion"));
self.inner.unary(req, path, codec).await
}
pub async fn batch_read_version(
&mut self,
request: impl tonic::IntoRequest<super::BatchReadVersionRequest>,
) -> std::result::Result<tonic::Response<super::BatchReadVersionResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/BatchReadVersion");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "BatchReadVersion"));
self.inner.unary(req, path, codec).await
}
pub async fn read_xl(
&mut self,
request: impl tonic::IntoRequest<super::ReadXlRequest>,
@@ -2610,6 +2645,10 @@ pub mod node_service_server {
&self,
request: tonic::Request<super::ReadVersionRequest>,
) -> std::result::Result<tonic::Response<super::ReadVersionResponse>, tonic::Status>;
async fn batch_read_version(
&self,
request: tonic::Request<super::BatchReadVersionRequest>,
) -> std::result::Result<tonic::Response<super::BatchReadVersionResponse>, tonic::Status>;
async fn read_xl(
&self,
request: tonic::Request<super::ReadXlRequest>,
@@ -3706,6 +3745,34 @@ pub mod node_service_server {
};
Box::pin(fut)
}
"/node_service.NodeService/BatchReadVersion" => {
#[allow(non_camel_case_types)]
struct BatchReadVersionSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::UnaryService<super::BatchReadVersionRequest> for BatchReadVersionSvc<T> {
type Response = super::BatchReadVersionResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::BatchReadVersionRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::batch_read_version(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = BatchReadVersionSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/ReadXL" => {
#[allow(non_camel_case_types)]
struct ReadXLSvc<T: NodeService>(pub Arc<T>);
crates/protos/src/node.proto
+14
View File
@@ -371,6 +371,19 @@ message ReadVersionResponse {
bytes file_info_bin = 4;
}
message BatchReadVersionRequest {
string disk = 1;
string batch_read_version_req = 2;
bytes batch_read_version_req_bin = 3;
}
message BatchReadVersionResponse {
bool success = 1;
repeated string batch_read_version_resps = 2;
optional Error error = 3;
repeated bytes batch_read_version_resps_bin = 4;
}
message ReadXLRequest {
string disk = 1;
string volume = 2;
@@ -868,6 +881,7 @@ service NodeService {
rpc ReadMetadata(ReadMetadataRequest) returns (ReadMetadataResponse) {};
rpc WriteMetadata(WriteMetadataRequest) returns (WriteMetadataResponse) {};
rpc ReadVersion(ReadVersionRequest) returns (ReadVersionResponse) {};
rpc BatchReadVersion(BatchReadVersionRequest) returns (BatchReadVersionResponse) {};
rpc ReadXL(ReadXLRequest) returns (ReadXLResponse) {};
rpc DeleteVersion(DeleteVersionRequest) returns (DeleteVersionResponse) {};
rpc DeleteVersions(DeleteVersionsRequest) returns (DeleteVersionsResponse) {};
crates/rio/src/http_reader.rs
+524 -30
View File
@@ -15,14 +15,14 @@
use crate::{EtagResolvable, HashReaderDetector, HashReaderMut};
use bytes::{Bytes, BytesMut};
use futures::{Stream, TryStreamExt as _};
use http::HeaderMap;
use http::{HeaderMap, Version};
use pin_project_lite::pin_project;
use reqwest::{Certificate, Client, Identity, Method, RequestBuilder};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
};
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
use rustfs_utils::get_env_opt_str;
use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_opt_u64, get_env_opt_usize};
use rustls_pki_types::pem::PemObject;
use std::io::IoSlice;
use std::io::{self, Error};
@@ -31,7 +31,7 @@ use std::ops::Not as _;
use std::pin::Pin;
use std::sync::LazyLock;
use std::task::{Context, Poll};
use std::time::Duration;
use std::time::{Duration, Instant};
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
use tokio::sync::{Mutex, mpsc};
use tokio::time::{self, Sleep};
@@ -42,6 +42,11 @@ use tracing::{error, warn};
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
const HTTP_VERSION_09_LABEL: &str = "http/0.9";
const HTTP_VERSION_10_LABEL: &str = "http/1.0";
const HTTP_VERSION_11_LABEL: &str = "http/1.1";
const HTTP_VERSION_2_LABEL: &str = "h2";
const HTTP_VERSION_UNKNOWN_LABEL: &str = "unknown";
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub enum InternodeHttpErrorKind {
@@ -213,18 +218,192 @@ fn add_root_certificates_from_der(builder: reqwest::ClientBuilder, certs_der: &[
struct CachedClients {
generation: u64,
client: Client,
local_client: Client,
no_proxy_client: Client,
}
static CLIENT_CACHE: LazyLock<Mutex<Option<CachedClients>>> = LazyLock::new(|| Mutex::new(None));
async fn build_http_client(disable_proxy: bool, outbound_tls: &rustfs_tls_runtime::GlobalPublishedOutboundTlsState) -> Client {
const INTERNODE_HTTP_PROFILE_LEGACY: &str = "legacy";
const INTERNODE_HTTP_PROFILE_BALANCED: &str = "balanced";
const INTERNODE_HTTP_PROFILE_THROUGHPUT: &str = "throughput";
const INTERNODE_HTTP_PROXY_LEGACY: &str = "legacy";
const INTERNODE_HTTP_PROXY_OFF: &str = "off";
const INTERNODE_HTTP_PROXY_SYSTEM: &str = "system";
const INTERNODE_HTTP2_WINDOW_MIN: u32 = 65_535;
const INTERNODE_HTTP2_WINDOW_MAX: u32 = 64 * 1024 * 1024;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum InternodeHttpTuningProfile {
Legacy,
Balanced,
Throughput,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum InternodeHttpProxyMode {
Legacy,
NoProxy,
System,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct InternodeHttpClientTuning {
profile: InternodeHttpTuningProfile,
pool_max_idle_per_host: Option<usize>,
pool_idle_timeout_secs: Option<u64>,
http2_initial_stream_window_size: Option<u32>,
http2_initial_connection_window_size: Option<u32>,
http2_adaptive_window: bool,
proxy_mode: InternodeHttpProxyMode,
}
fn internode_http_client_tuning() -> InternodeHttpClientTuning {
// The tuning env vars cannot change at runtime; parse them once instead of
// re-reading the environment on every internode request.
static TUNING: LazyLock<InternodeHttpClientTuning> = LazyLock::new(InternodeHttpClientTuning::from_env);
*TUNING
}
impl InternodeHttpClientTuning {
fn from_env() -> Self {
let profile =
parse_internode_http_tuning_profile(get_env_opt_str(rustfs_config::ENV_INTERNODE_HTTP_TUNING_PROFILE).as_deref());
Self::from_values(
profile,
get_env_opt_usize(rustfs_config::ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST),
get_env_opt_u64(rustfs_config::ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS),
get_env_opt_u64(rustfs_config::ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE),
get_env_opt_u64(rustfs_config::ENV_INTERNODE_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE),
get_env_bool(
rustfs_config::ENV_INTERNODE_HTTP2_ADAPTIVE_WINDOW,
profile.default_http2_adaptive_window(),
),
get_env_opt_str(rustfs_config::ENV_INTERNODE_HTTP_PROXY).as_deref(),
)
}
fn from_values(
profile: InternodeHttpTuningProfile,
pool_max_idle_per_host: Option<usize>,
pool_idle_timeout_secs: Option<u64>,
stream_window_size: Option<u64>,
connection_window_size: Option<u64>,
http2_adaptive_window: bool,
proxy_mode: Option<&str>,
) -> Self {
Self {
profile,
pool_max_idle_per_host: pool_max_idle_per_host.or_else(|| profile.default_pool_max_idle_per_host()),
pool_idle_timeout_secs: pool_idle_timeout_secs.or_else(|| profile.default_pool_idle_timeout_secs()),
http2_initial_stream_window_size: clamp_http2_window(
stream_window_size.or_else(|| profile.default_stream_window_size()),
),
http2_initial_connection_window_size: clamp_http2_window(
connection_window_size.or_else(|| profile.default_connection_window_size()),
),
http2_adaptive_window,
proxy_mode: parse_internode_http_proxy_mode(proxy_mode, profile),
}
}
}
impl InternodeHttpTuningProfile {
fn default_pool_max_idle_per_host(self) -> Option<usize> {
match self {
Self::Legacy => None,
Self::Balanced => Some(64),
Self::Throughput => Some(256),
}
}
fn default_pool_idle_timeout_secs(self) -> Option<u64> {
match self {
Self::Legacy => None,
Self::Balanced => Some(120),
Self::Throughput => Some(300),
}
}
fn default_stream_window_size(self) -> Option<u64> {
match self {
Self::Legacy => None,
Self::Balanced => Some(1024 * 1024),
Self::Throughput => Some(4 * 1024 * 1024),
}
}
fn default_connection_window_size(self) -> Option<u64> {
match self {
Self::Legacy => None,
Self::Balanced => Some(4 * 1024 * 1024),
Self::Throughput => Some(16 * 1024 * 1024),
}
}
fn default_http2_adaptive_window(self) -> bool {
matches!(self, Self::Throughput)
}
}
fn parse_internode_http_tuning_profile(value: Option<&str>) -> InternodeHttpTuningProfile {
match value.map(|value| value.trim().to_ascii_lowercase()) {
Some(value) if value == INTERNODE_HTTP_PROFILE_LEGACY => InternodeHttpTuningProfile::Legacy,
Some(value) if value == INTERNODE_HTTP_PROFILE_BALANCED => InternodeHttpTuningProfile::Balanced,
Some(value) if value == INTERNODE_HTTP_PROFILE_THROUGHPUT => InternodeHttpTuningProfile::Throughput,
_ => InternodeHttpTuningProfile::Legacy,
}
}
fn parse_internode_http_proxy_mode(value: Option<&str>, profile: InternodeHttpTuningProfile) -> InternodeHttpProxyMode {
match value.map(|value| value.trim().to_ascii_lowercase()) {
Some(value) if value == INTERNODE_HTTP_PROXY_SYSTEM => InternodeHttpProxyMode::System,
Some(value) if value == INTERNODE_HTTP_PROXY_LEGACY => InternodeHttpProxyMode::Legacy,
Some(value) if matches!(value.as_str(), INTERNODE_HTTP_PROXY_OFF | "none" | "no_proxy" | "disabled") => {
InternodeHttpProxyMode::NoProxy
}
Some(_) | None if matches!(profile, InternodeHttpTuningProfile::Legacy) => InternodeHttpProxyMode::Legacy,
Some(_) | None => InternodeHttpProxyMode::NoProxy,
}
}
fn clamp_http2_window(value: Option<u64>) -> Option<u32> {
let value = value?;
let value = value.clamp(u64::from(INTERNODE_HTTP2_WINDOW_MIN), u64::from(INTERNODE_HTTP2_WINDOW_MAX));
u32::try_from(value).ok()
}
fn apply_http_client_tuning(mut builder: reqwest::ClientBuilder, tuning: InternodeHttpClientTuning) -> reqwest::ClientBuilder {
if let Some(pool_max_idle_per_host) = tuning.pool_max_idle_per_host {
builder = builder.pool_max_idle_per_host(pool_max_idle_per_host);
}
if let Some(pool_idle_timeout_secs) = tuning.pool_idle_timeout_secs {
builder = builder.pool_idle_timeout(std::time::Duration::from_secs(pool_idle_timeout_secs));
}
if tuning.http2_adaptive_window {
builder = builder.http2_adaptive_window(true);
} else {
if let Some(stream_window_size) = tuning.http2_initial_stream_window_size {
builder = builder.http2_initial_stream_window_size(stream_window_size);
}
if let Some(connection_window_size) = tuning.http2_initial_connection_window_size {
builder = builder.http2_initial_connection_window_size(connection_window_size);
}
}
builder
}
async fn build_http_client(
disable_proxy: bool,
tuning: InternodeHttpClientTuning,
outbound_tls: &rustfs_tls_runtime::GlobalPublishedOutboundTlsState,
) -> Client {
let mut builder = Client::builder()
.connect_timeout(std::time::Duration::from_secs(5))
.tcp_keepalive(std::time::Duration::from_secs(10))
.http2_keep_alive_interval(std::time::Duration::from_secs(5))
.http2_keep_alive_timeout(std::time::Duration::from_secs(3))
.http2_keep_alive_while_idle(true);
builder = apply_http_client_tuning(builder, tuning);
if disable_proxy {
builder = builder.no_proxy();
@@ -292,10 +471,19 @@ fn should_bypass_proxy_for_url(url: &str) -> bool {
host.eq_ignore_ascii_case("localhost") || host.parse::<IpAddr>().is_ok_and(|addr| addr.is_loopback())
}
fn should_disable_proxy_for_url(url: &str, tuning: InternodeHttpClientTuning) -> bool {
match tuning.proxy_mode {
InternodeHttpProxyMode::System => false,
InternodeHttpProxyMode::NoProxy => true,
InternodeHttpProxyMode::Legacy => should_bypass_proxy_for_url(url),
}
}
async fn get_http_client(url: &str) -> Client {
// Reuse HTTP connection pools while keeping loopback traffic away from
// system proxies so local RPC/tests do not leak to proxy listeners.
let disable_proxy = should_bypass_proxy_for_url(url);
let tuning = internode_http_client_tuning();
// Reuse HTTP connection pools while honoring the configured internode proxy
// policy. The legacy profile only bypasses loopback URLs to preserve defaults.
let disable_proxy = should_disable_proxy_for_url(url, tuning);
// Fast path: check generation first (cheap atomic read) to avoid cloning
// the full PEM + identity bytes when the TLS state hasn't changed.
@@ -305,7 +493,7 @@ async fn get_http_client(url: &str) -> Client {
if let Some(cached) = guard.as_ref() {
if cached.generation == generation {
return if disable_proxy {
cached.local_client.clone()
cached.no_proxy_client.clone()
} else {
cached.client.clone()
};
@@ -317,16 +505,16 @@ async fn get_http_client(url: &str) -> Client {
// Cache miss or stale generation — load full outbound TLS state.
let outbound_tls = crate::http_runtime_sources::outbound_tls_state().await;
let client = build_http_client(false, &outbound_tls).await;
let local_client = build_http_client(true, &outbound_tls).await;
let client = build_http_client(false, tuning, &outbound_tls).await;
let no_proxy_client = build_http_client(true, tuning, &outbound_tls).await;
let cached = CachedClients {
generation,
client,
local_client,
no_proxy_client,
};
let return_client = if disable_proxy {
cached.local_client.clone()
cached.no_proxy_client.clone()
} else {
cached.client.clone()
};
@@ -419,6 +607,8 @@ pin_project! {
internode_operation: Option<&'static str>,
stall_timeout: Option<Duration>,
stall_timer: Option<Pin<Box<Sleep>>>,
request_started: Instant,
duration_recorded: bool,
#[pin]
inner: StreamReader<Pin<Box<dyn Stream<Item=std::io::Result<Bytes>>+Send+Sync>>, Bytes>,
}
@@ -467,13 +657,17 @@ impl HttpReader {
request = request.body(body);
}
let request_started = Instant::now();
let resp = request.send().await.map_err(|e| {
record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed());
record_internode_error(track_internode_metrics, internode_operation);
record_internode_classified_error(track_internode_metrics, internode_operation, classify_reqwest_error(&e));
internode_reqwest_error(&method, &url, internode_operation, e)
})?;
record_internode_http_version(track_internode_metrics, internode_operation, http_version_metric_label(resp.version()));
if resp.status().is_success().not() {
record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed());
record_internode_error(track_internode_metrics, internode_operation);
record_internode_classified_error(track_internode_metrics, internode_operation, classify_http_status(resp.status()));
return Err(internode_status_error(&method, &url, internode_operation, resp.status()));
@@ -498,6 +692,8 @@ impl HttpReader {
internode_operation,
stall_timer: None,
stall_timeout,
request_started,
duration_recorded: false,
})
}
pub fn url(&self) -> &str {
@@ -521,6 +717,13 @@ impl AsyncRead for HttpReader {
let bytes_read = buf.filled().len().saturating_sub(filled_before);
if bytes_read > 0 {
record_internode_recv_bytes(*this.track_internode_metrics, *this.internode_operation, bytes_read);
} else {
record_internode_operation_duration_once(
*this.track_internode_metrics,
*this.internode_operation,
*this.request_started,
this.duration_recorded,
);
}
*this.stall_timer = None;
Poll::Ready(Ok(()))
@@ -531,6 +734,13 @@ impl AsyncRead for HttpReader {
};
let timer = this.stall_timer.get_or_insert_with(|| Box::pin(time::sleep(stall_timeout)));
if timer.as_mut().poll(cx).is_ready() {
record_internode_operation_duration_once(
*this.track_internode_metrics,
*this.internode_operation,
*this.request_started,
this.duration_recorded,
);
record_internode_stall_timeout(*this.track_internode_metrics, *this.internode_operation);
record_internode_error(*this.track_internode_metrics, *this.internode_operation);
Poll::Ready(Err(Error::new(
io::ErrorKind::TimedOut,
@@ -540,7 +750,15 @@ impl AsyncRead for HttpReader {
Poll::Pending
}
}
other => other,
Poll::Ready(Err(err)) => {
record_internode_operation_duration_once(
*this.track_internode_metrics,
*this.internode_operation,
*this.request_started,
this.duration_recorded,
);
Poll::Ready(Err(err))
}
}
}
}
@@ -610,6 +828,8 @@ pin_project! {
handle: tokio::task::JoinHandle<std::io::Result<()>>,
pending_chunk: BytesMut,
finish:bool,
track_internode_metrics: bool,
internode_operation: Option<&'static str>,
}
}
@@ -648,10 +868,17 @@ impl HttpWriter {
.body(body);
// Hold the request until the shutdown signal is received
let request_started = Instant::now();
let response = request.send().await;
match response {
Ok(resp) => {
record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed());
record_internode_http_version(
track_internode_metrics,
internode_operation,
http_version_metric_label(resp.version()),
);
// http_log!("[HttpWriter::spawn] got response: status={}", resp.status());
if !resp.status().is_success() {
record_internode_error(track_internode_metrics, internode_operation);
@@ -667,6 +894,7 @@ impl HttpWriter {
}
}
Err(e) => {
record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed());
record_internode_error(track_internode_metrics, internode_operation);
let classified = classify_reqwest_error(&e);
record_internode_classified_error(track_internode_metrics, internode_operation, classified);
@@ -691,6 +919,8 @@ impl HttpWriter {
handle,
pending_chunk: BytesMut::with_capacity(HTTP_WRITER_BUFFER_SIZE),
finish: false,
track_internode_metrics,
internode_operation,
})
}
@@ -721,6 +951,20 @@ fn internode_rpc_operation(url: &str) -> Option<&'static str> {
}
}
fn http_version_metric_label(version: Version) -> &'static str {
if version == Version::HTTP_09 {
HTTP_VERSION_09_LABEL
} else if version == Version::HTTP_10 {
HTTP_VERSION_10_LABEL
} else if version == Version::HTTP_11 {
HTTP_VERSION_11_LABEL
} else if version == Version::HTTP_2 {
HTTP_VERSION_2_LABEL
} else {
HTTP_VERSION_UNKNOWN_LABEL
}
}
fn record_internode_outgoing_request(track: bool, operation: Option<&'static str>) {
if !track {
return;
@@ -763,6 +1007,60 @@ fn record_internode_classified_error(track: bool, operation: Option<&'static str
}
}
fn record_internode_operation_duration(track: bool, operation: Option<&'static str>, duration: Duration) {
if !track {
return;
}
if let Some(operation) = operation {
crate::http_runtime_sources::record_duration(operation, duration);
}
}
fn record_internode_operation_duration_once(
track: bool,
operation: Option<&'static str>,
request_started: Instant,
duration_recorded: &mut bool,
) {
if *duration_recorded {
return;
}
*duration_recorded = true;
record_internode_operation_duration(track, operation, request_started.elapsed());
}
fn record_internode_http_version(track: bool, operation: Option<&'static str>, http_version: &'static str) {
if !track {
return;
}
if let Some(operation) = operation {
crate::http_runtime_sources::record_http_version(operation, http_version);
}
}
fn record_internode_stall_timeout(track: bool, operation: Option<&'static str>) {
if !track {
return;
}
if let Some(operation) = operation {
crate::http_runtime_sources::record_stall_timeout(operation);
}
}
fn record_internode_write_shutdown_error(track: bool, operation: Option<&'static str>) {
if !track {
return;
}
if let Some(operation) = operation {
crate::http_runtime_sources::record_write_shutdown_error(operation);
}
}
fn poll_send_error_to_io<T>(err: tokio_util::sync::PollSendError<T>, context: &str) -> io::Error {
Error::other(format!("{context}: {err}"))
}
@@ -772,6 +1070,13 @@ fn send_error_to_io<T>(err: tokio_util::sync::PollSendError<T>, context: &str) -
}
impl HttpWriter {
fn take_background_error(&mut self) -> io::Result<()> {
match self.err_rx.try_recv() {
Ok(err) => Err(err),
Err(tokio::sync::oneshot::error::TryRecvError::Empty | tokio::sync::oneshot::error::TryRecvError::Closed) => Ok(()),
}
}
fn poll_send_pending_chunk(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
if self.pending_chunk.is_empty() {
return Poll::Ready(Ok(()));
@@ -799,11 +1104,10 @@ impl AsyncWrite for HttpWriter {
// self.method,
// buf.len()
// );
if let Ok(e) = Pin::new(&mut self.err_rx).try_recv() {
return Poll::Ready(Err(e));
}
let this = self.as_mut().get_mut();
if let Err(err) = this.take_background_error() {
return Poll::Ready(Err(err));
}
if this.pending_chunk.len() >= HTTP_WRITER_BUFFER_SIZE {
match this.poll_send_pending_chunk(cx) {
@@ -832,15 +1136,19 @@ impl AsyncWrite for HttpWriter {
}
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
self.as_mut().get_mut().poll_send_pending_chunk(cx)
let this = self.as_mut().get_mut();
if let Err(err) = this.take_background_error() {
return Poll::Ready(Err(err));
}
this.poll_send_pending_chunk(cx)
}
fn poll_write_vectored(mut self: Pin<&mut Self>, cx: &mut Context<'_>, bufs: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
if let Ok(e) = Pin::new(&mut self.err_rx).try_recv() {
return Poll::Ready(Err(e));
}
let this = self.as_mut().get_mut();
if let Err(err) = this.take_background_error() {
return Poll::Ready(Err(err));
}
if this.pending_chunk.len() >= HTTP_WRITER_BUFFER_SIZE {
match this.poll_send_pending_chunk(cx) {
@@ -883,9 +1191,17 @@ impl AsyncWrite for HttpWriter {
// let url = self.url.clone();
// let method = self.method.clone();
if let Err(err) = self.as_mut().get_mut().take_background_error() {
record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation);
return Poll::Ready(Err(err));
}
match self.as_mut().get_mut().poll_send_pending_chunk(cx) {
Poll::Ready(Ok(())) => {}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
Poll::Ready(Err(err)) => {
record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation);
return Poll::Ready(Err(err));
}
Poll::Pending => return Poll::Pending,
}
@@ -894,11 +1210,15 @@ impl AsyncWrite for HttpWriter {
let this = self.as_mut().get_mut();
match this.sender.poll_reserve(cx) {
Poll::Ready(Ok(())) => {
this.sender
.send_item(None)
.map_err(|e| send_error_to_io(e, "HttpWriter shutdown error"))?;
this.sender.send_item(None).map_err(|e| {
record_internode_write_shutdown_error(this.track_internode_metrics, this.internode_operation);
send_error_to_io(e, "HttpWriter shutdown error")
})?;
}
Poll::Ready(Err(err)) => {
record_internode_write_shutdown_error(this.track_internode_metrics, this.internode_operation);
return Poll::Ready(Err(poll_send_error_to_io(err, "HttpWriter shutdown error")));
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(poll_send_error_to_io(err, "HttpWriter shutdown error"))),
Poll::Pending => return Poll::Pending,
}
// http_log!(
@@ -911,15 +1231,20 @@ impl AsyncWrite for HttpWriter {
}
// Wait for the HTTP request to complete
use futures::FutureExt;
match Pin::new(&mut self.get_mut().handle).poll_unpin(cx) {
Poll::Ready(Ok(_)) => {
match Pin::new(&mut self.as_mut().get_mut().handle).poll_unpin(cx) {
Poll::Ready(Ok(Ok(()))) => {
// http_log!(
// "[HttpWriter::poll_shutdown] HTTP request finished successfully, url: {}, method: {:?}",
// url,
// method
// );
}
Poll::Ready(Ok(Err(err))) => {
record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation);
return Poll::Ready(Err(err));
}
Poll::Ready(Err(e)) => {
record_internode_write_shutdown_error(self.track_internode_metrics, self.internode_operation);
// http_log!("[HttpWriter::poll_shutdown] HTTP request failed: {e}, url: {}, method: {:?}", url, method);
return Poll::Ready(Err(Error::other(format!("HTTP request failed: {e}"))));
}
@@ -992,6 +1317,13 @@ mod tests {
StatusCode::OK
}
async fn reject_put(State(state): State<TestState>, body: Body) -> impl IntoResponse {
state.put_count.fetch_add(1, Ordering::SeqCst);
let bytes = body.collect().await.unwrap().to_bytes();
state.put_bodies.lock().await.push(bytes.to_vec());
StatusCode::INTERNAL_SERVER_ERROR
}
async fn start_test_server(state: TestState) -> Option<(String, tokio::task::JoinHandle<()>)> {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
@@ -1001,6 +1333,7 @@ mod tests {
let addr = listener.local_addr().expect("listener local address should be available");
let app = Router::new()
.route("/stream", get(get_stream).head(reject_head).put(accept_put))
.route("/reject-put", get(get_stream).put(reject_put))
.route("/stall", get(get_stalling_stream))
.route("/delayed-first", get(get_delayed_first_chunk))
.with_state(state);
@@ -1033,6 +1366,14 @@ mod tests {
);
}
#[test]
fn http_version_metrics_labels_are_low_cardinality() {
assert_eq!(http_version_metric_label(Version::HTTP_09), HTTP_VERSION_09_LABEL);
assert_eq!(http_version_metric_label(Version::HTTP_10), HTTP_VERSION_10_LABEL);
assert_eq!(http_version_metric_label(Version::HTTP_11), HTTP_VERSION_11_LABEL);
assert_eq!(http_version_metric_label(Version::HTTP_2), HTTP_VERSION_2_LABEL);
}
#[tokio::test]
async fn http_reader_does_not_send_preflight_head() {
let state = TestState::default();
@@ -1174,6 +1515,37 @@ mod tests {
handle.abort();
}
#[tokio::test]
async fn http_writer_shutdown_reports_http_status_error() {
let state = TestState::default();
let Some((base_url, handle)) = start_test_server(state.clone()).await else {
return;
};
let url = base_url.replace("/stream", "/reject-put");
let mut writer = HttpWriter::new(url, Method::PUT, HeaderMap::new()).await.unwrap();
writer.write_all(b"payload").await.unwrap();
let err = writer
.shutdown()
.await
.expect_err("shutdown should report the HTTP response failure");
let source = err
.get_ref()
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
.expect("expected shutdown error to carry InternodeHttpError source");
assert_eq!(
source.kind(),
InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR)
);
assert_eq!(source.context().method(), "PUT");
assert!(source.context().target().contains("/reject-put"));
assert_eq!(state.put_count.load(Ordering::SeqCst), 1);
assert_eq!(state.put_bodies.lock().await.as_slice(), &[b"payload".to_vec()]);
handle.abort();
}
#[tokio::test]
async fn http_reader_request_error_includes_method_and_url() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
@@ -1267,4 +1639,126 @@ mod tests {
assert!(!should_bypass_proxy_for_url("http://example.com/stream"));
assert!(!should_bypass_proxy_for_url("not-a-url"));
}
#[test]
fn internode_http_tuning_profile_parses_known_values_and_falls_back_to_legacy() {
assert_eq!(parse_internode_http_tuning_profile(None), InternodeHttpTuningProfile::Legacy);
assert_eq!(
parse_internode_http_tuning_profile(Some("balanced")),
InternodeHttpTuningProfile::Balanced
);
assert_eq!(
parse_internode_http_tuning_profile(Some(" Throughput ")),
InternodeHttpTuningProfile::Throughput
);
assert_eq!(
parse_internode_http_tuning_profile(Some("aggressive")),
InternodeHttpTuningProfile::Legacy
);
}
#[test]
fn legacy_internode_http_tuning_keeps_existing_defaults() {
let tuning = InternodeHttpClientTuning::from_values(
InternodeHttpTuningProfile::Legacy,
None,
None,
None,
None,
InternodeHttpTuningProfile::Legacy.default_http2_adaptive_window(),
None,
);
assert_eq!(tuning.pool_max_idle_per_host, None);
assert_eq!(tuning.pool_idle_timeout_secs, None);
assert_eq!(tuning.http2_initial_stream_window_size, None);
assert_eq!(tuning.http2_initial_connection_window_size, None);
assert!(!tuning.http2_adaptive_window);
assert_eq!(tuning.proxy_mode, InternodeHttpProxyMode::Legacy);
}
#[test]
fn balanced_and_throughput_profiles_apply_conservative_defaults() {
let balanced = InternodeHttpClientTuning::from_values(
InternodeHttpTuningProfile::Balanced,
None,
None,
None,
None,
InternodeHttpTuningProfile::Balanced.default_http2_adaptive_window(),
None,
);
let throughput = InternodeHttpClientTuning::from_values(
InternodeHttpTuningProfile::Throughput,
None,
None,
None,
None,
InternodeHttpTuningProfile::Throughput.default_http2_adaptive_window(),
None,
);
assert_eq!(balanced.pool_max_idle_per_host, Some(64));
assert_eq!(balanced.pool_idle_timeout_secs, Some(120));
assert_eq!(balanced.http2_initial_stream_window_size, Some(1024 * 1024));
assert_eq!(balanced.http2_initial_connection_window_size, Some(4 * 1024 * 1024));
assert!(!balanced.http2_adaptive_window);
assert_eq!(balanced.proxy_mode, InternodeHttpProxyMode::NoProxy);
assert_eq!(throughput.pool_max_idle_per_host, Some(256));
assert_eq!(throughput.pool_idle_timeout_secs, Some(300));
assert!(throughput.http2_adaptive_window);
assert_eq!(throughput.proxy_mode, InternodeHttpProxyMode::NoProxy);
}
#[test]
fn internode_http_tuning_overrides_are_clamped() {
let tuning = InternodeHttpClientTuning::from_values(
InternodeHttpTuningProfile::Balanced,
Some(8),
Some(30),
Some(1),
Some(u64::MAX),
false,
Some("system"),
);
assert_eq!(tuning.pool_max_idle_per_host, Some(8));
assert_eq!(tuning.pool_idle_timeout_secs, Some(30));
assert_eq!(tuning.http2_initial_stream_window_size, Some(INTERNODE_HTTP2_WINDOW_MIN));
assert_eq!(tuning.http2_initial_connection_window_size, Some(INTERNODE_HTTP2_WINDOW_MAX));
assert_eq!(tuning.proxy_mode, InternodeHttpProxyMode::System);
}
#[test]
fn internode_http_proxy_policy_matches_profile_and_overrides() {
let legacy =
InternodeHttpClientTuning::from_values(InternodeHttpTuningProfile::Legacy, None, None, None, None, false, None);
let balanced =
InternodeHttpClientTuning::from_values(InternodeHttpTuningProfile::Balanced, None, None, None, None, false, None);
let system_proxy = InternodeHttpClientTuning::from_values(
InternodeHttpTuningProfile::Throughput,
None,
None,
None,
None,
true,
Some("system"),
);
let no_proxy = InternodeHttpClientTuning::from_values(
InternodeHttpTuningProfile::Legacy,
None,
None,
None,
None,
false,
Some("off"),
);
assert!(should_disable_proxy_for_url("http://127.0.0.1:9000/stream", legacy));
assert!(!should_disable_proxy_for_url("http://192.168.1.10:9000/stream", legacy));
assert!(should_disable_proxy_for_url("http://192.168.1.10:9000/stream", balanced));
assert!(!should_disable_proxy_for_url("http://127.0.0.1:9000/stream", system_proxy));
assert!(should_disable_proxy_for_url("http://example.com/stream", no_proxy));
}
}
crates/rio/src/http_runtime_sources.rs
+26
View File
@@ -17,6 +17,7 @@ use rustfs_tls_runtime::{
GlobalPublishedOutboundTlsState, load_global_outbound_tls_generation, load_global_outbound_tls_state,
record_tls_consumer_stale_generation,
};
use std::time::Duration;
pub(crate) fn outbound_tls_generation() -> u64 {
load_global_outbound_tls_generation().0
@@ -76,3 +77,28 @@ pub(crate) fn record_classified_error(operation: &'static str, classification: &
classification,
);
}
pub(crate) fn record_duration(operation: &'static str, duration: Duration) {
global_internode_metrics().record_duration_for_operation_and_backend(
operation,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
duration,
);
}
pub(crate) fn record_http_version(operation: &'static str, http_version: &'static str) {
global_internode_metrics().record_http_version_for_operation_and_backend(
operation,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
http_version,
);
}
pub(crate) fn record_stall_timeout(operation: &'static str) {
global_internode_metrics().record_stall_timeout_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP);
}
pub(crate) fn record_write_shutdown_error(operation: &'static str) {
global_internode_metrics()
.record_write_shutdown_error_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP);
}
rustfs/src/storage/concurrency/manager.rs
+3 -7
View File
@@ -679,7 +679,6 @@ mod integration_tests {
use super::ConcurrencyManager;
use crate::storage::storage_api::concurrency_consumer::PutObjectGuard;
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass};
use rustfs_config::MI_B;
use rustfs_io_core::io_profile::{AccessPattern, StorageMedia};
use serial_test::serial;
use std::time::Duration;
@@ -989,13 +988,10 @@ mod integration_tests {
StorageMedia::Hdd => config.hdd_buffer_cap,
StorageMedia::Unknown => config.ssd_buffer_cap,
};
let expected_max = media_cap.min(MI_B);
let expected_max = media_cap;
// Large base buffer should be constrained by storage cap first, then global clamp.
assert_eq!(
strategy.buffer_size, expected_max,
"Buffer should be capped by media profile and global clamp"
);
// Large base buffer should be constrained by the active storage media cap.
assert_eq!(strategy.buffer_size, expected_max, "Buffer should be capped by the active media profile");
}
#[tokio::test]
rustfs/src/storage/rpc/disk.rs
+213 -38
View File
@@ -14,8 +14,8 @@
use super::NodeService;
use crate::storage::storage_api::rpc_consumer::node_service::{
DeleteOptions, DiskError, DiskInfoOptions, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
StorageDiskRpcExt as _, UpdateMetadataOpts,
BatchReadVersionReq, BatchReadVersionResp, DeleteOptions, DiskError, DiskInfoOptions, FileInfoVersions, ReadMultipleReq,
ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts, validate_batch_read_version_item_count,
};
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use bytes::Bytes;
@@ -55,6 +55,40 @@ fn encode_msgpack_named<T: serde::Serialize>(value: &T, value_name: &str) -> std
Ok(serializer.into_inner())
}
fn encode_read_multiple_response_payloads(
read_multiple_resps: &[ReadMultipleResp],
) -> std::result::Result<(Vec<String>, Vec<Bytes>), DiskError> {
let mut read_multiple_resps_json = Vec::with_capacity(read_multiple_resps.len());
let mut read_multiple_resps_bin = Vec::with_capacity(read_multiple_resps.len());
for read_multiple_resp in read_multiple_resps {
read_multiple_resps_json.push(
serde_json::to_string(read_multiple_resp)
.map_err(|err| DiskError::other(format!("encode ReadMultipleResp json failed: {err}")))?,
);
read_multiple_resps_bin.push(Bytes::from(encode_msgpack(read_multiple_resp, "ReadMultipleResp")?));
}
Ok((read_multiple_resps_json, read_multiple_resps_bin))
}
fn encode_batch_read_version_response_payloads(
batch_read_version_resps: &[BatchReadVersionResp],
) -> std::result::Result<(Vec<String>, Vec<Bytes>), DiskError> {
let mut batch_read_version_resps_json = Vec::with_capacity(batch_read_version_resps.len());
let mut batch_read_version_resps_bin = Vec::with_capacity(batch_read_version_resps.len());
for batch_read_version_resp in batch_read_version_resps {
batch_read_version_resps_json.push(
serde_json::to_string(batch_read_version_resp)
.map_err(|err| DiskError::other(format!("encode BatchReadVersionResp json failed: {err}")))?,
);
batch_read_version_resps_bin.push(Bytes::from(encode_msgpack(batch_read_version_resp, "BatchReadVersionResp")?));
}
Ok((batch_read_version_resps_json, batch_read_version_resps_bin))
}
impl NodeService {
pub(super) async fn handle_disk_info(&self, request: Request<DiskInfoRequest>) -> Result<Response<DiskInfoResponse>, Status> {
let request = request.into_inner();
@@ -92,7 +126,7 @@ impl NodeService {
Ok(Response::new(DiskInfoResponse {
success: false,
disk_info: "".to_string(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -116,7 +150,7 @@ impl NodeService {
} else {
Ok(Response::new(DeleteVolumeResponse {
success: false,
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -144,16 +178,18 @@ impl NodeService {
};
match disk.read_multiple(read_multiple_req).await {
Ok(read_multiple_resps) => {
let read_multiple_resps: Vec<String> = read_multiple_resps
.into_iter()
.filter_map(|read_multiple_resp| serde_json::to_string(&read_multiple_resp).ok())
.collect();
let read_multiple_resps_bin = read_multiple_resps
.iter()
.filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(json_str).ok())
.filter_map(|resp| encode_msgpack(&resp, "ReadMultipleResp").ok())
.map(Into::into)
.collect();
let (read_multiple_resps, read_multiple_resps_bin) =
match encode_read_multiple_response_payloads(&read_multiple_resps) {
Ok(payloads) => payloads,
Err(err) => {
return Ok(Response::new(ReadMultipleResponse {
success: false,
read_multiple_resps: Vec::new(),
read_multiple_resps_bin: Vec::new(),
error: Some(err.into()),
}));
}
};
Ok(Response::new(ReadMultipleResponse {
success: true,
@@ -174,7 +210,77 @@ impl NodeService {
success: false,
read_multiple_resps: Vec::new(),
read_multiple_resps_bin: Vec::new(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
pub(super) async fn handle_batch_read_version(
&self,
request: Request<BatchReadVersionRequest>,
) -> Result<Response<BatchReadVersionResponse>, Status> {
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
let batch_read_version_req: BatchReadVersionReq = match decode_msgpack_or_json(
&request.batch_read_version_req_bin,
&request.batch_read_version_req,
"BatchReadVersionReq",
) {
Ok(batch_read_version_req) => batch_read_version_req,
Err(err) => {
return Ok(Response::new(BatchReadVersionResponse {
success: false,
batch_read_version_resps: Vec::new(),
batch_read_version_resps_bin: Vec::new(),
error: Some(DiskError::other(format!("decode BatchReadVersionReq failed: {err}")).into()),
}));
}
};
if let Err(err) = validate_batch_read_version_item_count(batch_read_version_req.items.len()) {
return Ok(Response::new(BatchReadVersionResponse {
success: false,
batch_read_version_resps: Vec::new(),
batch_read_version_resps_bin: Vec::new(),
error: Some(err.into()),
}));
}
match disk.batch_read_version(batch_read_version_req).await {
Ok(batch_read_version_resps) => {
let (batch_read_version_resps, batch_read_version_resps_bin) =
match encode_batch_read_version_response_payloads(&batch_read_version_resps) {
Ok(payloads) => payloads,
Err(err) => {
return Ok(Response::new(BatchReadVersionResponse {
success: false,
batch_read_version_resps: Vec::new(),
batch_read_version_resps_bin: Vec::new(),
error: Some(err.into()),
}));
}
};
Ok(Response::new(BatchReadVersionResponse {
success: true,
batch_read_version_resps,
batch_read_version_resps_bin,
error: None,
}))
}
Err(err) => Ok(Response::new(BatchReadVersionResponse {
success: false,
batch_read_version_resps: Vec::new(),
batch_read_version_resps_bin: Vec::new(),
error: Some(err.into()),
})),
}
} else {
Ok(Response::new(BatchReadVersionResponse {
success: false,
batch_read_version_resps: Vec::new(),
batch_read_version_resps_bin: Vec::new(),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -228,7 +334,7 @@ impl NodeService {
Ok(Response::new(DeleteVersionsResponse {
success: false,
errors: Vec::new(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -285,7 +391,7 @@ impl NodeService {
Ok(Response::new(DeleteVersionResponse {
success: false,
raw_file_info: "".to_string(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -330,7 +436,7 @@ impl NodeService {
success: false,
raw_file_info: String::new(),
raw_file_info_bin: Vec::new().into(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -392,7 +498,7 @@ impl NodeService {
success: false,
file_info: String::new(),
file_info_bin: Vec::new().into(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -425,7 +531,7 @@ impl NodeService {
} else {
Ok(Response::new(WriteMetadataResponse {
success: false,
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -469,7 +575,7 @@ impl NodeService {
} else {
Ok(Response::new(UpdateMetadataResponse {
success: false,
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -496,7 +602,7 @@ impl NodeService {
Ok(Response::new(ReadMetadataResponse {
success: false,
data: Bytes::new(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -520,7 +626,7 @@ impl NodeService {
} else {
Ok(Response::new(DeletePathsResponse {
success: false,
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -554,7 +660,7 @@ impl NodeService {
Ok(Response::new(StatVolumeResponse {
success: false,
volume_info: String::new(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -587,7 +693,7 @@ impl NodeService {
Ok(Response::new(ListVolumesResponse {
success: false,
volume_infos: Vec::new(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -611,7 +717,7 @@ impl NodeService {
} else {
Ok(Response::new(MakeVolumeResponse {
success: false,
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -635,7 +741,7 @@ impl NodeService {
} else {
Ok(Response::new(MakeVolumesResponse {
success: false,
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -697,7 +803,7 @@ impl NodeService {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -721,7 +827,7 @@ impl NodeService {
Ok(Response::new(ListDirResponse {
success: false,
volumes: Vec::new(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -752,7 +858,7 @@ impl NodeService {
} else {
Ok(Response::new(RenameFileResponse {
success: false,
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -785,7 +891,7 @@ impl NodeService {
} else {
Ok(Response::new(RenamePartResponse {
success: false,
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -834,7 +940,7 @@ impl NodeService {
Ok(Response::new(CheckPartsResponse {
success: false,
check_parts_resp: "".to_string(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -874,7 +980,7 @@ impl NodeService {
Ok(Response::new(ReadPartsResponse {
success: false,
object_part_infos: Bytes::new(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -923,7 +1029,7 @@ impl NodeService {
Ok(Response::new(VerifyFileResponse {
success: false,
check_parts_resp: "".to_string(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -953,7 +1059,7 @@ impl NodeService {
} else {
Ok(Response::new(DeleteResponse {
success: false,
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -992,7 +1098,7 @@ impl NodeService {
metrics.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
Ok(Response::new(WriteAllResponse {
success: false,
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -1037,7 +1143,7 @@ impl NodeService {
Ok(Response::new(ReadAllResponse {
success: false,
data: Bytes::new(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
@@ -1045,7 +1151,12 @@ impl NodeService {
#[cfg(test)]
mod tests {
use super::{decode_msgpack_or_json, encode_msgpack};
use super::{
decode_msgpack_or_json, encode_batch_read_version_response_payloads, encode_msgpack,
encode_read_multiple_response_payloads,
};
use crate::storage::storage_api::ReadMultipleResp;
use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp;
use serde::{Deserialize, Serialize};
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -1080,4 +1191,68 @@ mod tests {
}
);
}
#[test]
fn encode_read_multiple_response_payloads_keeps_json_and_msgpack_in_sync() {
let responses = vec![
ReadMultipleResp {
bucket: "bucket".to_string(),
prefix: "prefix".to_string(),
file: "a".to_string(),
exists: true,
data: b"alpha".to_vec(),
..Default::default()
},
ReadMultipleResp {
bucket: "bucket".to_string(),
prefix: "prefix".to_string(),
file: "b".to_string(),
exists: true,
data: b"beta".to_vec(),
..Default::default()
},
];
let (json_payloads, msgpack_payloads) =
encode_read_multiple_response_payloads(&responses).expect("read multiple responses should encode");
assert_eq!(json_payloads.len(), responses.len());
assert_eq!(msgpack_payloads.len(), responses.len());
let json_decoded: ReadMultipleResp =
serde_json::from_str(&json_payloads[0]).expect("json read multiple response should decode");
let msgpack_decoded = decode_msgpack_or_json::<ReadMultipleResp>(&msgpack_payloads[0], "", "ReadMultipleResp")
.expect("msgpack read multiple response should decode");
assert_eq!(json_decoded.file, responses[0].file);
assert_eq!(msgpack_decoded.file, responses[0].file);
assert_eq!(msgpack_decoded.data, responses[0].data);
}
#[test]
fn encode_batch_read_version_response_payloads_keeps_json_and_msgpack_in_sync() {
let responses = vec![BatchReadVersionResp {
index: 3,
path: "object-a".to_string(),
version_id: "version-a".to_string(),
success: false,
error: "file version not found".to_string(),
..Default::default()
}];
let (json_payloads, msgpack_payloads) =
encode_batch_read_version_response_payloads(&responses).expect("batch read version responses should encode");
assert_eq!(json_payloads.len(), responses.len());
assert_eq!(msgpack_payloads.len(), responses.len());
let json_decoded: BatchReadVersionResp =
serde_json::from_str(&json_payloads[0]).expect("json batch read version response should decode");
let msgpack_decoded = decode_msgpack_or_json::<BatchReadVersionResp>(&msgpack_payloads[0], "", "BatchReadVersionResp")
.expect("msgpack batch read version response should decode");
assert_eq!(json_decoded.index, responses[0].index);
assert_eq!(msgpack_decoded.path, responses[0].path);
assert_eq!(msgpack_decoded.error, responses[0].error);
}
}
rustfs/src/storage/rpc/node_service.rs
+7
View File
@@ -495,6 +495,13 @@ impl Node for NodeService {
self.handle_read_version(request).await
}
async fn batch_read_version(
&self,
request: Request<BatchReadVersionRequest>,
) -> Result<Response<BatchReadVersionResponse>, Status> {
self.handle_batch_read_version(request).await
}
async fn read_xl(&self, request: Request<ReadXlRequest>) -> Result<Response<ReadXlResponse>, Status> {
self.handle_read_xl(request).await
}
rustfs/src/storage/storage_api.rs
+18 -8
View File
@@ -186,11 +186,12 @@ pub(crate) mod rpc_consumer {
pub(crate) mod node_service {
pub(crate) use super::super::{
CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, ECStore, Error, FileInfoVersions,
LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt, StoragePeerS3ClientExt,
UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_local_server_property,
load_bucket_metadata, reload_transition_tier_config, set_bucket_metadata,
BatchReadVersionReq, BatchReadVersionResp, CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore,
ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq,
ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt,
StoragePeerS3ClientExt, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref,
get_local_server_property, load_bucket_metadata, reload_transition_tier_config, set_bucket_metadata,
validate_batch_read_version_item_count,
};
pub(crate) type StorageResult<T> = super::super::Result<T>;
@@ -358,9 +359,10 @@ pub(crate) mod ecstore_data_usage {
#[allow(unused_imports)]
pub(crate) mod ecstore_disk {
pub(crate) use rustfs_ecstore::api::disk::{
CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter,
RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo,
WalkDirOptions, get_object_disk_read_timeout,
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore,
FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout,
validate_batch_read_version_item_count,
};
pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce};
}
@@ -510,6 +512,8 @@ pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints;
pub(crate) type WorkloadAdmissionSnapshotProviderRef = rustfs_ecstore::WorkloadAdmissionSnapshotProviderRef;
pub(crate) type QuotaError = ecstore_bucket::quota::QuotaError;
pub(crate) type RawFileInfo = rustfs_filemeta::RawFileInfo;
pub(crate) type BatchReadVersionReq = ecstore_disk::BatchReadVersionReq;
pub(crate) type BatchReadVersionResp = ecstore_disk::BatchReadVersionResp;
pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq;
pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp;
pub(crate) type ReadOptions = ecstore_disk::ReadOptions;
@@ -518,6 +522,7 @@ pub(crate) type ReplicationStats = StorageReplicationStatsHandle;
pub(crate) type SetupType = ecstore_layout::SetupType;
pub(crate) type StorageError = ecstore_error::StorageError;
pub(crate) type TierConfigMgr = ecstore_tier::TierConfigMgr;
pub(crate) use ecstore_disk::validate_batch_read_version_item_count;
pub(crate) type TransitionState = ecstore_bucket::lifecycle::bucket_lifecycle_ops::TransitionState;
pub(crate) type Error = ecstore_error::Error;
pub(crate) type Result<T> = ecstore_error::Result<T>;
@@ -787,6 +792,7 @@ pub(crate) trait StorageDiskRpcExt {
async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult<DiskInfo>;
async fn delete_volume(&self, volume: &str) -> DiskResult<()>;
async fn read_multiple(&self, req: ReadMultipleReq) -> DiskResult<Vec<ReadMultipleResp>>;
async fn batch_read_version(&self, req: BatchReadVersionReq) -> DiskResult<Vec<BatchReadVersionResp>>;
async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions)
-> Vec<Option<DiskError>>;
async fn delete_version(
@@ -872,6 +878,10 @@ where
ecstore_disk::DiskAPI::read_multiple(self, req).await
}
async fn batch_read_version(&self, req: BatchReadVersionReq) -> DiskResult<Vec<BatchReadVersionResp>> {
ecstore_disk::DiskAPI::batch_read_version(self, req).await
}
async fn delete_versions(
&self,
volume: &str,
scripts/run_issue797_local_4node_16disk_ab.sh
Executable
+582
View File
@@ -0,0 +1,582 @@
#!/usr/bin/env bash
set -euo pipefail
# Local 4-node / 16-disk RustFS runner for rustfs/backlog#797.
# It starts four local RustFS processes, runs warp workloads, and captures
# health, logs, benchmark summaries, and optional signed admin metrics.
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RUSTFS_BIN="${RUSTFS_BIN:-${PROJECT_ROOT}/target/debug/rustfs}"
BUILD_BIN="${BUILD_BIN:-true}"
BASE_PORT="${BASE_PORT:-19100}"
NODE_COUNT="${NODE_COUNT:-4}"
DISKS_PER_NODE="${DISKS_PER_NODE:-4}"
ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin}"
SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}"
REGION="${REGION:-us-east-1}"
SIZES="${SIZES:-4KiB,1MiB}"
CONCURRENCY="${CONCURRENCY:-8}"
DURATION="${DURATION:-60s}"
WARP_BIN="${WARP_BIN:-warp}"
WARP_MODE="${WARP_MODE:-mixed}"
WARP_EXTRA_ARGS="${WARP_EXTRA_ARGS:---noclear}"
PROFILES="${PROFILES:-baseline,metrics_logging}"
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/bench/issue-797-local-4node-16disk-ab-$(date -u +%Y%m%dT%H%M%SZ)}"
DATA_ROOT="${DATA_ROOT:-/tmp/issue797-local-4node-16disk-ab-$(date -u +%Y%m%dT%H%M%SZ)}"
KEEP_DATA="${KEEP_DATA:-false}"
WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}"
HEALTH_POLL_SECS="${HEALTH_POLL_SECS:-2}"
CAPTURE_ADMIN_METRICS="${CAPTURE_ADMIN_METRICS:-true}"
AWSCURL_BIN="${AWSCURL_BIN:-awscurl}"
CURL_BIN="${CURL_BIN:-curl}"
RG_BIN="${RG_BIN:-rg}"
DRY_RUN=false
PIDS=()
usage() {
cat <<'USAGE'
Usage:
scripts/run_issue797_local_4node_16disk_ab.sh [options]
Options:
--rustfs-bin <path> RustFS binary (default: target/debug/rustfs)
--skip-build Do not build rustfs before running
--base-port <port> First node port; uses port..port+3
--sizes <csv> Object sizes for warp (default: 4KiB,1MiB)
--concurrency <n> Warp concurrency (default: 8)
--duration <dur> Warp duration per size/profile (default: 60s)
--profiles <csv> baseline,metrics_logging,locality_on
--out-dir <dir> Output directory
--data-root <dir> Temporary disk root
--keep-data Keep data root after exit
--warp-extra-args <args> Extra args appended to warp (default: --noclear)
--skip-admin-metrics Do not attempt signed admin metrics capture
--dry-run Print planned layout without running
-h, --help Show help
Profiles:
baseline Metrics exports, file logging, shard locality, and batch
processor observation disabled.
metrics_logging Enables metrics export gate, bounded warn-level file
logging, shard locality observe mode, and batch processor
observe mode.
locality_on Same as metrics_logging, but uses shard locality on mode.
Notes:
Admin metrics are captured from /rustfs/admin/v3/metrics when awscurl or
curl --aws-sigv4 is available. If neither is available, benchmark still runs
and records a skipped metrics status file.
USAGE
}
log() {
printf '[INFO] %s\n' "$*"
}
warn() {
printf '[WARN] %s\n' "$*" >&2
}
die() {
printf '[ERROR] %s\n' "$*" >&2
exit 1
}
arg_value() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" || "$value" == --* ]]; then
die "missing value for ${flag}"
fi
printf '%s\n' "$value"
}
# Like arg_value, but accepts values that themselves start with `--`
# (e.g. `--warp-extra-args --noclear`).
arg_value_allow_dashes() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" ]]; then
die "missing value for ${flag}"
fi
printf '%s\n' "$value"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--rustfs-bin) RUSTFS_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--skip-build) BUILD_BIN=false; shift ;;
--base-port) BASE_PORT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--sizes) SIZES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--concurrency) CONCURRENCY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--duration) DURATION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--profiles) PROFILES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--data-root) DATA_ROOT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--keep-data) KEEP_DATA=true; shift ;;
--warp-extra-args) WARP_EXTRA_ARGS="$(arg_value_allow_dashes "$1" "${2:-}")"; shift 2 ;;
--skip-admin-metrics) CAPTURE_ADMIN_METRICS=false; shift ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) usage; exit 0 ;;
*) die "unknown arg: $1" ;;
esac
done
}
is_positive_integer() {
[[ "$1" =~ ^[1-9][0-9]*$ ]]
}
validate_args() {
is_positive_integer "$BASE_PORT" || die "--base-port must be a positive integer"
is_positive_integer "$CONCURRENCY" || die "--concurrency must be a positive integer"
[[ "$NODE_COUNT" == "4" ]] || die "NODE_COUNT is fixed to 4 for this runner"
[[ "$DISKS_PER_NODE" == "4" ]] || die "DISKS_PER_NODE is fixed to 4 for this runner"
}
require_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
die "command not found: $1"
fi
}
print_redacted_command() {
local redact_next=false
local arg
printf 'Command:'
for arg in "$@"; do
if [[ "$redact_next" == "true" ]]; then
printf ' %q' "REDACTED"
redact_next=false
continue
fi
case "$arg" in
--access-key|--secret-key)
printf ' %q' "$arg"
redact_next=true
;;
-accessKey=*|-secretKey=*)
printf ' %q' "${arg%%=*}=REDACTED"
;;
*)
printf ' %q' "$arg"
;;
esac
done
printf '\n'
}
endpoint_for_node() {
local endpoint_node_id="$1"
printf 'http://127.0.0.1:%s' "$((BASE_PORT + endpoint_node_id - 1))"
}
endpoint_label() {
local endpoint="$1"
local label
label="${endpoint#*://}"
label="${label%%/*}"
printf '%s' "$label" | tr -c 'A-Za-z0-9._-' '_'
}
all_endpoints_csv() {
local endpoint_loop_id
local endpoints=()
for ((endpoint_loop_id = 1; endpoint_loop_id <= NODE_COUNT; endpoint_loop_id++)); do
endpoints+=("$(endpoint_for_node "$endpoint_loop_id")")
done
local IFS=','
printf '%s' "${endpoints[*]}"
}
metrics_url() {
# SCANNER(1) + NET(32) + RPC(256): enough for readiness context plus
# internode aggregate traffic without collecting the heavier all-metrics set.
printf '%s/rustfs/admin/v3/metrics?types=289&by-host=true&n=1\n' "${1%/}"
}
curl_supports_aws_sigv4() {
"$CURL_BIN" --help all 2>/dev/null | grep -q -- '--aws-sigv4'
}
profile_is_supported() {
case "$1" in
baseline|metrics_logging|locality_on) return 0 ;;
*) return 1 ;;
esac
}
profile_env_file() {
local profile="$1"
local env_file="$2"
case "$profile" in
baseline)
cat >"$env_file" <<'EOF'
RUSTFS_OBS_LOGGER_LEVEL=off
RUSTFS_OBS_TRACES_EXPORT_ENABLED=false
RUSTFS_OBS_METRICS_EXPORT_ENABLED=false
RUSTFS_OBS_LOGS_EXPORT_ENABLED=false
RUSTFS_OBS_PROFILING_EXPORT_ENABLED=false
RUSTFS_OBS_USE_STDOUT=false
RUSTFS_OBS_LOG_STDOUT_ENABLED=false
RUSTFS_BATCH_PROCESSOR_ADAPTIVE=off
RUSTFS_SHARD_LOCALITY_SCHEDULING=off
EOF
;;
metrics_logging)
cat >"$env_file" <<'EOF'
RUSTFS_OBS_LOGGER_LEVEL=warn
RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318
RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS=500
RUSTFS_OBS_TRACES_EXPORT_ENABLED=false
RUSTFS_OBS_METRICS_EXPORT_ENABLED=true
RUSTFS_OBS_LOGS_EXPORT_ENABLED=false
RUSTFS_OBS_PROFILING_EXPORT_ENABLED=false
RUSTFS_OBS_USE_STDOUT=false
RUSTFS_OBS_LOG_STDOUT_ENABLED=false
RUSTFS_OBS_METER_INTERVAL=5
RUSTFS_OBS_LOG_KEEP_FILES=2
RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES=268435456
RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES=134217728
RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS=0
RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS=30
RUSTFS_BATCH_PROCESSOR_ADAPTIVE=observe
RUSTFS_SHARD_LOCALITY_SCHEDULING=observe
EOF
;;
locality_on)
cat >"$env_file" <<'EOF'
RUSTFS_OBS_LOGGER_LEVEL=warn
RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318
RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS=500
RUSTFS_OBS_TRACES_EXPORT_ENABLED=false
RUSTFS_OBS_METRICS_EXPORT_ENABLED=true
RUSTFS_OBS_LOGS_EXPORT_ENABLED=false
RUSTFS_OBS_PROFILING_EXPORT_ENABLED=false
RUSTFS_OBS_USE_STDOUT=false
RUSTFS_OBS_LOG_STDOUT_ENABLED=false
RUSTFS_OBS_METER_INTERVAL=5
RUSTFS_OBS_LOG_KEEP_FILES=2
RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES=268435456
RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES=134217728
RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS=0
RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS=30
RUSTFS_BATCH_PROCESSOR_ADAPTIVE=observe
RUSTFS_SHARD_LOCALITY_SCHEDULING=on
EOF
;;
esac
}
load_profile_env() {
local env_file="$1"
set -a
# shellcheck disable=SC1090
source "$env_file"
set +a
}
prepare_profile_layout() {
local profile="$1"
local profile_dir="$OUT_DIR/$profile"
mkdir -p "$profile_dir"/{bench,health,logs,metrics,pids}
profile_env_file "$profile" "$profile_dir/profile.env"
}
build_volumes() {
local profile="$1"
local node_index disk_index
local volumes=()
for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do
for ((disk_index = 1; disk_index <= DISKS_PER_NODE; disk_index++)); do
volumes+=("http://127.0.0.1:$((BASE_PORT + node_index - 1))${DATA_ROOT}/${profile}/node${node_index}/disk${disk_index}")
done
done
printf '%s ' "${volumes[@]}"
}
prepare_data_dirs() {
local profile="$1"
local node_index disk_index
for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do
for ((disk_index = 1; disk_index <= DISKS_PER_NODE; disk_index++)); do
mkdir -p "${DATA_ROOT}/${profile}/node${node_index}/disk${disk_index}"
done
done
}
stop_nodes() {
local pid
if [[ ${#PIDS[@]} -eq 0 ]]; then
return
fi
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" >/dev/null 2>&1; then
kill "$pid" >/dev/null 2>&1 || true
fi
done
for ((_ = 1; _ <= 5; _++)); do
local remaining=0
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" >/dev/null 2>&1; then
remaining=$((remaining + 1))
fi
done
[[ "$remaining" -eq 0 ]] && break
sleep 1
done
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" >/dev/null 2>&1; then
kill -9 "$pid" >/dev/null 2>&1 || true
fi
wait "$pid" >/dev/null 2>&1 || true
done
PIDS=()
}
cleanup() {
stop_nodes
if [[ "$KEEP_DATA" != "true" && "$DRY_RUN" != "true" && -n "$DATA_ROOT" && -d "$DATA_ROOT" ]]; then
rm -rf "$DATA_ROOT"
fi
}
trap cleanup EXIT
start_nodes() {
local profile="$1"
local profile_dir="$OUT_DIR/$profile"
local volumes
volumes="$(build_volumes "$profile")"
prepare_data_dirs "$profile"
load_profile_env "$profile_dir/profile.env"
local node_index endpoint log_file pid_file
for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do
endpoint="$(endpoint_for_node "$node_index")"
log_file="$profile_dir/logs/node${node_index}.log"
pid_file="$profile_dir/pids/node${node_index}.pid"
(
export RUSTFS_ACCESS_KEY="$ACCESS_KEY"
export RUSTFS_SECRET_KEY="$SECRET_KEY"
export RUSTFS_ADDRESS="127.0.0.1:$((BASE_PORT + node_index - 1))"
export RUSTFS_CONSOLE_ENABLE=false
export RUSTFS_SCANNER_ENABLED=false
export RUSTFS_SCANNER_START_DELAY_SECS=3600
export RUSTFS_SCANNER_CYCLE=3600
export RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true
export RUSTFS_VOLUMES="$volumes"
export RUSTFS_OBS_SERVICE_NAME="rustfs-issue797-${profile}-node${node_index}"
export RUSTFS_OBS_LOG_DIRECTORY="$profile_dir/logs/node${node_index}"
mkdir -p "$RUSTFS_OBS_LOG_DIRECTORY"
# Keep both env and positional volumes: current CLI still requires the
# positional server volumes, while env makes the layout visible in logs.
# shellcheck disable=SC2086
exec "$RUSTFS_BIN" server $volumes
) >"$log_file" 2>&1 &
PIDS+=("$!")
printf '%s\n' "$!" >"$pid_file"
log "started ${profile} node${node_index} ${endpoint} pid=$!"
done
}
wait_for_health() {
local profile="$1"
local profile_dir="$OUT_DIR/$profile"
local deadline node_index endpoint status_file
deadline=$((SECONDS + WAIT_TIMEOUT_SECS))
while (( SECONDS < deadline )); do
local ready=0
for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do
endpoint="$(endpoint_for_node "$node_index")"
status_file="$profile_dir/health/node${node_index}.live"
if "$CURL_BIN" -fsS --max-time 2 "${endpoint}/health/live" >"$status_file" 2>"$status_file.err"; then
ready=$((ready + 1))
fi
done
if [[ "$ready" -eq "$NODE_COUNT" ]]; then
log "${profile}: all nodes are live"
return 0
fi
sleep "$HEALTH_POLL_SECS"
done
for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do
warn "${profile} node${node_index} log tail:"
tail -n 40 "$profile_dir/logs/node${node_index}.log" >&2 || true
done
die "${profile}: timed out waiting for health"
}
capture_admin_metrics() {
local profile="$1"
local phase="$2"
local profile_dir="$OUT_DIR/$profile"
local status_file="$profile_dir/metrics/${phase}.status"
local endpoint label metrics_file endpoints
if [[ "$CAPTURE_ADMIN_METRICS" != "true" ]]; then
echo "skipped: disabled" >"$status_file"
return 0
fi
local capture_method=""
if command -v "$AWSCURL_BIN" >/dev/null 2>&1; then
capture_method="awscurl"
elif curl_supports_aws_sigv4; then
capture_method="curl-sigv4"
else
echo "skipped: awscurl not found and curl lacks --aws-sigv4" >"$status_file"
return 0
fi
echo "capturing: ${capture_method}" >"$status_file"
IFS=',' read -r -a endpoints <<<"$(all_endpoints_csv)"
for endpoint in "${endpoints[@]}"; do
label="$(endpoint_label "$endpoint")"
metrics_file="$profile_dir/metrics/${phase}.${label}.ndjson"
if [[ "$capture_method" == "awscurl" ]]; then
AWS_ACCESS_KEY_ID="$ACCESS_KEY" \
AWS_SECRET_ACCESS_KEY="$SECRET_KEY" \
AWS_DEFAULT_REGION="$REGION" \
"$AWSCURL_BIN" \
--service s3 \
--region "$REGION" \
--request GET \
"$(metrics_url "$endpoint")" \
>"$metrics_file" 2>"$metrics_file.err" || true
else
local curl_config="$profile_dir/metrics/.curl-sigv4-${phase}.${label}.conf"
{
printf 'aws-sigv4 = "aws:amz:%s:s3"\n' "$REGION"
printf 'user = "%s:%s"\n' "$ACCESS_KEY" "$SECRET_KEY"
printf 'request = "GET"\n'
printf 'max-time = 10\n'
printf 'fail\n'
printf 'silent\n'
printf 'show-error\n'
} >"$curl_config"
chmod 600 "$curl_config"
"$CURL_BIN" --config "$curl_config" "$(metrics_url "$endpoint")" >"$metrics_file" 2>"$metrics_file.err" || true
rm -f "$curl_config"
fi
done
echo "done: ${capture_method}" >"$status_file"
}
run_bench() {
local profile="$1"
local profile_dir="$OUT_DIR/$profile"
local bucket_profile="${profile//_/-}"
local -a cmd=(
"${PROJECT_ROOT}/scripts/run_object_batch_bench.sh"
--tool warp
--endpoint "$(all_endpoints_csv)"
--access-key "$ACCESS_KEY"
--secret-key "$SECRET_KEY"
--region "$REGION"
--auto-new-bucket
--bucket-prefix "issue797-${bucket_profile}"
--sizes "$SIZES"
--concurrency "$CONCURRENCY"
--duration "$DURATION"
--warp-bin "$WARP_BIN"
--warp-mode "$WARP_MODE"
--out-dir "$profile_dir/bench"
)
if [[ -n "$WARP_EXTRA_ARGS" ]]; then
cmd+=(--extra-args "$WARP_EXTRA_ARGS")
fi
print_redacted_command "${cmd[@]}" >"$profile_dir/bench/command.txt"
log "${profile}: running warp benchmark"
"${cmd[@]}" 2>&1 | tee "$profile_dir/bench/run.log"
}
extract_tail_summary() {
local profile="$1"
local profile_dir="$OUT_DIR/$profile"
local output="$profile_dir/bench/tail_latency_summary.txt"
: >"$output"
if ! command -v "$RG_BIN" >/dev/null 2>&1; then
echo "rg not found; skipped" >"$output"
return 0
fi
"$RG_BIN" -n 'Average:|Median:|90th:|99th:|Fastest:|Slowest:|StdDev:|Total:|Throughput by host|warp: <ERROR>' \
"$profile_dir/bench"/*.log >"$output" || true
}
write_run_meta() {
local meta="$OUT_DIR/run-meta.txt"
mkdir -p "$OUT_DIR"
{
echo "created_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "rustfs_bin=$RUSTFS_BIN"
echo "base_port=$BASE_PORT"
echo "node_count=$NODE_COUNT"
echo "disks_per_node=$DISKS_PER_NODE"
echo "sizes=$SIZES"
echo "concurrency=$CONCURRENCY"
echo "duration=$DURATION"
echo "profiles=$PROFILES"
echo "warp_extra_args=$WARP_EXTRA_ARGS"
echo "endpoints=$(all_endpoints_csv)"
echo "data_root=$DATA_ROOT"
echo "keep_data=$KEEP_DATA"
echo "capture_admin_metrics=$CAPTURE_ADMIN_METRICS"
} >"$meta"
}
run_profile() {
local profile="$1"
profile_is_supported "$profile" || die "unsupported profile: $profile"
prepare_profile_layout "$profile"
start_nodes "$profile"
wait_for_health "$profile"
capture_admin_metrics "$profile" before
run_bench "$profile"
capture_admin_metrics "$profile" after
extract_tail_summary "$profile"
stop_nodes
}
main() {
parse_args "$@"
validate_args
write_run_meta
if [[ "$DRY_RUN" == "true" ]]; then
log "dry run only"
cat "$OUT_DIR/run-meta.txt"
return 0
fi
require_cmd "$CURL_BIN"
require_cmd "$WARP_BIN"
if [[ "$BUILD_BIN" == "true" ]]; then
log "building rustfs binary"
cargo build -p rustfs --bin rustfs
fi
[[ -x "$RUSTFS_BIN" ]] || die "rustfs binary is not executable: $RUSTFS_BIN"
local profile
IFS=',' read -r -a profile_arr <<<"$PROFILES"
for profile in "${profile_arr[@]}"; do
profile="${profile//[[:space:]]/}"
[[ -z "$profile" ]] && continue
run_profile "$profile"
done
log "done. Output dir: $OUT_DIR"
}
main "$@"