mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
安正超
Generated
+102
-100
@@ -772,7 +772,7 @@ dependencies = [
|
||||
"http 0.2.12",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"lru 0.16.3",
|
||||
"lru 0.16.4",
|
||||
"percent-encoding",
|
||||
"regex-lite",
|
||||
"sha2 0.10.9",
|
||||
@@ -1091,9 +1091,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.8.8"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8"
|
||||
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
|
||||
dependencies = [
|
||||
"axum-core",
|
||||
"bytes",
|
||||
@@ -1223,9 +1223,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.11.0"
|
||||
version = "2.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
|
||||
checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
|
||||
|
||||
[[package]]
|
||||
name = "blake2"
|
||||
@@ -1462,7 +1462,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"cipher 0.5.1",
|
||||
"cpufeatures 0.3.0",
|
||||
"rand_core 0.10.0",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1998,7 +1998,7 @@ dependencies = [
|
||||
"cpubits",
|
||||
"ctutils",
|
||||
"num-traits",
|
||||
"rand_core 0.10.0",
|
||||
"rand_core 0.10.1",
|
||||
"serdect",
|
||||
"zeroize",
|
||||
]
|
||||
@@ -2021,7 +2021,7 @@ checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710"
|
||||
dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"hybrid-array",
|
||||
"rand_core 0.10.0",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2032,7 +2032,7 @@ checksum = "21f41f23de7d24cdbda7f0c4d9c0351f99a4ceb258ef30e5c1927af8987ffe5a"
|
||||
dependencies = [
|
||||
"crypto-bigint 0.7.3",
|
||||
"libm",
|
||||
"rand_core 0.10.0",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2275,7 +2275,7 @@ dependencies = [
|
||||
"object_store",
|
||||
"parking_lot 0.12.5",
|
||||
"parquet",
|
||||
"rand 0.9.3",
|
||||
"rand 0.9.4",
|
||||
"regex",
|
||||
"sqlparser",
|
||||
"tempfile",
|
||||
@@ -2397,7 +2397,7 @@ dependencies = [
|
||||
"liblzma",
|
||||
"log",
|
||||
"object_store",
|
||||
"rand 0.9.3",
|
||||
"rand 0.9.4",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"url",
|
||||
@@ -2529,7 +2529,7 @@ dependencies = [
|
||||
"log",
|
||||
"object_store",
|
||||
"parking_lot 0.12.5",
|
||||
"rand 0.9.3",
|
||||
"rand 0.9.4",
|
||||
"tempfile",
|
||||
"url",
|
||||
]
|
||||
@@ -2595,7 +2595,7 @@ dependencies = [
|
||||
"md-5 0.10.6",
|
||||
"memchr",
|
||||
"num-traits",
|
||||
"rand 0.9.3",
|
||||
"rand 0.9.4",
|
||||
"regex",
|
||||
"sha2 0.10.9",
|
||||
"unicode-segmentation",
|
||||
@@ -2913,7 +2913,7 @@ dependencies = [
|
||||
"http-body-util",
|
||||
"libc",
|
||||
"log",
|
||||
"lru 0.16.3",
|
||||
"lru 0.16.4",
|
||||
"mime_guess",
|
||||
"parking_lot 0.12.5",
|
||||
"percent-encoding",
|
||||
@@ -3064,8 +3064,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9-tokio-telemetry"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/dial9-rs/dial9-tokio-telemetry.git?rev=7958b5e6562ff8d3baa944d4316c6ed2d7b73794#7958b5e6562ff8d3baa944d4316c6ed2d7b73794"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35c8348045a09a7dee69d000124d42f773f59f85737d3212ae65aa64ac0b4e3e"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"bon",
|
||||
@@ -3076,7 +3077,6 @@ dependencies = [
|
||||
"hostname",
|
||||
"libc",
|
||||
"metrique",
|
||||
"metrique-timesource",
|
||||
"metrique-writer",
|
||||
"pin-project-lite",
|
||||
"serde",
|
||||
@@ -3090,7 +3090,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "dial9-trace-format"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/dial9-rs/dial9-tokio-telemetry.git?rev=7958b5e6562ff8d3baa944d4316c6ed2d7b73794#7958b5e6562ff8d3baa944d4316c6ed2d7b73794"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "80e0ee560b05f09bf817602d57644947e31e83c521d4e0277f723a6e64d44f92"
|
||||
dependencies = [
|
||||
"dial9-trace-format-derive",
|
||||
"serde",
|
||||
@@ -3099,7 +3100,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "dial9-trace-format-derive"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/dial9-rs/dial9-tokio-telemetry.git?rev=7958b5e6562ff8d3baa944d4316c6ed2d7b73794#7958b5e6562ff8d3baa944d4316c6ed2d7b73794"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9dbbd8126d4d6613931317cfe2a7275c1cd487e41c961e42456ab5f956570030"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -3547,7 +3549,7 @@ version = "25.12.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
@@ -3769,7 +3771,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"libc",
|
||||
"r-efi 6.0.0",
|
||||
"rand_core 0.10.0",
|
||||
"rand_core 0.10.1",
|
||||
"wasip2",
|
||||
"wasip3",
|
||||
"wasm-bindgen",
|
||||
@@ -3820,9 +3822,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-auth"
|
||||
version = "1.8.0"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27e658fc9f8b6bdf9a5c816ebca6dd6bcd32f8550e5c6580652b2c0eac1980f6"
|
||||
checksum = "54a26c047222f874ea87177368ad07c65a9f66534ad3a3f9401f1322c802ccac"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"aws-lc-rs",
|
||||
@@ -3831,7 +3833,7 @@ dependencies = [
|
||||
"chrono",
|
||||
"google-cloud-gax",
|
||||
"hex",
|
||||
"hmac 0.12.1",
|
||||
"hmac 0.13.0",
|
||||
"http 1.4.0",
|
||||
"jsonwebtoken",
|
||||
"reqwest 0.13.2",
|
||||
@@ -3840,7 +3842,7 @@ dependencies = [
|
||||
"rustls-pki-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.18",
|
||||
"time",
|
||||
"tokio",
|
||||
@@ -3849,9 +3851,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-gax"
|
||||
version = "1.9.0"
|
||||
version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "505f3e57fbb875646b25c3ccc859c6446bfa411e1958d267bab288980e5afa19"
|
||||
checksum = "7dc387965cc2efc28d73896d6707125815c16792c23c33a0c67794f3d6e31cc8"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
@@ -3869,9 +3871,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-gax-internal"
|
||||
version = "0.7.11"
|
||||
version = "0.7.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65d462b4fcee5f495bfb58edbf4a9250c230a1079d410bdcb8505bc5f713dcee"
|
||||
checksum = "9529b797c68dc5c9d37effe7faa5d96d683a00e728287a9934bf6d3a86b58706"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -3879,6 +3881,7 @@ dependencies = [
|
||||
"google-cloud-gax",
|
||||
"google-cloud-rpc",
|
||||
"google-cloud-wkt",
|
||||
"h2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
@@ -3907,9 +3910,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-iam-v1"
|
||||
version = "1.7.0"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30ce870ac18f3e0a474000cd57eab8bf3c64af8b5ed820468df8612182709c9a"
|
||||
checksum = "103d11cc828e7f33759c4dc6618a98415786fabba268147028e607c4fe1199aa"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -3925,9 +3928,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-longrunning"
|
||||
version = "1.8.0"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebab215997c51f786852840fec8c76174b8a4af96d08e5fc1569742805baab09"
|
||||
checksum = "fa340b46d452fcd5e701b21dab6ed187454b6f1e4300e0bdb928fceefac847fa"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -3943,9 +3946,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-lro"
|
||||
version = "1.4.0"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "82a4f93a1ec8e6e5448899877ea6021e0f5d06e6b08ccd9b0bd99bc837ca357b"
|
||||
checksum = "3bfd3fa41649a3cd53675a94988b7d3789f2d8f2eb1b24af237450fe08ae9643"
|
||||
dependencies = [
|
||||
"google-cloud-gax",
|
||||
"google-cloud-longrunning",
|
||||
@@ -3957,9 +3960,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-rpc"
|
||||
version = "1.3.0"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "691ae06142c69c73bcef2f5c6fa5a6858521aab4cdf1886a6ba70ba1316c7093"
|
||||
checksum = "7e3b123ea17ff20539fbdf145e6213e0464cc0a30b0d078a68bf90405ef17fb7"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"google-cloud-wkt",
|
||||
@@ -3970,9 +3973,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-storage"
|
||||
version = "1.10.0"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f85a4e9a65a2f2c3a1d05c1c3b9deb0177a25488128616a3a96195ab5fa41bef"
|
||||
checksum = "c4cf4d4e84fd3238cd5f5179b0a8e424c7ccc8106423a9fad627ae27ebc2b7b4"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -4001,7 +4004,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"sha2 0.10.9",
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
@@ -4012,9 +4015,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-type"
|
||||
version = "1.3.0"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c310636aa7b660539c3f9259ae7a1fa2fd8bd7965a471bf6467094493cdb715a"
|
||||
checksum = "bbcae00eb50388f5127e65cf2d45679fc4ec0e3596975bb6ec41aec6319a51c9"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"google-cloud-wkt",
|
||||
@@ -4025,9 +4028,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-wkt"
|
||||
version = "1.2.1"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a0ade65b0e4fa9cb4b6f147c8e726803bff453e3190910a53cbd3b0c019f5c2a"
|
||||
checksum = "0b30ccefdb9276269bb0336afe207c5e0ba1a544a5eb0034763af051f2b9eb63"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
@@ -4994,9 +4997,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.184"
|
||||
version = "0.2.185"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af"
|
||||
checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
|
||||
|
||||
[[package]]
|
||||
name = "libflate"
|
||||
@@ -5074,7 +5077,7 @@ version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"libc",
|
||||
"plain",
|
||||
"redox_syscall 0.7.4",
|
||||
@@ -5105,7 +5108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9f8ff371890db2cf65a0758dba9a79f9cd965de369f6dbdc6581a22780af45e"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dashmap",
|
||||
@@ -5185,9 +5188,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.16.3"
|
||||
version = "0.16.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593"
|
||||
checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39"
|
||||
dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
]
|
||||
@@ -5364,16 +5367,16 @@ dependencies = [
|
||||
"ordered-float 5.3.0",
|
||||
"quanta",
|
||||
"radix_trie",
|
||||
"rand 0.9.3",
|
||||
"rand 0.9.4",
|
||||
"rand_xoshiro",
|
||||
"sketches-ddsketch",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "metrique"
|
||||
version = "0.1.23"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c41212ded0b2ba808836b36db163067eb6da41447e7f224267a7331b670009b9"
|
||||
checksum = "f2cb1b5bc8898d2cb0da392c35f91826182c4cced80dddf061abd63bb7e34318"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"jiff",
|
||||
@@ -5391,9 +5394,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique-core"
|
||||
version = "0.1.18"
|
||||
version = "0.1.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5f67f8bf383a36ddb563846a9b143228505fd1f833aa936fdefbc9cb1e46855"
|
||||
checksum = "67e3d9e65d3978d38102e3c4bd4142b91351467bb9c698db99654375e8f3858b"
|
||||
dependencies = [
|
||||
"itertools 0.14.0",
|
||||
"metrique-writer-core",
|
||||
@@ -5414,9 +5417,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique-service-metrics"
|
||||
version = "0.1.19"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07a742784bddd4a8636cf4e952ec4bbd49d4dab34b7dd24d11026ab16adbfe19"
|
||||
checksum = "cd93adb879c2d72ea8a23178deb5c9b379a6e459bfa08eb2c86a9c8c213cb3a6"
|
||||
dependencies = [
|
||||
"metrique-writer",
|
||||
]
|
||||
@@ -5429,9 +5432,9 @@ checksum = "d607939211e4eaaa8cd35394fa5e57faffb7390d0ac513b39992edcaf3cc526c"
|
||||
|
||||
[[package]]
|
||||
name = "metrique-writer"
|
||||
version = "0.1.20"
|
||||
version = "0.1.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ea8f6776cef2fed8ebaea2044971e4bfec96268eb47ccccb0e93797a198b025"
|
||||
checksum = "e7885027d11fce09ba86f50b11ed8c34166a76e68b1568de3e9cb6689c18ea39"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"crossbeam-queue",
|
||||
@@ -5441,7 +5444,7 @@ dependencies = [
|
||||
"metrique-core",
|
||||
"metrique-writer-core",
|
||||
"metrique-writer-macro",
|
||||
"rand 0.9.3",
|
||||
"rand 0.9.4",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -5450,9 +5453,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique-writer-core"
|
||||
version = "0.1.14"
|
||||
version = "0.1.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5399135c49a096ba9565872ddfef24627a4129501e930cbffe84d66897d4b12c"
|
||||
checksum = "c2aa7ddd300b7405e6fc576fa57026b45097b5bf7b7c88c9be180dce6f4a1e09"
|
||||
dependencies = [
|
||||
"derive-where",
|
||||
"itertools 0.14.0",
|
||||
@@ -5577,7 +5580,7 @@ version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22f9786d56d972959e1408b6a93be6af13b9c1392036c5c1fafa08a1b0c6ee87"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"byteorder",
|
||||
"derive_builder",
|
||||
"getset",
|
||||
@@ -5636,7 +5639,7 @@ version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -5649,7 +5652,7 @@ version = "0.30.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -5829,7 +5832,7 @@ version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f049ae562349fefb8e837eb15443da1e7c6dcbd8a11f52a228f92220c2e5c85e"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"libloading",
|
||||
"nvml-wrapper-sys",
|
||||
"static_assertions",
|
||||
@@ -5871,7 +5874,7 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6093,7 +6096,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"opentelemetry",
|
||||
"percent-encoding",
|
||||
"rand 0.9.3",
|
||||
"rand 0.9.4",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
@@ -6337,7 +6340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "500fa4cdeacd98997c5865e3d0d1cb8fe7e9d7d75ecc775e07989a433a9a9a59"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"thiserror 2.0.18",
|
||||
"zerocopy",
|
||||
"zerocopy-derive",
|
||||
@@ -6503,9 +6506,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.32"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
||||
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
|
||||
|
||||
[[package]]
|
||||
name = "plain"
|
||||
@@ -6859,7 +6862,7 @@ version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c3a14896dfa883796f1cb410461aef38810ea05f2b2c33c5aded3649095fdad"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"memchr",
|
||||
"unicase",
|
||||
]
|
||||
@@ -6969,7 +6972,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"getrandom 0.3.4",
|
||||
"lru-slab",
|
||||
"rand 0.9.3",
|
||||
"rand 0.9.4",
|
||||
"ring",
|
||||
"rustc-hash",
|
||||
"rustls",
|
||||
@@ -7039,9 +7042,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.3"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166"
|
||||
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
|
||||
dependencies = [
|
||||
"rand_chacha 0.9.0",
|
||||
"rand_core 0.9.5",
|
||||
@@ -7055,7 +7058,7 @@ checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
|
||||
dependencies = [
|
||||
"chacha20",
|
||||
"getrandom 0.4.2",
|
||||
"rand_core 0.10.0",
|
||||
"rand_core 0.10.1",
|
||||
"serde",
|
||||
]
|
||||
|
||||
@@ -7099,9 +7102,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.10.0"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba"
|
||||
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
||||
|
||||
[[package]]
|
||||
name = "rand_xoshiro"
|
||||
@@ -7129,14 +7132,14 @@ version = "11.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.11.0"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
|
||||
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
@@ -7207,7 +7210,7 @@ version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7216,7 +7219,7 @@ version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7546,7 +7549,7 @@ dependencies = [
|
||||
"digest 0.11.2",
|
||||
"pkcs1 0.8.0-rc.4",
|
||||
"pkcs8 0.11.0-rc.11",
|
||||
"rand_core 0.10.0",
|
||||
"rand_core 0.10.1",
|
||||
"signature 3.0.0-rc.10",
|
||||
"spki 0.8.0",
|
||||
"zeroize",
|
||||
@@ -7949,7 +7952,6 @@ dependencies = [
|
||||
"rustfs-config",
|
||||
"rustfs-credentials",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-io-core",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-lock",
|
||||
"rustfs-madmin",
|
||||
@@ -8702,7 +8704,7 @@ version = "0.38.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.4.15",
|
||||
@@ -8715,7 +8717,7 @@ version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
@@ -8789,9 +8791,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.11"
|
||||
version = "0.103.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "20a6af516fea4b20eccceaf166e8aa666ac996208e8a644ce3ef5aa783bc7cd4"
|
||||
checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
@@ -8973,7 +8975,7 @@ version = "3.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"core-foundation 0.10.1",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
@@ -9288,7 +9290,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f1880df446116126965eeec169136b2e0251dba37c6223bcc819569550edea3"
|
||||
dependencies = [
|
||||
"digest 0.11.2",
|
||||
"rand_core 0.10.0",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9636,9 +9638,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "symbolic-common"
|
||||
version = "12.17.4"
|
||||
version = "12.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "803d14d7cb9c6fa5b95a6f3de8af95b356a528d391998fa45a07d320a5573e51"
|
||||
checksum = "6aba7211a1803a826f108af9f4d86d25abe880712f2a6449479279e861b293f8"
|
||||
dependencies = [
|
||||
"debugid",
|
||||
"memmap2 0.9.10",
|
||||
@@ -9648,9 +9650,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "symbolic-demangle"
|
||||
version = "12.17.4"
|
||||
version = "12.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39505731ae891b2dde47b0e4ae2ec40a7ced3476ab1129f1bf829e3fba62bb83"
|
||||
checksum = "595bddd9d363c2ef6fc9fb33b98416ff209c5aae8bdca89a7dbbf2ef5e0ecc45"
|
||||
dependencies = [
|
||||
"cpp_demangle",
|
||||
"rustc-demangle",
|
||||
@@ -9732,7 +9734,7 @@ version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"core-foundation 0.9.4",
|
||||
"system-configuration-sys",
|
||||
]
|
||||
@@ -10150,7 +10152,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
@@ -10320,7 +10322,7 @@ dependencies = [
|
||||
"http 1.4.0",
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.9.3",
|
||||
"rand 0.9.4",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"sha1 0.10.6",
|
||||
@@ -10659,7 +10661,7 @@ version = "0.244.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"hashbrown 0.15.5",
|
||||
"indexmap 2.14.0",
|
||||
"semver",
|
||||
@@ -11165,7 +11167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags 2.11.0",
|
||||
"bitflags 2.11.1",
|
||||
"indexmap 2.14.0",
|
||||
"log",
|
||||
"serde",
|
||||
|
||||
+1
-1
@@ -288,7 +288,7 @@ zstd = "0.13.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.3"
|
||||
dial9-tokio-telemetry = { version = "0.2", git = "https://github.com/dial9-rs/dial9-tokio-telemetry.git", rev = "7958b5e6562ff8d3baa944d4316c6ed2d7b73794" }
|
||||
dial9-tokio-telemetry = "0.2"
|
||||
opentelemetry = { version = "0.31.0" }
|
||||
opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] }
|
||||
opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] }
|
||||
|
||||
@@ -39,7 +39,6 @@ abd = "abd"
|
||||
mak = "mak"
|
||||
gae = "gae"
|
||||
GAE = "GAE"
|
||||
writeable = "writeable"
|
||||
# s3-tests original test names (cannot be changed)
|
||||
nonexisted = "nonexisted"
|
||||
consts = "consts"
|
||||
|
||||
@@ -49,34 +49,6 @@ pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE";
|
||||
/// to regular I/O without errors.
|
||||
pub const DEFAULT_OBJECT_ZERO_COPY_ENABLE: bool = true;
|
||||
|
||||
/// Environment variable for zero-copy read operating mode.
|
||||
///
|
||||
/// Supported values:
|
||||
/// - `off`: disable mmap-backed chunk fast path and always use the compatibility path
|
||||
/// - `conservative`: allow a single mmap window per request
|
||||
/// - `balanced`: allow multiple mmap windows with the default size guardrails
|
||||
/// - `aggressive`: allow multi-window mmap and relax the small-object cutoff
|
||||
pub const ENV_OBJECT_ZERO_COPY_MODE: &str = "RUSTFS_OBJECT_ZERO_COPY_MODE";
|
||||
|
||||
/// Default zero-copy read mode.
|
||||
pub const DEFAULT_OBJECT_ZERO_COPY_MODE: &str = "balanced";
|
||||
|
||||
/// Environment variable for the maximum mmap window size used by the chunk fast path.
|
||||
///
|
||||
/// This controls the visible bytes per mapped chunk before the implementation emits a new window.
|
||||
pub const ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES";
|
||||
|
||||
/// Default mmap window size for chunk fast path reads: 8 MiB.
|
||||
pub const DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Environment variable for the maximum total active mmap bytes.
|
||||
///
|
||||
/// Requests that would exceed this active window budget fall back to the compatibility path.
|
||||
pub const ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES";
|
||||
|
||||
/// Default maximum active mmap bytes across concurrent local chunk fast-path reads: 256 MiB.
|
||||
pub const DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: usize = 256 * 1024 * 1024;
|
||||
|
||||
// =============================================================================
|
||||
// Direct I/O Configuration
|
||||
// =============================================================================
|
||||
|
||||
@@ -20,11 +20,6 @@ license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "small_put_bench"
|
||||
path = "src/bin/small_put_bench.rs"
|
||||
test = false
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
|
||||
@@ -1,441 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{Delete, ObjectIdentifier};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use bytes::Bytes;
|
||||
use clap::Parser;
|
||||
use serde::Serialize;
|
||||
use std::path::PathBuf;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "small_put_bench")]
|
||||
#[command(about = "Rust-native small PUT benchmark for RustFS-compatible S3 endpoints")]
|
||||
struct Args {
|
||||
#[arg(long, env = "RUSTFS_BENCH_ENDPOINT")]
|
||||
endpoint: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_ACCESS_KEY", default_value = "rustfsadmin")]
|
||||
access_key: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_SECRET_KEY", default_value = "rustfsadmin")]
|
||||
secret_key: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_REGION", default_value = "us-east-1")]
|
||||
region: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_BUCKET", default_value = "small-put-benchmark")]
|
||||
bucket: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_SIZES", default_value = "4KiB,16KiB,64KiB,256KiB,1MiB")]
|
||||
sizes: String,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_CONCURRENCY", default_value_t = 8)]
|
||||
concurrency: usize,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_DURATION_SECS", default_value_t = 10)]
|
||||
duration_secs: u64,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_TIMEOUT_SECS", default_value_t = 15)]
|
||||
timeout_secs: u64,
|
||||
|
||||
#[arg(long, env = "RUSTFS_BENCH_PREFIX")]
|
||||
prefix: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
output_json: Option<PathBuf>,
|
||||
|
||||
#[arg(long, default_value_t = false)]
|
||||
cleanup: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct SizeSpec {
|
||||
label: String,
|
||||
slug: String,
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Sample {
|
||||
ok: bool,
|
||||
duration_ms: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct SizeSummary {
|
||||
label: String,
|
||||
bytes: usize,
|
||||
total: usize,
|
||||
succeeded: usize,
|
||||
failed: usize,
|
||||
wall_secs: f64,
|
||||
object_rate: f64,
|
||||
throughput_mib_per_sec: f64,
|
||||
avg_ms: Option<f64>,
|
||||
p50_ms: Option<f64>,
|
||||
p90_ms: Option<f64>,
|
||||
p99_ms: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RunSummary {
|
||||
run_id: String,
|
||||
endpoint: String,
|
||||
bucket: String,
|
||||
concurrency: usize,
|
||||
duration_secs: u64,
|
||||
timeout_secs: u64,
|
||||
sizes: Vec<SizeSummary>,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.context("failed to build tokio runtime")?;
|
||||
runtime.block_on(async_main())
|
||||
}
|
||||
|
||||
async fn async_main() -> Result<()> {
|
||||
let args = Args::parse();
|
||||
validate_args(&args)?;
|
||||
|
||||
let sizes = parse_size_list(&args.sizes)?;
|
||||
let run_id = args.prefix.clone().unwrap_or_else(default_run_id);
|
||||
let client = build_s3_client(&args.endpoint, &args.access_key, &args.secret_key, &args.region);
|
||||
|
||||
ensure_bucket(&client, &args.bucket).await?;
|
||||
|
||||
let mut size_summaries = Vec::with_capacity(sizes.len());
|
||||
for size in &sizes {
|
||||
let summary = run_size_benchmark(
|
||||
client.clone(),
|
||||
args.bucket.clone(),
|
||||
run_id.clone(),
|
||||
size.clone(),
|
||||
args.concurrency,
|
||||
Duration::from_secs(args.duration_secs),
|
||||
Duration::from_secs(args.timeout_secs),
|
||||
)
|
||||
.await?;
|
||||
print_size_summary(&summary);
|
||||
size_summaries.push(summary);
|
||||
}
|
||||
|
||||
if args.cleanup {
|
||||
cleanup_prefix(&client, &args.bucket, &run_id).await?;
|
||||
}
|
||||
|
||||
let summary = RunSummary {
|
||||
run_id,
|
||||
endpoint: args.endpoint,
|
||||
bucket: args.bucket,
|
||||
concurrency: args.concurrency,
|
||||
duration_secs: args.duration_secs,
|
||||
timeout_secs: args.timeout_secs,
|
||||
sizes: size_summaries,
|
||||
};
|
||||
|
||||
if let Some(path) = args.output_json {
|
||||
let json = serde_json::to_vec_pretty(&summary).context("failed to serialize benchmark summary")?;
|
||||
std::fs::write(&path, json).with_context(|| format!("failed to write benchmark summary to {}", path.display()))?;
|
||||
println!("Wrote summary to {}", path.display());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_args(args: &Args) -> Result<()> {
|
||||
if args.concurrency == 0 {
|
||||
bail!("--concurrency must be greater than zero");
|
||||
}
|
||||
if args.duration_secs == 0 {
|
||||
bail!("--duration-secs must be greater than zero");
|
||||
}
|
||||
if args.timeout_secs == 0 {
|
||||
bail!("--timeout-secs must be greater than zero");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn build_s3_client(endpoint: &str, access_key: &str, secret_key: &str, region: &str) -> Client {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "small-put-bench");
|
||||
let mut config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new(region.to_string()))
|
||||
.endpoint_url(endpoint)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest();
|
||||
|
||||
if endpoint.starts_with("http://") {
|
||||
config = config.http_client(SmithyHttpClientBuilder::new().build_http());
|
||||
}
|
||||
|
||||
Client::from_conf(config.build())
|
||||
}
|
||||
|
||||
async fn ensure_bucket(client: &Client, bucket: &str) -> Result<()> {
|
||||
if client.head_bucket().bucket(bucket).send().await.is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match client.create_bucket().bucket(bucket).send().await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
let rendered = err.to_string();
|
||||
if rendered.contains("BucketAlreadyOwnedByYou") || rendered.contains("BucketAlreadyExists") {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(err).with_context(|| format!("failed to create benchmark bucket {bucket}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_size_benchmark(
|
||||
client: Client,
|
||||
bucket: String,
|
||||
run_id: String,
|
||||
size: SizeSpec,
|
||||
concurrency: usize,
|
||||
duration: Duration,
|
||||
timeout: Duration,
|
||||
) -> Result<SizeSummary> {
|
||||
let payload = Bytes::from(vec![0_u8; size.bytes]);
|
||||
let deadline = Instant::now() + duration;
|
||||
let wall_start = Instant::now();
|
||||
|
||||
let mut handles = Vec::with_capacity(concurrency);
|
||||
for worker in 0..concurrency {
|
||||
let client = client.clone();
|
||||
let bucket = bucket.clone();
|
||||
let payload = payload.clone();
|
||||
let prefix = format!("{run_id}/{}/worker-{worker}", size.slug);
|
||||
handles.push(tokio::spawn(async move {
|
||||
let mut samples = Vec::new();
|
||||
let mut idx = 0usize;
|
||||
|
||||
while Instant::now() < deadline {
|
||||
let key = format!("{prefix}/obj-{idx}.bin");
|
||||
let started_at = Instant::now();
|
||||
let request = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload.clone()))
|
||||
.content_type("application/octet-stream");
|
||||
|
||||
let ok = matches!(tokio::time::timeout(timeout, request.send()).await, Ok(Ok(_)));
|
||||
samples.push(Sample {
|
||||
ok,
|
||||
duration_ms: started_at.elapsed().as_secs_f64() * 1000.0,
|
||||
});
|
||||
idx += 1;
|
||||
}
|
||||
|
||||
samples
|
||||
}));
|
||||
}
|
||||
|
||||
let mut samples = Vec::new();
|
||||
for handle in handles {
|
||||
samples.extend(handle.await.map_err(|err| anyhow!("benchmark worker join error: {err}"))?);
|
||||
}
|
||||
|
||||
Ok(build_size_summary(&size, samples, wall_start.elapsed()))
|
||||
}
|
||||
|
||||
fn build_size_summary(size: &SizeSpec, mut samples: Vec<Sample>, wall_elapsed: Duration) -> SizeSummary {
|
||||
let total = samples.len();
|
||||
let succeeded = samples.iter().filter(|sample| sample.ok).count();
|
||||
let failed = total.saturating_sub(succeeded);
|
||||
let wall_secs = wall_elapsed.as_secs_f64();
|
||||
let object_rate = if wall_secs > 0.0 { succeeded as f64 / wall_secs } else { 0.0 };
|
||||
let throughput_mib_per_sec = if wall_secs > 0.0 {
|
||||
((size.bytes * succeeded) as f64 / (1024.0 * 1024.0)) / wall_secs
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let avg_ms = if total > 0 {
|
||||
Some(samples.iter().map(|sample| sample.duration_ms).sum::<f64>() / total as f64)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
samples.sort_by(|lhs, rhs| lhs.duration_ms.total_cmp(&rhs.duration_ms));
|
||||
let durations: Vec<f64> = samples.into_iter().map(|sample| sample.duration_ms).collect();
|
||||
|
||||
SizeSummary {
|
||||
label: size.label.clone(),
|
||||
bytes: size.bytes,
|
||||
total,
|
||||
succeeded,
|
||||
failed,
|
||||
wall_secs,
|
||||
object_rate,
|
||||
throughput_mib_per_sec,
|
||||
avg_ms,
|
||||
p50_ms: percentile(&durations, 0.50),
|
||||
p90_ms: percentile(&durations, 0.90),
|
||||
p99_ms: percentile(&durations, 0.99),
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_prefix(client: &Client, bucket: &str, prefix: &str) -> Result<()> {
|
||||
let mut continuation_token = None;
|
||||
loop {
|
||||
let response = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.prefix(prefix)
|
||||
.set_continuation_token(continuation_token.clone())
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("failed to list objects for cleanup under {bucket}/{prefix}"))?;
|
||||
|
||||
let objects: Vec<ObjectIdentifier> = response
|
||||
.contents
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter_map(|object| object.key.map(|key| ObjectIdentifier::builder().key(key).build().ok()))
|
||||
.flatten()
|
||||
.collect();
|
||||
|
||||
for chunk in objects.chunks(1_000) {
|
||||
if chunk.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.set_objects(Some(chunk.to_vec()))
|
||||
.quiet(true)
|
||||
.build()
|
||||
.context("failed to build delete request")?,
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("failed to delete cleanup batch under {bucket}/{prefix}"))?;
|
||||
}
|
||||
|
||||
if response.is_truncated.unwrap_or(false) {
|
||||
continuation_token = response.next_continuation_token;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_size_list(input: &str) -> Result<Vec<SizeSpec>> {
|
||||
input
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|item| !item.is_empty())
|
||||
.map(parse_size_spec)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse_size_spec(input: &str) -> Result<SizeSpec> {
|
||||
let normalized = input.trim();
|
||||
let lower = normalized.to_ascii_lowercase();
|
||||
|
||||
let (number_part, multiplier) = if let Some(value) = lower.strip_suffix("kib") {
|
||||
(value, 1024usize)
|
||||
} else if let Some(value) = lower.strip_suffix("mib") {
|
||||
(value, 1024usize * 1024usize)
|
||||
} else if let Some(value) = lower.strip_suffix('b') {
|
||||
(value, 1usize)
|
||||
} else {
|
||||
(lower.as_str(), 1usize)
|
||||
};
|
||||
|
||||
let value = number_part
|
||||
.trim()
|
||||
.parse::<usize>()
|
||||
.with_context(|| format!("invalid size component: {input}"))?;
|
||||
let bytes = value
|
||||
.checked_mul(multiplier)
|
||||
.ok_or_else(|| anyhow!("size overflow for {input}"))?;
|
||||
|
||||
Ok(SizeSpec {
|
||||
label: normalized.to_string(),
|
||||
slug: normalized
|
||||
.chars()
|
||||
.filter(|ch| ch.is_ascii_alphanumeric())
|
||||
.collect::<String>()
|
||||
.to_ascii_lowercase(),
|
||||
bytes,
|
||||
})
|
||||
}
|
||||
|
||||
fn percentile(values: &[f64], percentile: f64) -> Option<f64> {
|
||||
if values.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let index = ((values.len() - 1) as f64 * percentile).floor() as usize;
|
||||
values.get(index).copied()
|
||||
}
|
||||
|
||||
fn default_run_id() -> String {
|
||||
format!("small-put-bench-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"))
|
||||
}
|
||||
|
||||
fn print_size_summary(summary: &SizeSummary) {
|
||||
println!(
|
||||
"{}: success={} failed={} obj/s={:.3} MiB/s={:.3} avg={:.3?} p50={:.3?} p90={:.3?} p99={:.3?}",
|
||||
summary.label,
|
||||
summary.succeeded,
|
||||
summary.failed,
|
||||
summary.object_rate,
|
||||
summary.throughput_mib_per_sec,
|
||||
summary.avg_ms,
|
||||
summary.p50_ms,
|
||||
summary.p90_ms,
|
||||
summary.p99_ms,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_size_spec_supports_binary_units() {
|
||||
let four_kib = parse_size_spec("4KiB").expect("4KiB should parse");
|
||||
assert_eq!(four_kib.bytes, 4 * 1024);
|
||||
|
||||
let one_mib = parse_size_spec("1MiB").expect("1MiB should parse");
|
||||
assert_eq!(one_mib.bytes, 1024 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percentile_returns_expected_bucket() {
|
||||
let values = vec![10.0, 20.0, 30.0, 40.0, 50.0];
|
||||
assert_eq!(percentile(&values, 0.50), Some(30.0));
|
||||
assert_eq!(percentile(&values, 0.90), Some(40.0));
|
||||
}
|
||||
}
|
||||
@@ -19,13 +19,9 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::{ByteStream, SdkBody};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
|
||||
use base64::Engine;
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use http_body::Frame;
|
||||
use http_body_util::StreamBody;
|
||||
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -68,53 +64,6 @@ mod tests {
|
||||
.encoded
|
||||
}
|
||||
|
||||
fn streamed_body_70kib_of_a() -> ByteStream {
|
||||
let bytes = Bytes::from_static(&[b'a'; 1024]);
|
||||
let stream = futures::stream::repeat_with(move || {
|
||||
let frame = Frame::data(bytes.clone());
|
||||
Ok::<_, std::io::Error>(frame)
|
||||
});
|
||||
let body = WithSizeHint::new(StreamBody::new(stream.take(70)), 70 * 1024);
|
||||
ByteStream::new(SdkBody::from_body_1_x(body))
|
||||
}
|
||||
|
||||
struct WithSizeHint<T> {
|
||||
inner: T,
|
||||
size_hint: usize,
|
||||
}
|
||||
|
||||
impl<T> WithSizeHint<T> {
|
||||
fn new(inner: T, size_hint: usize) -> Self {
|
||||
Self { inner, size_hint }
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> http_body::Body for WithSizeHint<T>
|
||||
where
|
||||
T: http_body::Body + Unpin,
|
||||
{
|
||||
type Data = T::Data;
|
||||
type Error = T::Error;
|
||||
|
||||
fn poll_frame(
|
||||
self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
) -> std::task::Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
|
||||
let this = self.get_mut();
|
||||
std::pin::Pin::new(&mut this.inner).poll_frame(cx)
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.inner.is_end_stream()
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
let mut hint = self.inner.size_hint();
|
||||
hint.set_exact(self.size_hint as u64);
|
||||
hint
|
||||
}
|
||||
}
|
||||
|
||||
/// PutObject with Content-MD5: upload succeeds and GetObject returns same content.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
@@ -187,121 +136,6 @@ mod tests {
|
||||
info!("PASSED: PutObject with checksum_sha256 and GetObject content match");
|
||||
}
|
||||
|
||||
/// Mirrors `s3s-e2e` behavior: only request `checksum_algorithm`, then expect
|
||||
/// both PutObject and GetObject(checksum_mode=enabled) to expose the same checksum.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_object_with_checksum_algorithm_only() {
|
||||
init_logging();
|
||||
info!("TEST: PutObject with checksum_algorithm only");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-checksum-algorithm-only";
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let key = "obj-with-checksum-algorithm-only.txt";
|
||||
let content = vec![b'a'; 70 * 1024];
|
||||
|
||||
let put_resp = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
.body(ByteStream::from(content.clone()))
|
||||
.send()
|
||||
.await
|
||||
.expect("PutObject with checksum_algorithm should succeed");
|
||||
|
||||
let put_checksum = put_resp
|
||||
.checksum_crc32()
|
||||
.expect("PutObject should return checksum_crc32 when checksum_algorithm is used")
|
||||
.to_string();
|
||||
|
||||
let mut get_resp = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("GetObject should succeed");
|
||||
|
||||
let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(aws_sdk_s3::primitives::SdkBody::empty()))
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect body")
|
||||
.into_bytes();
|
||||
|
||||
assert_eq!(body_bytes.as_ref(), content.as_slice(), "GetObject body must match uploaded content");
|
||||
assert_eq!(
|
||||
get_resp.checksum_crc32().map(str::to_string),
|
||||
Some(put_checksum),
|
||||
"GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum"
|
||||
);
|
||||
}
|
||||
|
||||
/// Matches the `s3s-e2e` streaming upload shape more closely than `ByteStream::from(Vec<u8>)`.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_object_with_checksum_algorithm_only_streaming_body() {
|
||||
init_logging();
|
||||
info!("TEST: PutObject with checksum_algorithm only using streaming body");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-checksum-algorithm-streaming";
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let key = "obj-with-checksum-algorithm-streaming.txt";
|
||||
let expected_content = vec![b'a'; 70 * 1024];
|
||||
|
||||
let put_resp = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
.body(streamed_body_70kib_of_a())
|
||||
.send()
|
||||
.await
|
||||
.expect("PutObject with streaming checksum_algorithm should succeed");
|
||||
|
||||
let put_checksum = put_resp
|
||||
.checksum_crc32()
|
||||
.expect("PutObject should return checksum_crc32 for streaming checksum_algorithm uploads")
|
||||
.to_string();
|
||||
|
||||
let mut get_resp = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("GetObject should succeed");
|
||||
|
||||
let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(SdkBody::empty()))
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect body")
|
||||
.into_bytes();
|
||||
|
||||
assert_eq!(
|
||||
body_bytes.as_ref(),
|
||||
expected_content.as_slice(),
|
||||
"GetObject body must match uploaded content"
|
||||
);
|
||||
assert_eq!(
|
||||
get_resp.checksum_crc32().map(str::to_string),
|
||||
Some(put_checksum),
|
||||
"GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum for streaming uploads"
|
||||
);
|
||||
}
|
||||
|
||||
/// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
|
||||
/// Uses part size >= 5MB (server minimum) for two parts.
|
||||
#[tokio::test]
|
||||
@@ -400,114 +234,6 @@ mod tests {
|
||||
info!("PASSED: MultipartUpload with checksum and GetObject content match");
|
||||
}
|
||||
|
||||
/// Mirrors `s3s-e2e` multipart behavior: request checksum algorithm at MPU creation,
|
||||
/// rely on auto checksum handling during UploadPart, and expect CompleteMultipartUpload to succeed.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_multipart_upload_with_crc32_algorithm_only() {
|
||||
init_logging();
|
||||
info!("TEST: MultipartUpload with checksum_algorithm only (CRC32)");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-multipart-checksum-crc32-auto";
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let key = "multipart-with-crc32-auto.bin";
|
||||
let part1_content = "a".repeat(5 * 1024 * 1024 + 1);
|
||||
let part2_content = "b".repeat(1024);
|
||||
|
||||
let create_resp = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
.send()
|
||||
.await
|
||||
.expect("CreateMultipartUpload should succeed");
|
||||
|
||||
let upload_id = create_resp.upload_id().expect("upload_id should be present");
|
||||
|
||||
let part1_resp = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(part1_content.clone().into_bytes()))
|
||||
.send()
|
||||
.await
|
||||
.expect("UploadPart 1 should succeed");
|
||||
let part1_checksum = part1_resp
|
||||
.checksum_crc32()
|
||||
.expect("UploadPart 1 should return checksum_crc32")
|
||||
.to_string();
|
||||
|
||||
let part2_resp = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(part2_content.clone().into_bytes()))
|
||||
.send()
|
||||
.await
|
||||
.expect("UploadPart 2 should succeed");
|
||||
let part2_checksum = part2_resp
|
||||
.checksum_crc32()
|
||||
.expect("UploadPart 2 should return checksum_crc32")
|
||||
.to_string();
|
||||
|
||||
let completed_upload = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(part1_resp.e_tag().expect("etag part 1"))
|
||||
.checksum_crc32(part1_checksum)
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(part2_resp.e_tag().expect("etag part 2"))
|
||||
.checksum_crc32(part2_checksum)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(completed_upload)
|
||||
.send()
|
||||
.await
|
||||
.expect("CompleteMultipartUpload should succeed");
|
||||
|
||||
let body_bytes = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect("GetObject should succeed")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect body")
|
||||
.into_bytes();
|
||||
|
||||
let expected_content = format!("{part1_content}{part2_content}");
|
||||
assert_eq!(
|
||||
body_bytes.as_ref(),
|
||||
expected_content.as_bytes(),
|
||||
"completed multipart object must match concatenated parts"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression test for issue #2282:
|
||||
/// CRC64NVME full-object checksum should match between direct PutObject and multipart upload.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -344,9 +344,6 @@ async fn test_local_kms_multipart_upload() {
|
||||
test_multipart_upload_with_sse_c(&s3_client, TEST_BUCKET)
|
||||
.await
|
||||
.expect("SSE-C multipart upload test failed");
|
||||
test_multipart_download_with_wrong_sse_c_key_fails(&s3_client, TEST_BUCKET)
|
||||
.await
|
||||
.expect("SSE-C multipart wrong-key download test failed");
|
||||
|
||||
// Test 4: Large multipart upload (test streaming encryption with multiple blocks)
|
||||
// TODO: Re-enable after fixing streaming encryption issues with large files
|
||||
@@ -651,99 +648,6 @@ async fn test_multipart_upload_with_sse_c(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn test_multipart_download_with_wrong_sse_c_key_fails(
|
||||
s3_client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let object_key = "multipart-sse-c-bad-download-test";
|
||||
let part_size = 5 * 1024 * 1024;
|
||||
let total_parts = 2;
|
||||
let total_size = part_size * total_parts;
|
||||
|
||||
let encryption_key = "01234567890123456789012345678901";
|
||||
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
|
||||
let key_md5 = sse_customer_key_md5_base64(encryption_key);
|
||||
|
||||
let wrong_key = "abcdefghijklmnopqrstuvwxyz012345";
|
||||
let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
|
||||
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
|
||||
|
||||
let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 5) % 256) as u8).collect();
|
||||
|
||||
let create_multipart_output = s3_client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&key_b64)
|
||||
.sse_customer_key_md5(&key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let upload_id = create_multipart_output.upload_id().unwrap();
|
||||
let mut completed_parts = Vec::new();
|
||||
|
||||
for part_number in 1..=total_parts {
|
||||
let start = (part_number - 1) * part_size;
|
||||
let end = std::cmp::min(start + part_size, total_size);
|
||||
let part_data = &test_data[start..end];
|
||||
|
||||
let upload_part_output = s3_client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(part_number as i32)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec()))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&key_b64)
|
||||
.sse_customer_key_md5(&key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
completed_parts.push(
|
||||
aws_sdk_s3::types::CompletedPart::builder()
|
||||
.part_number(part_number as i32)
|
||||
.e_tag(upload_part_output.e_tag().unwrap())
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
|
||||
let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
|
||||
.set_parts(Some(completed_parts))
|
||||
.build();
|
||||
|
||||
s3_client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(completed_multipart_upload)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let err = s3_client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&wrong_key_b64)
|
||||
.sse_customer_key_md5(&wrong_key_md5)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("multipart SSE-C download with the wrong key should fail");
|
||||
|
||||
let service_err = err.into_service_error();
|
||||
assert_eq!(
|
||||
service_err.meta().code(),
|
||||
Some("InvalidRequest"),
|
||||
"wrong-key multipart SSE-C download should return InvalidRequest, got {:?}",
|
||||
service_err.meta().code()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test large multipart upload to verify streaming encryption works correctly
|
||||
#[allow(dead_code)]
|
||||
async fn test_large_multipart_upload(
|
||||
|
||||
@@ -100,10 +100,6 @@ mod cluster_concurrency_test;
|
||||
#[cfg(test)]
|
||||
mod checksum_upload_test;
|
||||
|
||||
// Range request regression tests
|
||||
#[cfg(test)]
|
||||
mod range_request_test;
|
||||
|
||||
// Group deletion tests
|
||||
#[cfg(test)]
|
||||
mod group_delete_test;
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! End-to-end regression tests for invalid and suffix GET object ranges.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use serial_test::serial;
|
||||
use tracing::info;
|
||||
|
||||
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
|
||||
env.create_s3_client()
|
||||
}
|
||||
|
||||
async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
match client.create_bucket().bucket(bucket).send().await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
if err.to_string().contains("BucketAlreadyOwnedByYou") || err.to_string().contains("BucketAlreadyExists") {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Box::new(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_object_invalid_range_returns_416_issue_s3_implemented_tests() {
|
||||
init_logging();
|
||||
info!("TEST: GetObject invalid range should return InvalidRange/416");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-invalid-range";
|
||||
let key = "range.txt";
|
||||
let content = b"testcontent";
|
||||
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await
|
||||
.expect("PutObject should succeed");
|
||||
|
||||
let result = client.get_object().bucket(bucket).key(key).range("bytes=40-50").send().await;
|
||||
|
||||
let err = result.expect_err("GetObject with an unsatisfiable range should fail");
|
||||
match err {
|
||||
SdkError::ServiceError(service_err) => {
|
||||
assert_eq!(service_err.raw().status().as_u16(), 416, "invalid range should return HTTP 416");
|
||||
|
||||
let s3_err = service_err.into_err();
|
||||
assert_eq!(s3_err.meta().code(), Some("InvalidRange"), "invalid range should map to InvalidRange");
|
||||
}
|
||||
other_err => panic!("Expected S3 service error, got: {other_err:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_object_suffix_byte_range_returns_correct_body() {
|
||||
init_logging();
|
||||
info!("TEST: GetObject suffix byte-range should return correct body");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-suffix-range";
|
||||
let key = "range-suffix.bin";
|
||||
|
||||
// ~3 MB so the object spans multiple erasure blocks (block_size = 1 MB).
|
||||
// Suffix ranges on single-block objects never hit the bug.
|
||||
let file_size: usize = 3_095_910;
|
||||
let content: Vec<u8> = (0..file_size).map(|i| (i % 256) as u8).collect();
|
||||
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(content.clone()))
|
||||
.send()
|
||||
.await
|
||||
.expect("PutObject should succeed");
|
||||
|
||||
// bytes=-8 — last 8 bytes (parquet footer read)
|
||||
let result = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.range("bytes=-8")
|
||||
.send()
|
||||
.await
|
||||
.expect("bytes=-8 should succeed");
|
||||
let body = result.body.collect().await.expect("bytes=-8 body").into_bytes();
|
||||
assert_eq!(body.len(), 8, "bytes=-8 body length");
|
||||
assert_eq!(&body[..], &content[file_size - 8..], "bytes=-8 body content");
|
||||
|
||||
// bytes=-96 — last 96 bytes
|
||||
let result = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.range("bytes=-96")
|
||||
.send()
|
||||
.await
|
||||
.expect("bytes=-96 should succeed");
|
||||
let body = result.body.collect().await.expect("bytes=-96 body").into_bytes();
|
||||
assert_eq!(body.len(), 96, "bytes=-96 body length");
|
||||
assert_eq!(&body[..], &content[file_size - 96..], "bytes=-96 body content");
|
||||
|
||||
// bytes=-1 — last byte
|
||||
let result = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.range("bytes=-1")
|
||||
.send()
|
||||
.await
|
||||
.expect("bytes=-1 should succeed");
|
||||
let body = result.body.collect().await.expect("bytes=-1 body").into_bytes();
|
||||
assert_eq!(body.len(), 1, "bytes=-1 body length");
|
||||
assert_eq!(body[0], content[file_size - 1], "bytes=-1 body content");
|
||||
|
||||
// bytes=0-7 — absolute range (regression guard)
|
||||
let result = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.range("bytes=0-7")
|
||||
.send()
|
||||
.await
|
||||
.expect("bytes=0-7 should succeed");
|
||||
let body = result.body.collect().await.expect("bytes=0-7 body").into_bytes();
|
||||
assert_eq!(body.len(), 8, "bytes=0-7 body length");
|
||||
assert_eq!(&body[..], &content[0..8], "bytes=0-7 body content");
|
||||
|
||||
// Equivalent absolute range for the same last-8-bytes window
|
||||
let start = file_size - 8;
|
||||
let end = file_size - 1;
|
||||
let range = format!("bytes={start}-{end}");
|
||||
let result = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.range(&range)
|
||||
.send()
|
||||
.await
|
||||
.expect("absolute tail range should succeed");
|
||||
let body = result.body.collect().await.expect("absolute tail body").into_bytes();
|
||||
assert_eq!(body.len(), 8, "absolute tail body length");
|
||||
assert_eq!(&body[..], &content[start..], "absolute tail body content");
|
||||
}
|
||||
}
|
||||
@@ -70,7 +70,6 @@ reed-solomon-erasure = { workspace = true }
|
||||
reed-solomon-simd = { workspace = true }
|
||||
lazy_static.workspace = true
|
||||
rustfs-lock.workspace = true
|
||||
rustfs-io-core.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
regex = { workspace = true }
|
||||
path-absolutize = { workspace = true }
|
||||
|
||||
@@ -32,10 +32,6 @@
|
||||
|
||||
For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
|
||||
|
||||
## 📈 Benchmarks
|
||||
|
||||
ECStore ships Criterion benchmarks under [`crates/ecstore/benches/`](./benches/) for the remaining erasure-code and comparison workloads. Re-run benchmarks on your target machine before treating any local measurement as a regression baseline.
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
@@ -21,8 +21,6 @@ use std::time::Instant;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::debug;
|
||||
|
||||
const BITROT_READ_OPERATION: &str = "bitrot_read";
|
||||
|
||||
/// Create a BitrotReader from either inline data or disk file stream
|
||||
///
|
||||
/// # Parameters
|
||||
@@ -67,38 +65,20 @@ pub async fn create_bitrot_reader(
|
||||
} else if let Some(disk) = disk {
|
||||
// Read from disk
|
||||
if use_zero_copy {
|
||||
if !disk.is_local() {
|
||||
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
|
||||
rustfs_io_metrics::record_io_fallback(
|
||||
rustfs_io_metrics::IoStage::ReadSetup,
|
||||
rustfs_io_metrics::FallbackReason::NonLocalBackend,
|
||||
);
|
||||
|
||||
let rd = disk.read_file_stream(bucket, path, offset, length).await?;
|
||||
let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
|
||||
return Ok(Some(reader));
|
||||
}
|
||||
|
||||
// Try zero-copy read first (uses mmap on Unix)
|
||||
let start = Instant::now();
|
||||
match disk.read_file_zero_copy(bucket, path, offset, length).await {
|
||||
Ok(bytes) => {
|
||||
let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Fast);
|
||||
// `read_file_zero_copy()` returns a shared `Bytes` view, which preserves the
|
||||
// mmap-backed fast path without exposing chunk-native GET internals.
|
||||
rustfs_io_metrics::record_io_copy_mode(
|
||||
BITROT_READ_OPERATION,
|
||||
rustfs_io_metrics::CopyMode::SharedBytes,
|
||||
bytes.len(),
|
||||
);
|
||||
// Record zero-copy metrics
|
||||
rustfs_io_metrics::record_zero_copy_read(bytes.len(), duration_ms);
|
||||
|
||||
// Log successful zero-copy read
|
||||
debug!(
|
||||
size = bytes.len(),
|
||||
duration_ms,
|
||||
path = %path,
|
||||
"bitrot_fast_read_success"
|
||||
"zero_copy_read_success"
|
||||
);
|
||||
|
||||
// Wrap Bytes in Cursor for AsyncRead
|
||||
@@ -113,16 +93,14 @@ pub async fn create_bitrot_reader(
|
||||
Ok(Some(reader))
|
||||
}
|
||||
Err(e) => {
|
||||
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
|
||||
rustfs_io_metrics::record_io_fallback(
|
||||
rustfs_io_metrics::IoStage::ReadSetup,
|
||||
rustfs_io_metrics::FallbackReason::Unknown,
|
||||
);
|
||||
// Record zero-copy fallback
|
||||
rustfs_io_metrics::record_zero_copy_fallback(&format!("{:?}", e));
|
||||
|
||||
// Log zero-copy fallback
|
||||
debug!(
|
||||
reason = %e,
|
||||
reason = %format!("{:?}", e),
|
||||
path = %path,
|
||||
"bitrot_fast_read_fallback"
|
||||
"zero_copy_fallback"
|
||||
);
|
||||
|
||||
// Fall back to regular stream read on error
|
||||
@@ -139,7 +117,6 @@ pub async fn create_bitrot_reader(
|
||||
}
|
||||
}
|
||||
} else {
|
||||
rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
|
||||
// Use regular stream read
|
||||
match disk.read_file_stream(bucket, path, offset, length).await {
|
||||
Ok(rd) => {
|
||||
|
||||
@@ -24,7 +24,7 @@ use rustfs_utils::http::headers::{
|
||||
AMZ_STORAGE_CLASS,
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_CRC, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS,
|
||||
AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS,
|
||||
SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, has_internal_suffix, insert_bytes,
|
||||
is_internal_key,
|
||||
};
|
||||
@@ -230,10 +230,6 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(checksum) = fi.checksum.as_ref() {
|
||||
insert_bytes(&mut obj.meta_sys, SUFFIX_CRC, checksum.to_vec());
|
||||
}
|
||||
|
||||
if let Some(mod_time) = fi.mod_time {
|
||||
obj.mod_time = Some(mod_time);
|
||||
}
|
||||
|
||||
@@ -381,7 +381,7 @@ where
|
||||
let hash_reader = HashReader::from_stream(buf_reader, content_length, content_length, None, None, false)
|
||||
.map_err(|e| sanitize_storage_error("Hash reader creation", e))?;
|
||||
|
||||
// 15. Hand the hash reader to the chunk-native PUT data wrapper
|
||||
// 15. Wrap in PutObjReader as expected by storage layer
|
||||
let mut put_reader = PutObjReader::new(hash_reader);
|
||||
|
||||
// 16. Upload object to storage
|
||||
@@ -464,7 +464,7 @@ where
|
||||
let hash_reader = HashReader::from_stream(buf_reader, content_length, content_length, None, None, false)
|
||||
.map_err(|e| sanitize_storage_error("Hash reader creation", e))?;
|
||||
|
||||
// Hand the hash reader to the chunk-native PUT data wrapper
|
||||
// Wrap in PutObjReader
|
||||
let mut put_reader = PutObjReader::new(hash_reader);
|
||||
|
||||
// Upload object to storage
|
||||
|
||||
@@ -29,21 +29,11 @@ pub const RUSTFS_MULTIPART_CHECKSUM: &str = "x-rustfs-multipart-checksum";
|
||||
/// RustFS multipart checksum type metadata key
|
||||
pub const RUSTFS_MULTIPART_CHECKSUM_TYPE: &str = "x-rustfs-multipart-checksum-type";
|
||||
|
||||
const AMZ_CHECKSUM_ALGORITHM: &str = "x-amz-checksum-algorithm";
|
||||
const AMZ_SDK_CHECKSUM_ALGORITHM: &str = "x-amz-sdk-checksum-algorithm";
|
||||
|
||||
/// Checksum type enumeration with flags
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub struct ChecksumType(pub u32);
|
||||
|
||||
impl ChecksumType {
|
||||
fn algorithm_from_headers(headers: &HeaderMap) -> Option<&str> {
|
||||
headers
|
||||
.get(AMZ_CHECKSUM_ALGORITHM)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.or_else(|| headers.get(AMZ_SDK_CHECKSUM_ALGORITHM).and_then(|v| v.to_str().ok()))
|
||||
}
|
||||
|
||||
/// Checksum will be sent in trailing header
|
||||
pub const TRAILING: ChecksumType = ChecksumType(1 << 0);
|
||||
|
||||
@@ -166,7 +156,10 @@ impl ChecksumType {
|
||||
|
||||
pub fn from_header(headers: &HeaderMap) -> Self {
|
||||
Self::from_string_with_obj_type(
|
||||
Self::algorithm_from_headers(headers).unwrap_or(""),
|
||||
headers
|
||||
.get("x-amz-checksum-algorithm")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or(""),
|
||||
headers.get("x-amz-checksum-type").and_then(|v| v.to_str().ok()).unwrap_or(""),
|
||||
)
|
||||
}
|
||||
@@ -580,7 +573,7 @@ pub fn get_content_checksum(headers: &HeaderMap) -> Result<Option<Checksum>, std
|
||||
fn get_content_checksum_direct(headers: &HeaderMap) -> (ChecksumType, String) {
|
||||
let mut checksum_type = ChecksumType::NONE;
|
||||
|
||||
if let Some(alg) = ChecksumType::algorithm_from_headers(headers) {
|
||||
if let Some(alg) = headers.get("x-amz-checksum-algorithm").and_then(|v| v.to_str().ok()) {
|
||||
checksum_type = ChecksumType::from_string_with_obj_type(
|
||||
alg,
|
||||
headers.get("x-amz-checksum-type").and_then(|s| s.to_str().ok()).unwrap_or(""),
|
||||
@@ -1139,35 +1132,6 @@ fn crc64_combine(poly: u64, crc1: u64, crc2: u64, len2: i64) -> u64 {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Checksum, ChecksumType};
|
||||
use http::HeaderMap;
|
||||
|
||||
#[test]
|
||||
fn algorithm_from_headers_parses_amz_checksum_algorithm() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-checksum-algorithm", "SHA256".parse().expect("valid header value"));
|
||||
assert_eq!(ChecksumType::algorithm_from_headers(&headers), Some("SHA256"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn algorithm_from_headers_falls_back_to_sdk_header() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-sdk-checksum-algorithm", "CRC32C".parse().expect("valid header value"));
|
||||
assert_eq!(ChecksumType::algorithm_from_headers(&headers), Some("CRC32C"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn algorithm_from_headers_prefers_amz_over_sdk() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-checksum-algorithm", "SHA256".parse().expect("valid header value"));
|
||||
headers.insert("x-amz-sdk-checksum-algorithm", "CRC32C".parse().expect("valid header value"));
|
||||
assert_eq!(ChecksumType::algorithm_from_headers(&headers), Some("SHA256"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn algorithm_from_headers_returns_none_when_missing() {
|
||||
let headers = HeaderMap::new();
|
||||
assert_eq!(ChecksumType::algorithm_from_headers(&headers), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crc64_nvme_add_part_matches_full_object_checksum() {
|
||||
|
||||
+15
-13
@@ -257,10 +257,23 @@ impl From<StorageError> for ApiError {
|
||||
|
||||
impl From<std::io::Error> for ApiError {
|
||||
fn from(err: std::io::Error) -> Self {
|
||||
// Check if the error is a ChecksumMismatch (BadDigest)
|
||||
// Check if the inner error is a StorageError (e.g. InvalidRangeSpec wrapped by object-io)
|
||||
if let Some(inner) = err.get_ref() {
|
||||
if let Some(storage_error) = inner.downcast_ref::<StorageError>() {
|
||||
return storage_error.clone().into();
|
||||
let code = match storage_error {
|
||||
StorageError::InvalidRangeSpec(_) => S3ErrorCode::InvalidRange,
|
||||
_ => S3ErrorCode::InternalError,
|
||||
};
|
||||
let message = if code == S3ErrorCode::InternalError {
|
||||
storage_error.to_string()
|
||||
} else {
|
||||
ApiError::error_code_to_message(&code)
|
||||
};
|
||||
return ApiError {
|
||||
code,
|
||||
message,
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
if inner.downcast_ref::<rustfs_rio::ChecksumMismatch>().is_some() {
|
||||
return ApiError {
|
||||
@@ -555,15 +568,4 @@ mod tests {
|
||||
// This is expected because ApiError is not a typical Error implementation
|
||||
assert!(error.source().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_api_error_from_io_error_unwraps_invalid_range_storage_error() {
|
||||
let io_error = std::io::Error::from(StorageError::InvalidRangeSpec("range invalid".to_string()));
|
||||
|
||||
let api_error: ApiError = io_error.into();
|
||||
|
||||
assert_eq!(api_error.code, S3ErrorCode::InvalidRange);
|
||||
assert_eq!(api_error.message, ApiError::error_code_to_message(&S3ErrorCode::InvalidRange));
|
||||
assert!(api_error.source.is_some());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
RUSTFS_BIN="${ROOT_DIR}/target/debug/rustfs"
|
||||
WARP_BIN="${WARP_BIN:-warp}"
|
||||
MC_BIN="${MC_BIN:-mc}"
|
||||
|
||||
PORT="${RUSTFS_BENCH_PORT:-9900}"
|
||||
CONSOLE_PORT="${RUSTFS_BENCH_CONSOLE_PORT:-9901}"
|
||||
HOST="127.0.0.1:${PORT}"
|
||||
ACCESS_KEY="${RUSTFS_BENCH_ACCESS_KEY:-rustfsadmin}"
|
||||
SECRET_KEY="${RUSTFS_BENCH_SECRET_KEY:-rustfsadmin}"
|
||||
BENCH_ROOT="${ROOT_DIR}/target/benchmarks"
|
||||
RUN_ID="${RUSTFS_BENCH_RUN_ID:-small-put-$(date +%Y%m%d-%H%M%S)}"
|
||||
RUN_DIR="${BENCH_ROOT}/${RUN_ID}"
|
||||
VOLUME_DIR="${RUN_DIR}/volumes"
|
||||
LOG_DIR="${RUN_DIR}/logs"
|
||||
MC_CONFIG_DIR="${RUN_DIR}/mc-config"
|
||||
RESULTS_MD="${RUN_DIR}/RESULTS.md"
|
||||
SERVER_LOG="${LOG_DIR}/rustfs.log"
|
||||
WARP_BUCKET="${RUSTFS_BENCH_BUCKET:-warp-small-put-benchmark}"
|
||||
WARP_DURATION="${RUSTFS_BENCH_DURATION:-20s}"
|
||||
WARP_CONCURRENCY="${RUSTFS_BENCH_CONCURRENCY:-32}"
|
||||
|
||||
mkdir -p "${VOLUME_DIR}" "${LOG_DIR}" "${MC_CONFIG_DIR}"
|
||||
mkdir -p "${VOLUME_DIR}"/disk{1..4}
|
||||
|
||||
# Verify mc (MinIO Client) is available
|
||||
if ! command -v "${MC_BIN}" >/dev/null 2>&1; then
|
||||
echo "Error: 'mc' (MinIO Client) not found. MC_BIN=${MC_BIN}" >&2
|
||||
echo "" >&2
|
||||
echo "Install mc with one of the following:" >&2
|
||||
echo " macOS (Homebrew): brew install minio/stable/mc" >&2
|
||||
echo " Go: go install github.com/minio/mc@latest" >&2
|
||||
echo " Linux (amd64): curl -sSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2
|
||||
echo " Linux (arm64): curl -sSL https://dl.min.io/client/mc/release/linux-arm64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2
|
||||
echo "" >&2
|
||||
echo "Or set MC_BIN=/path/to/mc before running this script." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify warp is available
|
||||
if ! command -v "${WARP_BIN}" >/dev/null 2>&1; then
|
||||
echo "Error: 'warp' benchmark tool not found. WARP_BIN=${WARP_BIN}" >&2
|
||||
echo "" >&2
|
||||
echo "Install warp with one of the following:" >&2
|
||||
echo " Go: go install github.com/minio/warp@latest" >&2
|
||||
echo " Linux (amd64): curl -sSL https://github.com/minio/warp/releases/latest/download/warp_Linux_x86_64.tar.gz | tar -xz -C /usr/local/bin warp" >&2
|
||||
echo " macOS (amd64): curl -sSL https://github.com/minio/warp/releases/latest/download/warp_Darwin_x86_64.tar.gz | tar -xz -C /usr/local/bin warp" >&2
|
||||
echo " macOS (arm64): curl -sSL https://github.com/minio/warp/releases/latest/download/warp_Darwin_arm64.tar.gz | tar -xz -C /usr/local/bin warp" >&2
|
||||
echo "" >&2
|
||||
echo "Or set WARP_BIN=/path/to/warp before running this script." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SIZES=(
|
||||
"4KiB"
|
||||
"16KiB"
|
||||
"64KiB"
|
||||
"256KiB"
|
||||
"1MiB"
|
||||
)
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" >/dev/null 2>&1; then
|
||||
kill "${SERVER_PID}" >/dev/null 2>&1 || true
|
||||
wait "${SERVER_PID}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "Starting local RustFS benchmark server..."
|
||||
(
|
||||
cd "${ROOT_DIR}"
|
||||
export RUST_LOG="${RUST_LOG:-warn}"
|
||||
export RUSTFS_VOLUMES="${VOLUME_DIR}/disk{1...4}"
|
||||
export RUSTFS_ADDRESS=":${PORT}"
|
||||
export RUSTFS_CONSOLE_ENABLE=false
|
||||
export RUSTFS_CONSOLE_ADDRESS=":${CONSOLE_PORT}"
|
||||
export RUSTFS_ROOT_USER="${ACCESS_KEY}"
|
||||
export RUSTFS_ROOT_PASSWORD="${SECRET_KEY}"
|
||||
export RUSTFS_OBJECT_CACHE_ENABLE=false
|
||||
"${RUSTFS_BIN}" server >"${SERVER_LOG}" 2>&1
|
||||
) &
|
||||
SERVER_PID=$!
|
||||
|
||||
echo "Waiting for RustFS server to become ready..."
|
||||
for _ in $(seq 1 60); do
|
||||
if curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 || curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if ! curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 && ! curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then
|
||||
echo "RustFS did not become ready in time. See ${SERVER_LOG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Preparing mc alias..."
|
||||
MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" alias set bench "http://${HOST}" "${ACCESS_KEY}" "${SECRET_KEY}" >/dev/null
|
||||
MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" mb --ignore-existing "bench/${WARP_BUCKET}" >/dev/null
|
||||
|
||||
{
|
||||
echo "# Small PUT Benchmark Results"
|
||||
echo
|
||||
echo "- Run ID: \`${RUN_ID}\`"
|
||||
echo "- Host: \`${HOST}\`"
|
||||
echo "- Duration per size: \`${WARP_DURATION}\`"
|
||||
echo "- Concurrency: \`${WARP_CONCURRENCY}\`"
|
||||
echo "- Bucket: \`${WARP_BUCKET}\`"
|
||||
echo
|
||||
} >"${RESULTS_MD}"
|
||||
|
||||
for size in "${SIZES[@]}"; do
|
||||
echo
|
||||
echo "Running warp PUT benchmark for size ${size}..."
|
||||
size_slug="$(echo "${size}" | tr '[:upper:]' '[:lower:]')"
|
||||
OUT_FILE="${LOG_DIR}/warp-put-${size}.log"
|
||||
BENCHDATA_FILE="${RUN_DIR}/warp-put-${size}.csv.zst"
|
||||
"${WARP_BIN}" put \
|
||||
--no-color \
|
||||
--host "${HOST}" \
|
||||
--access-key "${ACCESS_KEY}" \
|
||||
--secret-key "${SECRET_KEY}" \
|
||||
--bucket "${WARP_BUCKET}" \
|
||||
--obj.size "${size}" \
|
||||
--duration "${WARP_DURATION}" \
|
||||
--concurrent "${WARP_CONCURRENCY}" \
|
||||
--prefix "${RUN_ID}/${size_slug}" \
|
||||
--disable-multipart \
|
||||
--noclear \
|
||||
--benchdata "${BENCHDATA_FILE}" \
|
||||
--insecure \
|
||||
>"${OUT_FILE}" 2>&1
|
||||
|
||||
{
|
||||
echo "## ${size}"
|
||||
echo
|
||||
echo '```text'
|
||||
tail -n 20 "${OUT_FILE}"
|
||||
echo '```'
|
||||
echo
|
||||
} >>"${RESULTS_MD}"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Benchmark completed."
|
||||
echo "Results: ${RESULTS_MD}"
|
||||
echo "Server log: ${SERVER_LOG}"
|
||||
@@ -1,300 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
RUSTFS_BIN="${RUSTFS_BIN:-${ROOT_DIR}/target/debug/rustfs}"
|
||||
MC_BIN="${MC_BIN:-mc}"
|
||||
|
||||
PORT="${RUSTFS_BENCH_PORT:-9910}"
|
||||
CONSOLE_PORT="${RUSTFS_BENCH_CONSOLE_PORT:-9911}"
|
||||
HOST="127.0.0.1:${PORT}"
|
||||
ACCESS_KEY="${RUSTFS_BENCH_ACCESS_KEY:-rustfsadmin}"
|
||||
SECRET_KEY="${RUSTFS_BENCH_SECRET_KEY:-rustfsadmin}"
|
||||
|
||||
OPS_PER_SIZE="${RUSTFS_BENCH_OPS_PER_SIZE:-0}"
|
||||
BENCH_SECONDS="${RUSTFS_BENCH_SECONDS:-10}"
|
||||
CONCURRENCY="${RUSTFS_BENCH_CONCURRENCY:-32}"
|
||||
PUT_TIMEOUT_SECS="${RUSTFS_BENCH_PUT_TIMEOUT_SECS:-15}"
|
||||
RUN_ID="${RUSTFS_BENCH_RUN_ID:-small-put-mc-$(date +%Y%m%d-%H%M%S)}"
|
||||
BENCH_ROOT="${ROOT_DIR}/target/benchmarks"
|
||||
RUN_DIR="${BENCH_ROOT}/${RUN_ID}"
|
||||
VOLUME_DIR="${RUN_DIR}/volumes"
|
||||
LOG_DIR="${RUN_DIR}/logs"
|
||||
FILE_DIR="${RUN_DIR}/files"
|
||||
MC_CONFIG_DIR="${RUN_DIR}/mc-config"
|
||||
RESULTS_MD="${RUN_DIR}/RESULTS.md"
|
||||
SERVER_LOG="${LOG_DIR}/rustfs.log"
|
||||
BUCKET="${RUSTFS_BENCH_BUCKET:-small-put-benchmark}"
|
||||
|
||||
# Verify mc (MinIO Client) is available
|
||||
if ! command -v "${MC_BIN}" >/dev/null 2>&1; then
|
||||
echo "Error: 'mc' (MinIO Client) not found. MC_BIN=${MC_BIN}" >&2
|
||||
echo "" >&2
|
||||
echo "Install mc with one of the following:" >&2
|
||||
echo " macOS (Homebrew): brew install minio/stable/mc" >&2
|
||||
echo " Go: go install github.com/minio/mc@latest" >&2
|
||||
echo " Linux (amd64): curl -sSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2
|
||||
echo " Linux (arm64): curl -sSL https://dl.min.io/client/mc/release/linux-arm64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2
|
||||
echo "" >&2
|
||||
echo "Or set MC_BIN=/path/to/mc before running this script." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${VOLUME_DIR}" "${LOG_DIR}" "${FILE_DIR}" "${MC_CONFIG_DIR}"
|
||||
mkdir -p "${VOLUME_DIR}"/disk{1..4}
|
||||
|
||||
SIZES=(
|
||||
"4KiB"
|
||||
"16KiB"
|
||||
"64KiB"
|
||||
"256KiB"
|
||||
"1MiB"
|
||||
)
|
||||
|
||||
size_to_bytes() {
|
||||
case "$1" in
|
||||
4KiB) echo 4096 ;;
|
||||
16KiB) echo 16384 ;;
|
||||
64KiB) echo 65536 ;;
|
||||
256KiB) echo 262144 ;;
|
||||
1MiB) echo 1048576 ;;
|
||||
*)
|
||||
echo "unsupported size: $1" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
now_secs() {
|
||||
perl -MTime::HiRes=time -e 'printf "%.6f\n", time'
|
||||
}
|
||||
|
||||
prepare_sample_file() {
|
||||
local size_label="$1"
|
||||
local bytes="$2"
|
||||
local file_path="${FILE_DIR}/${size_label}.bin"
|
||||
if [[ ! -f "${file_path}" ]]; then
|
||||
dd if=/dev/zero of="${file_path}" bs="${bytes}" count=1 status=none
|
||||
fi
|
||||
echo "${file_path}"
|
||||
}
|
||||
|
||||
run_mc_put_with_timeout() {
|
||||
local sample_file="$1"
|
||||
local key="$2"
|
||||
perl -e '
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
my $timeout = shift @ARGV;
|
||||
my $pid = fork();
|
||||
die "fork failed: $!" unless defined $pid;
|
||||
|
||||
if ($pid == 0) {
|
||||
exec @ARGV;
|
||||
die "exec failed: $!";
|
||||
}
|
||||
|
||||
my $timed_out = 0;
|
||||
local $SIG{ALRM} = sub {
|
||||
$timed_out = 1;
|
||||
kill "TERM", $pid;
|
||||
select undef, undef, undef, 0.2;
|
||||
kill "KILL", $pid;
|
||||
};
|
||||
|
||||
alarm($timeout);
|
||||
my $waited = waitpid($pid, 0);
|
||||
alarm(0);
|
||||
|
||||
exit(1) if $timed_out;
|
||||
exit(($waited == $pid && $? == 0) ? 0 : 1);
|
||||
' "${PUT_TIMEOUT_SECS}" env MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" put --quiet --disable-multipart "${sample_file}" "${key}" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
export -f run_mc_put_with_timeout
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" >/dev/null 2>&1; then
|
||||
kill "${SERVER_PID}" >/dev/null 2>&1 || true
|
||||
wait "${SERVER_PID}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "Starting local RustFS benchmark server..."
|
||||
(
|
||||
cd "${ROOT_DIR}"
|
||||
export RUST_LOG="${RUST_LOG:-warn}"
|
||||
export RUSTFS_VOLUMES="${VOLUME_DIR}/disk{1...4}"
|
||||
export RUSTFS_ADDRESS=":${PORT}"
|
||||
export RUSTFS_CONSOLE_ENABLE=false
|
||||
export RUSTFS_CONSOLE_ADDRESS=":${CONSOLE_PORT}"
|
||||
export RUSTFS_ROOT_USER="${ACCESS_KEY}"
|
||||
export RUSTFS_ROOT_PASSWORD="${SECRET_KEY}"
|
||||
export RUSTFS_OBJECT_CACHE_ENABLE=false
|
||||
"${RUSTFS_BIN}" server >"${SERVER_LOG}" 2>&1
|
||||
) &
|
||||
SERVER_PID=$!
|
||||
|
||||
echo "Waiting for RustFS server to become ready..."
|
||||
for _ in $(seq 1 60); do
|
||||
if curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 || curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if ! curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 && ! curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then
|
||||
echo "RustFS did not become ready in time. See ${SERVER_LOG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Preparing mc alias..."
|
||||
MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" alias set bench "http://${HOST}" "${ACCESS_KEY}" "${SECRET_KEY}" >/dev/null
|
||||
MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" mb --ignore-existing "bench/${BUCKET}" >/dev/null
|
||||
|
||||
{
|
||||
echo "# Small PUT Benchmark Results"
|
||||
echo
|
||||
echo "- Run ID: \`${RUN_ID}\`"
|
||||
echo "- Host: \`${HOST}\`"
|
||||
echo "- Bucket: \`${BUCKET}\`"
|
||||
if [[ "${OPS_PER_SIZE}" -gt 0 ]]; then
|
||||
echo "- Ops per size: \`${OPS_PER_SIZE}\`"
|
||||
else
|
||||
echo "- Duration per size: \`${BENCH_SECONDS}s\`"
|
||||
fi
|
||||
echo "- Concurrency: \`${CONCURRENCY}\`"
|
||||
echo "- Per request timeout: \`${PUT_TIMEOUT_SECS}s\`"
|
||||
echo
|
||||
} >"${RESULTS_MD}"
|
||||
|
||||
for size_label in "${SIZES[@]}"; do
|
||||
bytes="$(size_to_bytes "${size_label}")"
|
||||
sample_file="$(prepare_sample_file "${size_label}" "${bytes}")"
|
||||
raw_json="${RUN_DIR}/${size_label}.jsonl"
|
||||
prefix="$(echo "${size_label}" | tr '[:upper:]' '[:lower:]')"
|
||||
worker_dir="${RUN_DIR}/${size_label}-workers"
|
||||
mkdir -p "${worker_dir}"
|
||||
|
||||
echo "Running mc PUT benchmark for size ${size_label}..."
|
||||
batch_start="$(now_secs)"
|
||||
if [[ "${OPS_PER_SIZE}" -gt 0 ]]; then
|
||||
seq 1 "${OPS_PER_SIZE}" | xargs -n1 -P "${CONCURRENCY}" -I{} \
|
||||
env MC_CONFIG_DIR="${MC_CONFIG_DIR}" MC_BIN="${MC_BIN}" PUT_TIMEOUT_SECS="${PUT_TIMEOUT_SECS}" SAMPLE_FILE="${sample_file}" BUCKET="${BUCKET}" PREFIX="${prefix}" /bin/bash -lc '
|
||||
idx="$1"
|
||||
start=$(perl -MTime::HiRes=time -e '"'"'printf "%.6f\n", time'"'"')
|
||||
if run_mc_put_with_timeout "${SAMPLE_FILE}" "bench/${BUCKET}/${PREFIX}/obj-${idx}.bin"; then
|
||||
finish=$(perl -MTime::HiRes=time -e '"'"'printf "%.6f\n", time'"'"')
|
||||
perl -e '"'"'
|
||||
use strict;
|
||||
use warnings;
|
||||
my ($idx, $start, $finish) = @ARGV;
|
||||
my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000);
|
||||
print "{\"idx\":${idx},\"ok\":true,\"duration_ms\":${duration_ms}}\n";
|
||||
'"'"' -- "${idx}" "${start}" "${finish}"
|
||||
else
|
||||
finish=$(perl -MTime::HiRes=time -e '"'"'printf "%.6f\n", time'"'"')
|
||||
perl -e '"'"'
|
||||
use strict;
|
||||
use warnings;
|
||||
my ($idx, $start, $finish) = @ARGV;
|
||||
my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000);
|
||||
print "{\"idx\":${idx},\"ok\":false,\"duration_ms\":${duration_ms}}\n";
|
||||
'"'"' -- "${idx}" "${start}" "${finish}"
|
||||
fi
|
||||
' _ {} >"${raw_json}"
|
||||
else
|
||||
bench_deadline="$(perl -e 'use strict; use warnings; my ($start, $secs) = @ARGV; printf "%.6f", $start + $secs;' -- "${batch_start}" "${BENCH_SECONDS}")"
|
||||
pids=()
|
||||
for worker in $(seq 1 "${CONCURRENCY}"); do
|
||||
(
|
||||
out_file="${worker_dir}/worker-${worker}.jsonl"
|
||||
idx=0
|
||||
while true; do
|
||||
now="$(now_secs)"
|
||||
if perl -e 'use strict; use warnings; exit(($ARGV[0] >= $ARGV[1]) ? 0 : 1)' -- "${now}" "${bench_deadline}"; then
|
||||
break
|
||||
fi
|
||||
start="${now}"
|
||||
key="bench/${BUCKET}/${prefix}/worker-${worker}/obj-${idx}.bin"
|
||||
if run_mc_put_with_timeout "${sample_file}" "${key}"; then
|
||||
finish="$(now_secs)"
|
||||
perl -e '
|
||||
use strict;
|
||||
use warnings;
|
||||
my ($idx, $start, $finish) = @ARGV;
|
||||
my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000);
|
||||
print "{\"idx\":${idx},\"ok\":true,\"duration_ms\":${duration_ms}}\n";
|
||||
' -- "${idx}" "${start}" "${finish}" >>"${out_file}"
|
||||
else
|
||||
finish="$(now_secs)"
|
||||
perl -e '
|
||||
use strict;
|
||||
use warnings;
|
||||
my ($idx, $start, $finish) = @ARGV;
|
||||
my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000);
|
||||
print "{\"idx\":${idx},\"ok\":false,\"duration_ms\":${duration_ms}}\n";
|
||||
' -- "${idx}" "${start}" "${finish}" >>"${out_file}"
|
||||
fi
|
||||
idx=$((idx + 1))
|
||||
done
|
||||
) &
|
||||
pids+=($!)
|
||||
done
|
||||
for pid in "${pids[@]}"; do
|
||||
wait "${pid}"
|
||||
done
|
||||
cat "${worker_dir}"/worker-*.jsonl >"${raw_json}"
|
||||
fi
|
||||
batch_finish="$(now_secs)"
|
||||
|
||||
jq_summary="$(jq -s '
|
||||
def pct(p):
|
||||
if length == 0 then null
|
||||
else (sort_by(.duration_ms) | .[((length - 1) * p | floor)].duration_ms)
|
||||
end;
|
||||
{
|
||||
total: length,
|
||||
succeeded: (map(select(.ok == true)) | length),
|
||||
failed: (map(select(.ok == false)) | length),
|
||||
avg_ms: (if length == 0 then null else (map(.duration_ms) | add / length) end),
|
||||
p50_ms: pct(0.50),
|
||||
p90_ms: pct(0.90),
|
||||
p99_ms: pct(0.99)
|
||||
}
|
||||
' "${raw_json}")"
|
||||
|
||||
success_count="$(echo "${jq_summary}" | jq -r '.succeeded')"
|
||||
avg_ms="$(echo "${jq_summary}" | jq -r '.avg_ms')"
|
||||
p50_ms="$(echo "${jq_summary}" | jq -r '.p50_ms')"
|
||||
p90_ms="$(echo "${jq_summary}" | jq -r '.p90_ms')"
|
||||
p99_ms="$(echo "${jq_summary}" | jq -r '.p99_ms')"
|
||||
failed_count="$(echo "${jq_summary}" | jq -r '.failed')"
|
||||
|
||||
wall_secs="$(perl -e 'use strict; use warnings; my ($start, $finish) = @ARGV; printf "%.6f", ($finish - $start);' -- "${batch_start}" "${batch_finish}")"
|
||||
mib_per_sec="$(perl -e 'use strict; use warnings; my ($bytes, $count, $secs) = @ARGV; printf "%.3f", (($bytes * $count) / (1024 * 1024)) / $secs;' -- "${bytes}" "${success_count}" "${wall_secs}")"
|
||||
obj_per_sec="$(perl -e 'use strict; use warnings; my ($count, $secs) = @ARGV; printf "%.3f", $count / $secs;' -- "${success_count}" "${wall_secs}")"
|
||||
|
||||
{
|
||||
echo "## ${size_label}"
|
||||
echo
|
||||
echo "- Successful PUTs: \`${success_count}\`"
|
||||
echo "- Failed PUTs: \`${failed_count}\`"
|
||||
echo "- Wall time: \`${wall_secs}s\`"
|
||||
echo "- Throughput: \`${mib_per_sec} MiB/s\`"
|
||||
echo "- Object rate: \`${obj_per_sec} obj/s\`"
|
||||
echo "- Avg latency: \`${avg_ms} ms\`"
|
||||
echo "- p50 latency: \`${p50_ms} ms\`"
|
||||
echo "- p90 latency: \`${p90_ms} ms\`"
|
||||
echo "- p99 latency: \`${p99_ms} ms\`"
|
||||
echo
|
||||
} >>"${RESULTS_MD}"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Benchmark completed."
|
||||
echo "Results: ${RESULTS_MD}"
|
||||
echo "Server log: ${SERVER_LOG}"
|
||||
+2
-3
@@ -38,8 +38,7 @@ mkdir -p ./target/volume/test{1..4}
|
||||
|
||||
if [ -z "$RUST_LOG" ]; then
|
||||
export RUST_BACKTRACE=1
|
||||
# export RUST_LOG="info,rustfs=debug,rustfs_ecstore=info,s3s=debug,rustfs_iam=info,rustfs_notify=info"
|
||||
export RUST_LOG="error"
|
||||
export RUST_LOG="info,rustfs=debug,rustfs_ecstore=info,s3s=debug,rustfs_iam=info,rustfs_notify=info"
|
||||
fi
|
||||
|
||||
# export RUSTFS_ERASURE_SET_DRIVE_COUNT=5
|
||||
@@ -72,7 +71,7 @@ export RUSTFS_OBS_SERVICE_NAME=rustfs # Service name
|
||||
export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version
|
||||
export RUSTFS_OBS_ENVIRONMENT=production # Environment name development, staging, production
|
||||
export RUSTFS_OBS_LOGGER_LEVEL=info # Log level, supports trace, debug, info, warn, error
|
||||
export RUSTFS_OBS_LOG_STDOUT_ENABLED=false # Whether to enable local stdout logging
|
||||
export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging
|
||||
export RUSTFS_OBS_LOG_DIRECTORY="$current_dir/deploy/logs" # Log directory
|
||||
export RUSTFS_OBS_LOG_ROTATION_TIME="minutely" # Log rotation time unit, can be "minutely", "hourly", "daily"
|
||||
export RUSTFS_OBS_LOG_KEEP_FILES=10 # Number of log files to keep
|
||||
|
||||
Reference in New Issue
Block a user