diff --git a/Cargo.lock b/Cargo.lock index d4a76393f..c8b42ce34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -772,7 +772,7 @@ dependencies = [ "http 0.2.12", "http 1.4.0", "http-body 1.0.1", - "lru 0.16.3", + "lru 0.16.4", "percent-encoding", "regex-lite", "sha2 0.10.9", @@ -1091,9 +1091,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", "bytes", @@ -1223,9 +1223,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "blake2" @@ -1462,7 +1462,7 @@ dependencies = [ "cfg-if", "cipher 0.5.1", "cpufeatures 0.3.0", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -1998,7 +1998,7 @@ dependencies = [ "cpubits", "ctutils", "num-traits", - "rand_core 0.10.0", + "rand_core 0.10.1", "serdect", "zeroize", ] @@ -2021,7 +2021,7 @@ checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" dependencies = [ "getrandom 0.4.2", "hybrid-array", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -2032,7 +2032,7 @@ checksum = "21f41f23de7d24cdbda7f0c4d9c0351f99a4ceb258ef30e5c1927af8987ffe5a" dependencies = [ "crypto-bigint 0.7.3", "libm", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -2275,7 +2275,7 @@ dependencies = [ "object_store", "parking_lot 0.12.5", "parquet", - "rand 0.9.3", + "rand 0.9.4", "regex", "sqlparser", "tempfile", @@ -2397,7 +2397,7 @@ dependencies = [ "liblzma", "log", "object_store", - "rand 0.9.3", + "rand 0.9.4", "tokio", "tokio-util", "url", @@ -2529,7 +2529,7 @@ dependencies = [ "log", "object_store", "parking_lot 0.12.5", - "rand 0.9.3", + "rand 0.9.4", "tempfile", "url", ] @@ -2595,7 +2595,7 @@ dependencies = [ "md-5 0.10.6", "memchr", "num-traits", - "rand 0.9.3", + "rand 0.9.4", "regex", "sha2 0.10.9", "unicode-segmentation", @@ -2913,7 +2913,7 @@ dependencies = [ "http-body-util", "libc", "log", - "lru 0.16.3", + "lru 0.16.4", "mime_guess", "parking_lot 0.12.5", "percent-encoding", @@ -3064,8 +3064,9 @@ dependencies = [ [[package]] name = "dial9-tokio-telemetry" -version = "0.2.0" -source = "git+https://github.com/dial9-rs/dial9-tokio-telemetry.git?rev=7958b5e6562ff8d3baa944d4316c6ed2d7b73794#7958b5e6562ff8d3baa944d4316c6ed2d7b73794" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c8348045a09a7dee69d000124d42f773f59f85737d3212ae65aa64ac0b4e3e" dependencies = [ "arc-swap", "bon", @@ -3076,7 +3077,6 @@ dependencies = [ "hostname", "libc", "metrique", - "metrique-timesource", "metrique-writer", "pin-project-lite", "serde", @@ -3090,7 +3090,8 @@ dependencies = [ [[package]] name = "dial9-trace-format" version = "0.2.0" -source = "git+https://github.com/dial9-rs/dial9-tokio-telemetry.git?rev=7958b5e6562ff8d3baa944d4316c6ed2d7b73794#7958b5e6562ff8d3baa944d4316c6ed2d7b73794" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e0ee560b05f09bf817602d57644947e31e83c521d4e0277f723a6e64d44f92" dependencies = [ "dial9-trace-format-derive", "serde", @@ -3099,7 +3100,8 @@ dependencies = [ [[package]] name = "dial9-trace-format-derive" version = "0.2.0" -source = "git+https://github.com/dial9-rs/dial9-tokio-telemetry.git?rev=7958b5e6562ff8d3baa944d4316c6ed2d7b73794#7958b5e6562ff8d3baa944d4316c6ed2d7b73794" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dbbd8126d4d6613931317cfe2a7275c1cd487e41c961e42456ab5f956570030" dependencies = [ "proc-macro2", "quote", @@ -3547,7 +3549,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "rustc_version", ] @@ -3769,7 +3771,7 @@ dependencies = [ "js-sys", "libc", "r-efi 6.0.0", - "rand_core 0.10.0", + "rand_core 0.10.1", "wasip2", "wasip3", "wasm-bindgen", @@ -3820,9 +3822,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "google-cloud-auth" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27e658fc9f8b6bdf9a5c816ebca6dd6bcd32f8550e5c6580652b2c0eac1980f6" +checksum = "54a26c047222f874ea87177368ad07c65a9f66534ad3a3f9401f1322c802ccac" dependencies = [ "async-trait", "aws-lc-rs", @@ -3831,7 +3833,7 @@ dependencies = [ "chrono", "google-cloud-gax", "hex", - "hmac 0.12.1", + "hmac 0.13.0", "http 1.4.0", "jsonwebtoken", "reqwest 0.13.2", @@ -3840,7 +3842,7 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", - "sha2 0.10.9", + "sha2 0.11.0", "thiserror 2.0.18", "time", "tokio", @@ -3849,9 +3851,9 @@ dependencies = [ [[package]] name = "google-cloud-gax" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "505f3e57fbb875646b25c3ccc859c6446bfa411e1958d267bab288980e5afa19" +checksum = "7dc387965cc2efc28d73896d6707125815c16792c23c33a0c67794f3d6e31cc8" dependencies = [ "base64 0.22.1", "bytes", @@ -3869,9 +3871,9 @@ dependencies = [ [[package]] name = "google-cloud-gax-internal" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65d462b4fcee5f495bfb58edbf4a9250c230a1079d410bdcb8505bc5f713dcee" +checksum = "9529b797c68dc5c9d37effe7faa5d96d683a00e728287a9934bf6d3a86b58706" dependencies = [ "bytes", "futures", @@ -3879,6 +3881,7 @@ dependencies = [ "google-cloud-gax", "google-cloud-rpc", "google-cloud-wkt", + "h2", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -3907,9 +3910,9 @@ dependencies = [ [[package]] name = "google-cloud-iam-v1" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ce870ac18f3e0a474000cd57eab8bf3c64af8b5ed820468df8612182709c9a" +checksum = "103d11cc828e7f33759c4dc6618a98415786fabba268147028e607c4fe1199aa" dependencies = [ "async-trait", "bytes", @@ -3925,9 +3928,9 @@ dependencies = [ [[package]] name = "google-cloud-longrunning" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebab215997c51f786852840fec8c76174b8a4af96d08e5fc1569742805baab09" +checksum = "fa340b46d452fcd5e701b21dab6ed187454b6f1e4300e0bdb928fceefac847fa" dependencies = [ "async-trait", "bytes", @@ -3943,9 +3946,9 @@ dependencies = [ [[package]] name = "google-cloud-lro" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a4f93a1ec8e6e5448899877ea6021e0f5d06e6b08ccd9b0bd99bc837ca357b" +checksum = "3bfd3fa41649a3cd53675a94988b7d3789f2d8f2eb1b24af237450fe08ae9643" dependencies = [ "google-cloud-gax", "google-cloud-longrunning", @@ -3957,9 +3960,9 @@ dependencies = [ [[package]] name = "google-cloud-rpc" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "691ae06142c69c73bcef2f5c6fa5a6858521aab4cdf1886a6ba70ba1316c7093" +checksum = "7e3b123ea17ff20539fbdf145e6213e0464cc0a30b0d078a68bf90405ef17fb7" dependencies = [ "bytes", "google-cloud-wkt", @@ -3970,9 +3973,9 @@ dependencies = [ [[package]] name = "google-cloud-storage" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f85a4e9a65a2f2c3a1d05c1c3b9deb0177a25488128616a3a96195ab5fa41bef" +checksum = "c4cf4d4e84fd3238cd5f5179b0a8e424c7ccc8106423a9fad627ae27ebc2b7b4" dependencies = [ "async-trait", "base64 0.22.1", @@ -4001,7 +4004,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "sha2 0.10.9", + "sha2 0.11.0", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -4012,9 +4015,9 @@ dependencies = [ [[package]] name = "google-cloud-type" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c310636aa7b660539c3f9259ae7a1fa2fd8bd7965a471bf6467094493cdb715a" +checksum = "bbcae00eb50388f5127e65cf2d45679fc4ec0e3596975bb6ec41aec6319a51c9" dependencies = [ "bytes", "google-cloud-wkt", @@ -4025,9 +4028,9 @@ dependencies = [ [[package]] name = "google-cloud-wkt" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ade65b0e4fa9cb4b6f147c8e726803bff453e3190910a53cbd3b0c019f5c2a" +checksum = "0b30ccefdb9276269bb0336afe207c5e0ba1a544a5eb0034763af051f2b9eb63" dependencies = [ "base64 0.22.1", "bytes", @@ -4994,9 +4997,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.184" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libflate" @@ -5074,7 +5077,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "libc", "plain", "redox_syscall 0.7.4", @@ -5105,7 +5108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9f8ff371890db2cf65a0758dba9a79f9cd965de369f6dbdc6581a22780af45e" dependencies = [ "async-trait", - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "chrono", "dashmap", @@ -5185,9 +5188,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.3" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ "hashbrown 0.16.1", ] @@ -5364,16 +5367,16 @@ dependencies = [ "ordered-float 5.3.0", "quanta", "radix_trie", - "rand 0.9.3", + "rand 0.9.4", "rand_xoshiro", "sketches-ddsketch", ] [[package]] name = "metrique" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41212ded0b2ba808836b36db163067eb6da41447e7f224267a7331b670009b9" +checksum = "f2cb1b5bc8898d2cb0da392c35f91826182c4cced80dddf061abd63bb7e34318" dependencies = [ "itoa", "jiff", @@ -5391,9 +5394,9 @@ dependencies = [ [[package]] name = "metrique-core" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5f67f8bf383a36ddb563846a9b143228505fd1f833aa936fdefbc9cb1e46855" +checksum = "67e3d9e65d3978d38102e3c4bd4142b91351467bb9c698db99654375e8f3858b" dependencies = [ "itertools 0.14.0", "metrique-writer-core", @@ -5414,9 +5417,9 @@ dependencies = [ [[package]] name = "metrique-service-metrics" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07a742784bddd4a8636cf4e952ec4bbd49d4dab34b7dd24d11026ab16adbfe19" +checksum = "cd93adb879c2d72ea8a23178deb5c9b379a6e459bfa08eb2c86a9c8c213cb3a6" dependencies = [ "metrique-writer", ] @@ -5429,9 +5432,9 @@ checksum = "d607939211e4eaaa8cd35394fa5e57faffb7390d0ac513b39992edcaf3cc526c" [[package]] name = "metrique-writer" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea8f6776cef2fed8ebaea2044971e4bfec96268eb47ccccb0e93797a198b025" +checksum = "e7885027d11fce09ba86f50b11ed8c34166a76e68b1568de3e9cb6689c18ea39" dependencies = [ "ahash 0.8.12", "crossbeam-queue", @@ -5441,7 +5444,7 @@ dependencies = [ "metrique-core", "metrique-writer-core", "metrique-writer-macro", - "rand 0.9.3", + "rand 0.9.4", "smallvec", "tokio", "tracing", @@ -5450,9 +5453,9 @@ dependencies = [ [[package]] name = "metrique-writer-core" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5399135c49a096ba9565872ddfef24627a4129501e930cbffe84d66897d4b12c" +checksum = "c2aa7ddd300b7405e6fc576fa57026b45097b5bf7b7c88c9be180dce6f4a1e09" dependencies = [ "derive-where", "itertools 0.14.0", @@ -5577,7 +5580,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22f9786d56d972959e1408b6a93be6af13b9c1392036c5c1fafa08a1b0c6ee87" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "byteorder", "derive_builder", "getset", @@ -5636,7 +5639,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cfg-if", "cfg_aliases", "libc", @@ -5649,7 +5652,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cfg-if", "cfg_aliases", "libc", @@ -5829,7 +5832,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f049ae562349fefb8e837eb15443da1e7c6dcbd8a11f52a228f92220c2e5c85e" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "libloading", "nvml-wrapper-sys", "static_assertions", @@ -5871,7 +5874,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -6093,7 +6096,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand 0.9.3", + "rand 0.9.4", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -6337,7 +6340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "500fa4cdeacd98997c5865e3d0d1cb8fe7e9d7d75ecc775e07989a433a9a9a59" dependencies = [ "arrayvec", - "bitflags 2.11.0", + "bitflags 2.11.1", "thiserror 2.0.18", "zerocopy", "zerocopy-derive", @@ -6503,9 +6506,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plain" @@ -6859,7 +6862,7 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c3a14896dfa883796f1cb410461aef38810ea05f2b2c33c5aded3649095fdad" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "memchr", "unicase", ] @@ -6969,7 +6972,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.3", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -7039,9 +7042,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -7055,7 +7058,7 @@ checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", "getrandom 0.4.2", - "rand_core 0.10.0", + "rand_core 0.10.1", "serde", ] @@ -7099,9 +7102,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rand_xoshiro" @@ -7129,14 +7132,14 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -7207,7 +7210,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -7216,7 +7219,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -7546,7 +7549,7 @@ dependencies = [ "digest 0.11.2", "pkcs1 0.8.0-rc.4", "pkcs8 0.11.0-rc.11", - "rand_core 0.10.0", + "rand_core 0.10.1", "signature 3.0.0-rc.10", "spki 0.8.0", "zeroize", @@ -7949,7 +7952,6 @@ dependencies = [ "rustfs-config", "rustfs-credentials", "rustfs-filemeta", - "rustfs-io-core", "rustfs-io-metrics", "rustfs-lock", "rustfs-madmin", @@ -8702,7 +8704,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -8715,7 +8717,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.12.1", @@ -8789,9 +8791,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.11" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20a6af516fea4b20eccceaf166e8aa666ac996208e8a644ce3ef5aa783bc7cd4" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "aws-lc-rs", "ring", @@ -8973,7 +8975,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -9288,7 +9290,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f1880df446116126965eeec169136b2e0251dba37c6223bcc819569550edea3" dependencies = [ "digest 0.11.2", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -9636,9 +9638,9 @@ dependencies = [ [[package]] name = "symbolic-common" -version = "12.17.4" +version = "12.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803d14d7cb9c6fa5b95a6f3de8af95b356a528d391998fa45a07d320a5573e51" +checksum = "6aba7211a1803a826f108af9f4d86d25abe880712f2a6449479279e861b293f8" dependencies = [ "debugid", "memmap2 0.9.10", @@ -9648,9 +9650,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.17.4" +version = "12.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39505731ae891b2dde47b0e4ae2ec40a7ced3476ab1129f1bf829e3fba62bb83" +checksum = "595bddd9d363c2ef6fc9fb33b98416ff209c5aae8bdca89a7dbbf2ef5e0ecc45" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -9732,7 +9734,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -10150,7 +10152,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "async-compression", - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "futures-core", "futures-util", @@ -10320,7 +10322,7 @@ dependencies = [ "http 1.4.0", "httparse", "log", - "rand 0.9.3", + "rand 0.9.4", "rustls", "rustls-pki-types", "sha1 0.10.6", @@ -10659,7 +10661,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "hashbrown 0.15.5", "indexmap 2.14.0", "semver", @@ -11165,7 +11167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.0", + "bitflags 2.11.1", "indexmap 2.14.0", "log", "serde", diff --git a/Cargo.toml b/Cargo.toml index 6409d87d7..ef6382de0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -288,7 +288,7 @@ zstd = "0.13.3" # Observability and Metrics metrics = "0.24.3" -dial9-tokio-telemetry = { version = "0.2", git = "https://github.com/dial9-rs/dial9-tokio-telemetry.git", rev = "7958b5e6562ff8d3baa944d4316c6ed2d7b73794" } +dial9-tokio-telemetry = "0.2" opentelemetry = { version = "0.31.0" } opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] } opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] } diff --git a/_typos.toml b/_typos.toml index 6ddfb7ab1..2d1aa7e51 100644 --- a/_typos.toml +++ b/_typos.toml @@ -39,7 +39,6 @@ abd = "abd" mak = "mak" gae = "gae" GAE = "GAE" -writeable = "writeable" # s3-tests original test names (cannot be changed) nonexisted = "nonexisted" consts = "consts" diff --git a/crates/config/src/constants/zero_copy.rs b/crates/config/src/constants/zero_copy.rs index ff23d6059..e931bd02c 100644 --- a/crates/config/src/constants/zero_copy.rs +++ b/crates/config/src/constants/zero_copy.rs @@ -49,34 +49,6 @@ pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE"; /// to regular I/O without errors. pub const DEFAULT_OBJECT_ZERO_COPY_ENABLE: bool = true; -/// Environment variable for zero-copy read operating mode. -/// -/// Supported values: -/// - `off`: disable mmap-backed chunk fast path and always use the compatibility path -/// - `conservative`: allow a single mmap window per request -/// - `balanced`: allow multiple mmap windows with the default size guardrails -/// - `aggressive`: allow multi-window mmap and relax the small-object cutoff -pub const ENV_OBJECT_ZERO_COPY_MODE: &str = "RUSTFS_OBJECT_ZERO_COPY_MODE"; - -/// Default zero-copy read mode. -pub const DEFAULT_OBJECT_ZERO_COPY_MODE: &str = "balanced"; - -/// Environment variable for the maximum mmap window size used by the chunk fast path. -/// -/// This controls the visible bytes per mapped chunk before the implementation emits a new window. -pub const ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES"; - -/// Default mmap window size for chunk fast path reads: 8 MiB. -pub const DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: usize = 8 * 1024 * 1024; - -/// Environment variable for the maximum total active mmap bytes. -/// -/// Requests that would exceed this active window budget fall back to the compatibility path. -pub const ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES"; - -/// Default maximum active mmap bytes across concurrent local chunk fast-path reads: 256 MiB. -pub const DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: usize = 256 * 1024 * 1024; - // ============================================================================= // Direct I/O Configuration // ============================================================================= diff --git a/crates/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml index 1546d2f3d..8ade58c0f 100644 --- a/crates/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -20,11 +20,6 @@ license.workspace = true repository.workspace = true rust-version.workspace = true -[[bin]] -name = "small_put_bench" -path = "src/bin/small_put_bench.rs" -test = false - [lints] workspace = true diff --git a/crates/e2e_test/src/bin/small_put_bench.rs b/crates/e2e_test/src/bin/small_put_bench.rs deleted file mode 100644 index 75b5864c5..000000000 --- a/crates/e2e_test/src/bin/small_put_bench.rs +++ /dev/null @@ -1,441 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use anyhow::{Context, Result, anyhow, bail}; -use aws_sdk_s3::config::{Credentials, Region}; -use aws_sdk_s3::primitives::ByteStream; -use aws_sdk_s3::types::{Delete, ObjectIdentifier}; -use aws_sdk_s3::{Client, Config}; -use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; -use bytes::Bytes; -use clap::Parser; -use serde::Serialize; -use std::path::PathBuf; -use std::time::{Duration, Instant}; - -#[derive(Parser, Debug)] -#[command(name = "small_put_bench")] -#[command(about = "Rust-native small PUT benchmark for RustFS-compatible S3 endpoints")] -struct Args { - #[arg(long, env = "RUSTFS_BENCH_ENDPOINT")] - endpoint: String, - - #[arg(long, env = "RUSTFS_BENCH_ACCESS_KEY", default_value = "rustfsadmin")] - access_key: String, - - #[arg(long, env = "RUSTFS_BENCH_SECRET_KEY", default_value = "rustfsadmin")] - secret_key: String, - - #[arg(long, env = "RUSTFS_BENCH_REGION", default_value = "us-east-1")] - region: String, - - #[arg(long, env = "RUSTFS_BENCH_BUCKET", default_value = "small-put-benchmark")] - bucket: String, - - #[arg(long, env = "RUSTFS_BENCH_SIZES", default_value = "4KiB,16KiB,64KiB,256KiB,1MiB")] - sizes: String, - - #[arg(long, env = "RUSTFS_BENCH_CONCURRENCY", default_value_t = 8)] - concurrency: usize, - - #[arg(long, env = "RUSTFS_BENCH_DURATION_SECS", default_value_t = 10)] - duration_secs: u64, - - #[arg(long, env = "RUSTFS_BENCH_TIMEOUT_SECS", default_value_t = 15)] - timeout_secs: u64, - - #[arg(long, env = "RUSTFS_BENCH_PREFIX")] - prefix: Option, - - #[arg(long)] - output_json: Option, - - #[arg(long, default_value_t = false)] - cleanup: bool, -} - -#[derive(Clone, Debug)] -struct SizeSpec { - label: String, - slug: String, - bytes: usize, -} - -#[derive(Debug)] -struct Sample { - ok: bool, - duration_ms: f64, -} - -#[derive(Debug, Serialize)] -struct SizeSummary { - label: String, - bytes: usize, - total: usize, - succeeded: usize, - failed: usize, - wall_secs: f64, - object_rate: f64, - throughput_mib_per_sec: f64, - avg_ms: Option, - p50_ms: Option, - p90_ms: Option, - p99_ms: Option, -} - -#[derive(Debug, Serialize)] -struct RunSummary { - run_id: String, - endpoint: String, - bucket: String, - concurrency: usize, - duration_secs: u64, - timeout_secs: u64, - sizes: Vec, -} - -fn main() -> Result<()> { - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .context("failed to build tokio runtime")?; - runtime.block_on(async_main()) -} - -async fn async_main() -> Result<()> { - let args = Args::parse(); - validate_args(&args)?; - - let sizes = parse_size_list(&args.sizes)?; - let run_id = args.prefix.clone().unwrap_or_else(default_run_id); - let client = build_s3_client(&args.endpoint, &args.access_key, &args.secret_key, &args.region); - - ensure_bucket(&client, &args.bucket).await?; - - let mut size_summaries = Vec::with_capacity(sizes.len()); - for size in &sizes { - let summary = run_size_benchmark( - client.clone(), - args.bucket.clone(), - run_id.clone(), - size.clone(), - args.concurrency, - Duration::from_secs(args.duration_secs), - Duration::from_secs(args.timeout_secs), - ) - .await?; - print_size_summary(&summary); - size_summaries.push(summary); - } - - if args.cleanup { - cleanup_prefix(&client, &args.bucket, &run_id).await?; - } - - let summary = RunSummary { - run_id, - endpoint: args.endpoint, - bucket: args.bucket, - concurrency: args.concurrency, - duration_secs: args.duration_secs, - timeout_secs: args.timeout_secs, - sizes: size_summaries, - }; - - if let Some(path) = args.output_json { - let json = serde_json::to_vec_pretty(&summary).context("failed to serialize benchmark summary")?; - std::fs::write(&path, json).with_context(|| format!("failed to write benchmark summary to {}", path.display()))?; - println!("Wrote summary to {}", path.display()); - } - - Ok(()) -} - -fn validate_args(args: &Args) -> Result<()> { - if args.concurrency == 0 { - bail!("--concurrency must be greater than zero"); - } - if args.duration_secs == 0 { - bail!("--duration-secs must be greater than zero"); - } - if args.timeout_secs == 0 { - bail!("--timeout-secs must be greater than zero"); - } - Ok(()) -} - -fn build_s3_client(endpoint: &str, access_key: &str, secret_key: &str, region: &str) -> Client { - let credentials = Credentials::new(access_key, secret_key, None, None, "small-put-bench"); - let mut config = Config::builder() - .credentials_provider(credentials) - .region(Region::new(region.to_string())) - .endpoint_url(endpoint) - .force_path_style(true) - .behavior_version_latest(); - - if endpoint.starts_with("http://") { - config = config.http_client(SmithyHttpClientBuilder::new().build_http()); - } - - Client::from_conf(config.build()) -} - -async fn ensure_bucket(client: &Client, bucket: &str) -> Result<()> { - if client.head_bucket().bucket(bucket).send().await.is_ok() { - return Ok(()); - } - - match client.create_bucket().bucket(bucket).send().await { - Ok(_) => Ok(()), - Err(err) => { - let rendered = err.to_string(); - if rendered.contains("BucketAlreadyOwnedByYou") || rendered.contains("BucketAlreadyExists") { - Ok(()) - } else { - Err(err).with_context(|| format!("failed to create benchmark bucket {bucket}")) - } - } - } -} - -async fn run_size_benchmark( - client: Client, - bucket: String, - run_id: String, - size: SizeSpec, - concurrency: usize, - duration: Duration, - timeout: Duration, -) -> Result { - let payload = Bytes::from(vec![0_u8; size.bytes]); - let deadline = Instant::now() + duration; - let wall_start = Instant::now(); - - let mut handles = Vec::with_capacity(concurrency); - for worker in 0..concurrency { - let client = client.clone(); - let bucket = bucket.clone(); - let payload = payload.clone(); - let prefix = format!("{run_id}/{}/worker-{worker}", size.slug); - handles.push(tokio::spawn(async move { - let mut samples = Vec::new(); - let mut idx = 0usize; - - while Instant::now() < deadline { - let key = format!("{prefix}/obj-{idx}.bin"); - let started_at = Instant::now(); - let request = client - .put_object() - .bucket(&bucket) - .key(key) - .body(ByteStream::from(payload.clone())) - .content_type("application/octet-stream"); - - let ok = matches!(tokio::time::timeout(timeout, request.send()).await, Ok(Ok(_))); - samples.push(Sample { - ok, - duration_ms: started_at.elapsed().as_secs_f64() * 1000.0, - }); - idx += 1; - } - - samples - })); - } - - let mut samples = Vec::new(); - for handle in handles { - samples.extend(handle.await.map_err(|err| anyhow!("benchmark worker join error: {err}"))?); - } - - Ok(build_size_summary(&size, samples, wall_start.elapsed())) -} - -fn build_size_summary(size: &SizeSpec, mut samples: Vec, wall_elapsed: Duration) -> SizeSummary { - let total = samples.len(); - let succeeded = samples.iter().filter(|sample| sample.ok).count(); - let failed = total.saturating_sub(succeeded); - let wall_secs = wall_elapsed.as_secs_f64(); - let object_rate = if wall_secs > 0.0 { succeeded as f64 / wall_secs } else { 0.0 }; - let throughput_mib_per_sec = if wall_secs > 0.0 { - ((size.bytes * succeeded) as f64 / (1024.0 * 1024.0)) / wall_secs - } else { - 0.0 - }; - - let avg_ms = if total > 0 { - Some(samples.iter().map(|sample| sample.duration_ms).sum::() / total as f64) - } else { - None - }; - - samples.sort_by(|lhs, rhs| lhs.duration_ms.total_cmp(&rhs.duration_ms)); - let durations: Vec = samples.into_iter().map(|sample| sample.duration_ms).collect(); - - SizeSummary { - label: size.label.clone(), - bytes: size.bytes, - total, - succeeded, - failed, - wall_secs, - object_rate, - throughput_mib_per_sec, - avg_ms, - p50_ms: percentile(&durations, 0.50), - p90_ms: percentile(&durations, 0.90), - p99_ms: percentile(&durations, 0.99), - } -} - -async fn cleanup_prefix(client: &Client, bucket: &str, prefix: &str) -> Result<()> { - let mut continuation_token = None; - loop { - let response = client - .list_objects_v2() - .bucket(bucket) - .prefix(prefix) - .set_continuation_token(continuation_token.clone()) - .send() - .await - .with_context(|| format!("failed to list objects for cleanup under {bucket}/{prefix}"))?; - - let objects: Vec = response - .contents - .unwrap_or_default() - .into_iter() - .filter_map(|object| object.key.map(|key| ObjectIdentifier::builder().key(key).build().ok())) - .flatten() - .collect(); - - for chunk in objects.chunks(1_000) { - if chunk.is_empty() { - continue; - } - - client - .delete_objects() - .bucket(bucket) - .delete( - Delete::builder() - .set_objects(Some(chunk.to_vec())) - .quiet(true) - .build() - .context("failed to build delete request")?, - ) - .send() - .await - .with_context(|| format!("failed to delete cleanup batch under {bucket}/{prefix}"))?; - } - - if response.is_truncated.unwrap_or(false) { - continuation_token = response.next_continuation_token; - } else { - break; - } - } - - Ok(()) -} - -fn parse_size_list(input: &str) -> Result> { - input - .split(',') - .map(str::trim) - .filter(|item| !item.is_empty()) - .map(parse_size_spec) - .collect() -} - -fn parse_size_spec(input: &str) -> Result { - let normalized = input.trim(); - let lower = normalized.to_ascii_lowercase(); - - let (number_part, multiplier) = if let Some(value) = lower.strip_suffix("kib") { - (value, 1024usize) - } else if let Some(value) = lower.strip_suffix("mib") { - (value, 1024usize * 1024usize) - } else if let Some(value) = lower.strip_suffix('b') { - (value, 1usize) - } else { - (lower.as_str(), 1usize) - }; - - let value = number_part - .trim() - .parse::() - .with_context(|| format!("invalid size component: {input}"))?; - let bytes = value - .checked_mul(multiplier) - .ok_or_else(|| anyhow!("size overflow for {input}"))?; - - Ok(SizeSpec { - label: normalized.to_string(), - slug: normalized - .chars() - .filter(|ch| ch.is_ascii_alphanumeric()) - .collect::() - .to_ascii_lowercase(), - bytes, - }) -} - -fn percentile(values: &[f64], percentile: f64) -> Option { - if values.is_empty() { - return None; - } - - let index = ((values.len() - 1) as f64 * percentile).floor() as usize; - values.get(index).copied() -} - -fn default_run_id() -> String { - format!("small-put-bench-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S")) -} - -fn print_size_summary(summary: &SizeSummary) { - println!( - "{}: success={} failed={} obj/s={:.3} MiB/s={:.3} avg={:.3?} p50={:.3?} p90={:.3?} p99={:.3?}", - summary.label, - summary.succeeded, - summary.failed, - summary.object_rate, - summary.throughput_mib_per_sec, - summary.avg_ms, - summary.p50_ms, - summary.p90_ms, - summary.p99_ms, - ); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_size_spec_supports_binary_units() { - let four_kib = parse_size_spec("4KiB").expect("4KiB should parse"); - assert_eq!(four_kib.bytes, 4 * 1024); - - let one_mib = parse_size_spec("1MiB").expect("1MiB should parse"); - assert_eq!(one_mib.bytes, 1024 * 1024); - } - - #[test] - fn percentile_returns_expected_bucket() { - let values = vec![10.0, 20.0, 30.0, 40.0, 50.0]; - assert_eq!(percentile(&values, 0.50), Some(30.0)); - assert_eq!(percentile(&values, 0.90), Some(40.0)); - } -} diff --git a/crates/e2e_test/src/checksum_upload_test.rs b/crates/e2e_test/src/checksum_upload_test.rs index c15237a15..6be69df85 100644 --- a/crates/e2e_test/src/checksum_upload_test.rs +++ b/crates/e2e_test/src/checksum_upload_test.rs @@ -19,13 +19,9 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; - use aws_sdk_s3::primitives::{ByteStream, SdkBody}; + use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart}; use base64::Engine; - use bytes::Bytes; - use futures::StreamExt; - use http_body::Frame; - use http_body_util::StreamBody; use rustfs_rio::{Checksum, ChecksumType as RioChecksumType}; use serial_test::serial; use sha2::{Digest, Sha256}; @@ -68,53 +64,6 @@ mod tests { .encoded } - fn streamed_body_70kib_of_a() -> ByteStream { - let bytes = Bytes::from_static(&[b'a'; 1024]); - let stream = futures::stream::repeat_with(move || { - let frame = Frame::data(bytes.clone()); - Ok::<_, std::io::Error>(frame) - }); - let body = WithSizeHint::new(StreamBody::new(stream.take(70)), 70 * 1024); - ByteStream::new(SdkBody::from_body_1_x(body)) - } - - struct WithSizeHint { - inner: T, - size_hint: usize, - } - - impl WithSizeHint { - fn new(inner: T, size_hint: usize) -> Self { - Self { inner, size_hint } - } - } - - impl http_body::Body for WithSizeHint - where - T: http_body::Body + Unpin, - { - type Data = T::Data; - type Error = T::Error; - - fn poll_frame( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll, Self::Error>>> { - let this = self.get_mut(); - std::pin::Pin::new(&mut this.inner).poll_frame(cx) - } - - fn is_end_stream(&self) -> bool { - self.inner.is_end_stream() - } - - fn size_hint(&self) -> http_body::SizeHint { - let mut hint = self.inner.size_hint(); - hint.set_exact(self.size_hint as u64); - hint - } - } - /// PutObject with Content-MD5: upload succeeds and GetObject returns same content. #[tokio::test] #[serial] @@ -187,121 +136,6 @@ mod tests { info!("PASSED: PutObject with checksum_sha256 and GetObject content match"); } - /// Mirrors `s3s-e2e` behavior: only request `checksum_algorithm`, then expect - /// both PutObject and GetObject(checksum_mode=enabled) to expose the same checksum. - #[tokio::test] - #[serial] - async fn test_put_object_with_checksum_algorithm_only() { - init_logging(); - info!("TEST: PutObject with checksum_algorithm only"); - - let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); - env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); - - let client = create_s3_client(&env); - let bucket = "test-checksum-algorithm-only"; - create_bucket(&client, bucket).await.expect("Failed to create bucket"); - - let key = "obj-with-checksum-algorithm-only.txt"; - let content = vec![b'a'; 70 * 1024]; - - let put_resp = client - .put_object() - .bucket(bucket) - .key(key) - .checksum_algorithm(ChecksumAlgorithm::Crc32) - .body(ByteStream::from(content.clone())) - .send() - .await - .expect("PutObject with checksum_algorithm should succeed"); - - let put_checksum = put_resp - .checksum_crc32() - .expect("PutObject should return checksum_crc32 when checksum_algorithm is used") - .to_string(); - - let mut get_resp = client - .get_object() - .bucket(bucket) - .key(key) - .checksum_mode(ChecksumMode::Enabled) - .send() - .await - .expect("GetObject should succeed"); - - let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(aws_sdk_s3::primitives::SdkBody::empty())) - .collect() - .await - .expect("collect body") - .into_bytes(); - - assert_eq!(body_bytes.as_ref(), content.as_slice(), "GetObject body must match uploaded content"); - assert_eq!( - get_resp.checksum_crc32().map(str::to_string), - Some(put_checksum), - "GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum" - ); - } - - /// Matches the `s3s-e2e` streaming upload shape more closely than `ByteStream::from(Vec)`. - #[tokio::test] - #[serial] - async fn test_put_object_with_checksum_algorithm_only_streaming_body() { - init_logging(); - info!("TEST: PutObject with checksum_algorithm only using streaming body"); - - let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); - env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); - - let client = create_s3_client(&env); - let bucket = "test-checksum-algorithm-streaming"; - create_bucket(&client, bucket).await.expect("Failed to create bucket"); - - let key = "obj-with-checksum-algorithm-streaming.txt"; - let expected_content = vec![b'a'; 70 * 1024]; - - let put_resp = client - .put_object() - .bucket(bucket) - .key(key) - .checksum_algorithm(ChecksumAlgorithm::Crc32) - .body(streamed_body_70kib_of_a()) - .send() - .await - .expect("PutObject with streaming checksum_algorithm should succeed"); - - let put_checksum = put_resp - .checksum_crc32() - .expect("PutObject should return checksum_crc32 for streaming checksum_algorithm uploads") - .to_string(); - - let mut get_resp = client - .get_object() - .bucket(bucket) - .key(key) - .checksum_mode(ChecksumMode::Enabled) - .send() - .await - .expect("GetObject should succeed"); - - let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(SdkBody::empty())) - .collect() - .await - .expect("collect body") - .into_bytes(); - - assert_eq!( - body_bytes.as_ref(), - expected_content.as_slice(), - "GetObject body must match uploaded content" - ); - assert_eq!( - get_resp.checksum_crc32().map(str::to_string), - Some(put_checksum), - "GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum for streaming uploads" - ); - } - /// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content. /// Uses part size >= 5MB (server minimum) for two parts. #[tokio::test] @@ -400,114 +234,6 @@ mod tests { info!("PASSED: MultipartUpload with checksum and GetObject content match"); } - /// Mirrors `s3s-e2e` multipart behavior: request checksum algorithm at MPU creation, - /// rely on auto checksum handling during UploadPart, and expect CompleteMultipartUpload to succeed. - #[tokio::test] - #[serial] - async fn test_multipart_upload_with_crc32_algorithm_only() { - init_logging(); - info!("TEST: MultipartUpload with checksum_algorithm only (CRC32)"); - - let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); - env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); - - let client = create_s3_client(&env); - let bucket = "test-multipart-checksum-crc32-auto"; - create_bucket(&client, bucket).await.expect("Failed to create bucket"); - - let key = "multipart-with-crc32-auto.bin"; - let part1_content = "a".repeat(5 * 1024 * 1024 + 1); - let part2_content = "b".repeat(1024); - - let create_resp = client - .create_multipart_upload() - .bucket(bucket) - .key(key) - .checksum_algorithm(ChecksumAlgorithm::Crc32) - .send() - .await - .expect("CreateMultipartUpload should succeed"); - - let upload_id = create_resp.upload_id().expect("upload_id should be present"); - - let part1_resp = client - .upload_part() - .bucket(bucket) - .key(key) - .upload_id(upload_id) - .part_number(1) - .body(ByteStream::from(part1_content.clone().into_bytes())) - .send() - .await - .expect("UploadPart 1 should succeed"); - let part1_checksum = part1_resp - .checksum_crc32() - .expect("UploadPart 1 should return checksum_crc32") - .to_string(); - - let part2_resp = client - .upload_part() - .bucket(bucket) - .key(key) - .upload_id(upload_id) - .part_number(2) - .body(ByteStream::from(part2_content.clone().into_bytes())) - .send() - .await - .expect("UploadPart 2 should succeed"); - let part2_checksum = part2_resp - .checksum_crc32() - .expect("UploadPart 2 should return checksum_crc32") - .to_string(); - - let completed_upload = CompletedMultipartUpload::builder() - .parts( - CompletedPart::builder() - .part_number(1) - .e_tag(part1_resp.e_tag().expect("etag part 1")) - .checksum_crc32(part1_checksum) - .build(), - ) - .parts( - CompletedPart::builder() - .part_number(2) - .e_tag(part2_resp.e_tag().expect("etag part 2")) - .checksum_crc32(part2_checksum) - .build(), - ) - .build(); - - client - .complete_multipart_upload() - .bucket(bucket) - .key(key) - .upload_id(upload_id) - .multipart_upload(completed_upload) - .send() - .await - .expect("CompleteMultipartUpload should succeed"); - - let body_bytes = client - .get_object() - .bucket(bucket) - .key(key) - .send() - .await - .expect("GetObject should succeed") - .body - .collect() - .await - .expect("collect body") - .into_bytes(); - - let expected_content = format!("{part1_content}{part2_content}"); - assert_eq!( - body_bytes.as_ref(), - expected_content.as_bytes(), - "completed multipart object must match concatenated parts" - ); - } - /// Regression test for issue #2282: /// CRC64NVME full-object checksum should match between direct PutObject and multipart upload. #[tokio::test] diff --git a/crates/e2e_test/src/kms/kms_local_test.rs b/crates/e2e_test/src/kms/kms_local_test.rs index 208dbc6a2..d3a24404b 100644 --- a/crates/e2e_test/src/kms/kms_local_test.rs +++ b/crates/e2e_test/src/kms/kms_local_test.rs @@ -344,9 +344,6 @@ async fn test_local_kms_multipart_upload() { test_multipart_upload_with_sse_c(&s3_client, TEST_BUCKET) .await .expect("SSE-C multipart upload test failed"); - test_multipart_download_with_wrong_sse_c_key_fails(&s3_client, TEST_BUCKET) - .await - .expect("SSE-C multipart wrong-key download test failed"); // Test 4: Large multipart upload (test streaming encryption with multiple blocks) // TODO: Re-enable after fixing streaming encryption issues with large files @@ -651,99 +648,6 @@ async fn test_multipart_upload_with_sse_c( Ok(()) } -async fn test_multipart_download_with_wrong_sse_c_key_fails( - s3_client: &aws_sdk_s3::Client, - bucket: &str, -) -> Result<(), Box> { - let object_key = "multipart-sse-c-bad-download-test"; - let part_size = 5 * 1024 * 1024; - let total_parts = 2; - let total_size = part_size * total_parts; - - let encryption_key = "01234567890123456789012345678901"; - let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key); - let key_md5 = sse_customer_key_md5_base64(encryption_key); - - let wrong_key = "abcdefghijklmnopqrstuvwxyz012345"; - let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key); - let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key); - - let test_data: Vec = (0..total_size).map(|i| ((i * 5) % 256) as u8).collect(); - - let create_multipart_output = s3_client - .create_multipart_upload() - .bucket(bucket) - .key(object_key) - .sse_customer_algorithm("AES256") - .sse_customer_key(&key_b64) - .sse_customer_key_md5(&key_md5) - .send() - .await?; - - let upload_id = create_multipart_output.upload_id().unwrap(); - let mut completed_parts = Vec::new(); - - for part_number in 1..=total_parts { - let start = (part_number - 1) * part_size; - let end = std::cmp::min(start + part_size, total_size); - let part_data = &test_data[start..end]; - - let upload_part_output = s3_client - .upload_part() - .bucket(bucket) - .key(object_key) - .upload_id(upload_id) - .part_number(part_number as i32) - .body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec())) - .sse_customer_algorithm("AES256") - .sse_customer_key(&key_b64) - .sse_customer_key_md5(&key_md5) - .send() - .await?; - - completed_parts.push( - aws_sdk_s3::types::CompletedPart::builder() - .part_number(part_number as i32) - .e_tag(upload_part_output.e_tag().unwrap()) - .build(), - ); - } - - let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder() - .set_parts(Some(completed_parts)) - .build(); - - s3_client - .complete_multipart_upload() - .bucket(bucket) - .key(object_key) - .upload_id(upload_id) - .multipart_upload(completed_multipart_upload) - .send() - .await?; - - let err = s3_client - .get_object() - .bucket(bucket) - .key(object_key) - .sse_customer_algorithm("AES256") - .sse_customer_key(&wrong_key_b64) - .sse_customer_key_md5(&wrong_key_md5) - .send() - .await - .expect_err("multipart SSE-C download with the wrong key should fail"); - - let service_err = err.into_service_error(); - assert_eq!( - service_err.meta().code(), - Some("InvalidRequest"), - "wrong-key multipart SSE-C download should return InvalidRequest, got {:?}", - service_err.meta().code() - ); - - Ok(()) -} - /// Test large multipart upload to verify streaming encryption works correctly #[allow(dead_code)] async fn test_large_multipart_upload( diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index 1e642f2a6..1a877a1e2 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -100,10 +100,6 @@ mod cluster_concurrency_test; #[cfg(test)] mod checksum_upload_test; -// Range request regression tests -#[cfg(test)] -mod range_request_test; - // Group deletion tests #[cfg(test)] mod group_delete_test; diff --git a/crates/e2e_test/src/range_request_test.rs b/crates/e2e_test/src/range_request_test.rs deleted file mode 100644 index df0f1f4ed..000000000 --- a/crates/e2e_test/src/range_request_test.rs +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! End-to-end regression tests for invalid and suffix GET object ranges. - -#[cfg(test)] -mod tests { - use crate::common::{RustFSTestEnvironment, init_logging}; - use aws_sdk_s3::Client; - use aws_sdk_s3::error::SdkError; - use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; - use tracing::info; - - fn create_s3_client(env: &RustFSTestEnvironment) -> Client { - env.create_s3_client() - } - - async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box> { - match client.create_bucket().bucket(bucket).send().await { - Ok(_) => Ok(()), - Err(err) => { - if err.to_string().contains("BucketAlreadyOwnedByYou") || err.to_string().contains("BucketAlreadyExists") { - Ok(()) - } else { - Err(Box::new(err)) - } - } - } - } - - #[tokio::test] - #[serial] - async fn test_get_object_invalid_range_returns_416_issue_s3_implemented_tests() { - init_logging(); - info!("TEST: GetObject invalid range should return InvalidRange/416"); - - let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); - env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); - - let client = create_s3_client(&env); - let bucket = "test-invalid-range"; - let key = "range.txt"; - let content = b"testcontent"; - - create_bucket(&client, bucket).await.expect("Failed to create bucket"); - client - .put_object() - .bucket(bucket) - .key(key) - .body(ByteStream::from_static(content)) - .send() - .await - .expect("PutObject should succeed"); - - let result = client.get_object().bucket(bucket).key(key).range("bytes=40-50").send().await; - - let err = result.expect_err("GetObject with an unsatisfiable range should fail"); - match err { - SdkError::ServiceError(service_err) => { - assert_eq!(service_err.raw().status().as_u16(), 416, "invalid range should return HTTP 416"); - - let s3_err = service_err.into_err(); - assert_eq!(s3_err.meta().code(), Some("InvalidRange"), "invalid range should map to InvalidRange"); - } - other_err => panic!("Expected S3 service error, got: {other_err:?}"), - } - } - - #[tokio::test] - #[serial] - async fn test_get_object_suffix_byte_range_returns_correct_body() { - init_logging(); - info!("TEST: GetObject suffix byte-range should return correct body"); - - let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); - env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); - - let client = create_s3_client(&env); - let bucket = "test-suffix-range"; - let key = "range-suffix.bin"; - - // ~3 MB so the object spans multiple erasure blocks (block_size = 1 MB). - // Suffix ranges on single-block objects never hit the bug. - let file_size: usize = 3_095_910; - let content: Vec = (0..file_size).map(|i| (i % 256) as u8).collect(); - - create_bucket(&client, bucket).await.expect("Failed to create bucket"); - client - .put_object() - .bucket(bucket) - .key(key) - .body(ByteStream::from(content.clone())) - .send() - .await - .expect("PutObject should succeed"); - - // bytes=-8 — last 8 bytes (parquet footer read) - let result = client - .get_object() - .bucket(bucket) - .key(key) - .range("bytes=-8") - .send() - .await - .expect("bytes=-8 should succeed"); - let body = result.body.collect().await.expect("bytes=-8 body").into_bytes(); - assert_eq!(body.len(), 8, "bytes=-8 body length"); - assert_eq!(&body[..], &content[file_size - 8..], "bytes=-8 body content"); - - // bytes=-96 — last 96 bytes - let result = client - .get_object() - .bucket(bucket) - .key(key) - .range("bytes=-96") - .send() - .await - .expect("bytes=-96 should succeed"); - let body = result.body.collect().await.expect("bytes=-96 body").into_bytes(); - assert_eq!(body.len(), 96, "bytes=-96 body length"); - assert_eq!(&body[..], &content[file_size - 96..], "bytes=-96 body content"); - - // bytes=-1 — last byte - let result = client - .get_object() - .bucket(bucket) - .key(key) - .range("bytes=-1") - .send() - .await - .expect("bytes=-1 should succeed"); - let body = result.body.collect().await.expect("bytes=-1 body").into_bytes(); - assert_eq!(body.len(), 1, "bytes=-1 body length"); - assert_eq!(body[0], content[file_size - 1], "bytes=-1 body content"); - - // bytes=0-7 — absolute range (regression guard) - let result = client - .get_object() - .bucket(bucket) - .key(key) - .range("bytes=0-7") - .send() - .await - .expect("bytes=0-7 should succeed"); - let body = result.body.collect().await.expect("bytes=0-7 body").into_bytes(); - assert_eq!(body.len(), 8, "bytes=0-7 body length"); - assert_eq!(&body[..], &content[0..8], "bytes=0-7 body content"); - - // Equivalent absolute range for the same last-8-bytes window - let start = file_size - 8; - let end = file_size - 1; - let range = format!("bytes={start}-{end}"); - let result = client - .get_object() - .bucket(bucket) - .key(key) - .range(&range) - .send() - .await - .expect("absolute tail range should succeed"); - let body = result.body.collect().await.expect("absolute tail body").into_bytes(); - assert_eq!(body.len(), 8, "absolute tail body length"); - assert_eq!(&body[..], &content[start..], "absolute tail body content"); - } -} diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 3f5d854d2..5b4888dfc 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -70,7 +70,6 @@ reed-solomon-erasure = { workspace = true } reed-solomon-simd = { workspace = true } lazy_static.workspace = true rustfs-lock.workspace = true -rustfs-io-core.workspace = true rustfs-io-metrics.workspace = true regex = { workspace = true } path-absolutize = { workspace = true } diff --git a/crates/ecstore/README.md b/crates/ecstore/README.md index e9feac64e..bcca1fa8a 100644 --- a/crates/ecstore/README.md +++ b/crates/ecstore/README.md @@ -32,10 +32,6 @@ For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs). -## 📈 Benchmarks - -ECStore ships Criterion benchmarks under [`crates/ecstore/benches/`](./benches/) for the remaining erasure-code and comparison workloads. Re-run benchmarks on your target machine before treating any local measurement as a regression baseline. - ## 📄 License This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details. diff --git a/crates/ecstore/src/bitrot.rs b/crates/ecstore/src/bitrot.rs index d4af20dd4..6969edf73 100644 --- a/crates/ecstore/src/bitrot.rs +++ b/crates/ecstore/src/bitrot.rs @@ -21,8 +21,6 @@ use std::time::Instant; use tokio::io::AsyncRead; use tracing::debug; -const BITROT_READ_OPERATION: &str = "bitrot_read"; - /// Create a BitrotReader from either inline data or disk file stream /// /// # Parameters @@ -67,38 +65,20 @@ pub async fn create_bitrot_reader( } else if let Some(disk) = disk { // Read from disk if use_zero_copy { - if !disk.is_local() { - rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy); - rustfs_io_metrics::record_io_fallback( - rustfs_io_metrics::IoStage::ReadSetup, - rustfs_io_metrics::FallbackReason::NonLocalBackend, - ); - - let rd = disk.read_file_stream(bucket, path, offset, length).await?; - let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify); - return Ok(Some(reader)); - } - // Try zero-copy read first (uses mmap on Unix) let start = Instant::now(); match disk.read_file_zero_copy(bucket, path, offset, length).await { Ok(bytes) => { let duration_ms = start.elapsed().as_secs_f64() * 1000.0; - rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Fast); - // `read_file_zero_copy()` returns a shared `Bytes` view, which preserves the - // mmap-backed fast path without exposing chunk-native GET internals. - rustfs_io_metrics::record_io_copy_mode( - BITROT_READ_OPERATION, - rustfs_io_metrics::CopyMode::SharedBytes, - bytes.len(), - ); + // Record zero-copy metrics + rustfs_io_metrics::record_zero_copy_read(bytes.len(), duration_ms); + // Log successful zero-copy read debug!( size = bytes.len(), - duration_ms, path = %path, - "bitrot_fast_read_success" + "zero_copy_read_success" ); // Wrap Bytes in Cursor for AsyncRead @@ -113,16 +93,14 @@ pub async fn create_bitrot_reader( Ok(Some(reader)) } Err(e) => { - rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy); - rustfs_io_metrics::record_io_fallback( - rustfs_io_metrics::IoStage::ReadSetup, - rustfs_io_metrics::FallbackReason::Unknown, - ); + // Record zero-copy fallback + rustfs_io_metrics::record_zero_copy_fallback(&format!("{:?}", e)); + // Log zero-copy fallback debug!( - reason = %e, + reason = %format!("{:?}", e), path = %path, - "bitrot_fast_read_fallback" + "zero_copy_fallback" ); // Fall back to regular stream read on error @@ -139,7 +117,6 @@ pub async fn create_bitrot_reader( } } } else { - rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy); // Use regular stream read match disk.read_file_stream(bucket, path, offset, length).await { Ok(rd) => { diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 9f916dfec..c51cb2fea 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -24,7 +24,7 @@ use rustfs_utils::http::headers::{ AMZ_STORAGE_CLASS, }; use rustfs_utils::http::{ - AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_CRC, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS, + AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, has_internal_suffix, insert_bytes, is_internal_key, }; @@ -230,10 +230,6 @@ impl FileMeta { } } - if let Some(checksum) = fi.checksum.as_ref() { - insert_bytes(&mut obj.meta_sys, SUFFIX_CRC, checksum.to_vec()); - } - if let Some(mod_time) = fi.mod_time { obj.mod_time = Some(mod_time); } diff --git a/crates/protocols/src/swift/object.rs b/crates/protocols/src/swift/object.rs index c7715a602..1c59da07f 100644 --- a/crates/protocols/src/swift/object.rs +++ b/crates/protocols/src/swift/object.rs @@ -381,7 +381,7 @@ where let hash_reader = HashReader::from_stream(buf_reader, content_length, content_length, None, None, false) .map_err(|e| sanitize_storage_error("Hash reader creation", e))?; - // 15. Hand the hash reader to the chunk-native PUT data wrapper + // 15. Wrap in PutObjReader as expected by storage layer let mut put_reader = PutObjReader::new(hash_reader); // 16. Upload object to storage @@ -464,7 +464,7 @@ where let hash_reader = HashReader::from_stream(buf_reader, content_length, content_length, None, None, false) .map_err(|e| sanitize_storage_error("Hash reader creation", e))?; - // Hand the hash reader to the chunk-native PUT data wrapper + // Wrap in PutObjReader let mut put_reader = PutObjReader::new(hash_reader); // Upload object to storage diff --git a/crates/rio/src/checksum.rs b/crates/rio/src/checksum.rs index ef4b85a6d..86867cc15 100644 --- a/crates/rio/src/checksum.rs +++ b/crates/rio/src/checksum.rs @@ -29,21 +29,11 @@ pub const RUSTFS_MULTIPART_CHECKSUM: &str = "x-rustfs-multipart-checksum"; /// RustFS multipart checksum type metadata key pub const RUSTFS_MULTIPART_CHECKSUM_TYPE: &str = "x-rustfs-multipart-checksum-type"; -const AMZ_CHECKSUM_ALGORITHM: &str = "x-amz-checksum-algorithm"; -const AMZ_SDK_CHECKSUM_ALGORITHM: &str = "x-amz-sdk-checksum-algorithm"; - /// Checksum type enumeration with flags #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct ChecksumType(pub u32); impl ChecksumType { - fn algorithm_from_headers(headers: &HeaderMap) -> Option<&str> { - headers - .get(AMZ_CHECKSUM_ALGORITHM) - .and_then(|v| v.to_str().ok()) - .or_else(|| headers.get(AMZ_SDK_CHECKSUM_ALGORITHM).and_then(|v| v.to_str().ok())) - } - /// Checksum will be sent in trailing header pub const TRAILING: ChecksumType = ChecksumType(1 << 0); @@ -166,7 +156,10 @@ impl ChecksumType { pub fn from_header(headers: &HeaderMap) -> Self { Self::from_string_with_obj_type( - Self::algorithm_from_headers(headers).unwrap_or(""), + headers + .get("x-amz-checksum-algorithm") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""), headers.get("x-amz-checksum-type").and_then(|v| v.to_str().ok()).unwrap_or(""), ) } @@ -580,7 +573,7 @@ pub fn get_content_checksum(headers: &HeaderMap) -> Result, std fn get_content_checksum_direct(headers: &HeaderMap) -> (ChecksumType, String) { let mut checksum_type = ChecksumType::NONE; - if let Some(alg) = ChecksumType::algorithm_from_headers(headers) { + if let Some(alg) = headers.get("x-amz-checksum-algorithm").and_then(|v| v.to_str().ok()) { checksum_type = ChecksumType::from_string_with_obj_type( alg, headers.get("x-amz-checksum-type").and_then(|s| s.to_str().ok()).unwrap_or(""), @@ -1139,35 +1132,6 @@ fn crc64_combine(poly: u64, crc1: u64, crc2: u64, len2: i64) -> u64 { #[cfg(test)] mod tests { use super::{Checksum, ChecksumType}; - use http::HeaderMap; - - #[test] - fn algorithm_from_headers_parses_amz_checksum_algorithm() { - let mut headers = HeaderMap::new(); - headers.insert("x-amz-checksum-algorithm", "SHA256".parse().expect("valid header value")); - assert_eq!(ChecksumType::algorithm_from_headers(&headers), Some("SHA256")); - } - - #[test] - fn algorithm_from_headers_falls_back_to_sdk_header() { - let mut headers = HeaderMap::new(); - headers.insert("x-amz-sdk-checksum-algorithm", "CRC32C".parse().expect("valid header value")); - assert_eq!(ChecksumType::algorithm_from_headers(&headers), Some("CRC32C")); - } - - #[test] - fn algorithm_from_headers_prefers_amz_over_sdk() { - let mut headers = HeaderMap::new(); - headers.insert("x-amz-checksum-algorithm", "SHA256".parse().expect("valid header value")); - headers.insert("x-amz-sdk-checksum-algorithm", "CRC32C".parse().expect("valid header value")); - assert_eq!(ChecksumType::algorithm_from_headers(&headers), Some("SHA256")); - } - - #[test] - fn algorithm_from_headers_returns_none_when_missing() { - let headers = HeaderMap::new(); - assert_eq!(ChecksumType::algorithm_from_headers(&headers), None); - } #[test] fn crc64_nvme_add_part_matches_full_object_checksum() { diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 95b40dfb7..d2e164748 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -257,10 +257,23 @@ impl From for ApiError { impl From for ApiError { fn from(err: std::io::Error) -> Self { - // Check if the error is a ChecksumMismatch (BadDigest) + // Check if the inner error is a StorageError (e.g. InvalidRangeSpec wrapped by object-io) if let Some(inner) = err.get_ref() { if let Some(storage_error) = inner.downcast_ref::() { - return storage_error.clone().into(); + let code = match storage_error { + StorageError::InvalidRangeSpec(_) => S3ErrorCode::InvalidRange, + _ => S3ErrorCode::InternalError, + }; + let message = if code == S3ErrorCode::InternalError { + storage_error.to_string() + } else { + ApiError::error_code_to_message(&code) + }; + return ApiError { + code, + message, + source: Some(Box::new(err)), + }; } if inner.downcast_ref::().is_some() { return ApiError { @@ -555,15 +568,4 @@ mod tests { // This is expected because ApiError is not a typical Error implementation assert!(error.source().is_none()); } - - #[test] - fn test_api_error_from_io_error_unwraps_invalid_range_storage_error() { - let io_error = std::io::Error::from(StorageError::InvalidRangeSpec("range invalid".to_string())); - - let api_error: ApiError = io_error.into(); - - assert_eq!(api_error.code, S3ErrorCode::InvalidRange); - assert_eq!(api_error.message, ApiError::error_code_to_message(&S3ErrorCode::InvalidRange)); - assert!(api_error.source.is_some()); - } } diff --git a/scripts/bench-small-put-local.sh b/scripts/bench-small-put-local.sh deleted file mode 100755 index eb10b4a70..000000000 --- a/scripts/bench-small-put-local.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" -RUSTFS_BIN="${ROOT_DIR}/target/debug/rustfs" -WARP_BIN="${WARP_BIN:-warp}" -MC_BIN="${MC_BIN:-mc}" - -PORT="${RUSTFS_BENCH_PORT:-9900}" -CONSOLE_PORT="${RUSTFS_BENCH_CONSOLE_PORT:-9901}" -HOST="127.0.0.1:${PORT}" -ACCESS_KEY="${RUSTFS_BENCH_ACCESS_KEY:-rustfsadmin}" -SECRET_KEY="${RUSTFS_BENCH_SECRET_KEY:-rustfsadmin}" -BENCH_ROOT="${ROOT_DIR}/target/benchmarks" -RUN_ID="${RUSTFS_BENCH_RUN_ID:-small-put-$(date +%Y%m%d-%H%M%S)}" -RUN_DIR="${BENCH_ROOT}/${RUN_ID}" -VOLUME_DIR="${RUN_DIR}/volumes" -LOG_DIR="${RUN_DIR}/logs" -MC_CONFIG_DIR="${RUN_DIR}/mc-config" -RESULTS_MD="${RUN_DIR}/RESULTS.md" -SERVER_LOG="${LOG_DIR}/rustfs.log" -WARP_BUCKET="${RUSTFS_BENCH_BUCKET:-warp-small-put-benchmark}" -WARP_DURATION="${RUSTFS_BENCH_DURATION:-20s}" -WARP_CONCURRENCY="${RUSTFS_BENCH_CONCURRENCY:-32}" - -mkdir -p "${VOLUME_DIR}" "${LOG_DIR}" "${MC_CONFIG_DIR}" -mkdir -p "${VOLUME_DIR}"/disk{1..4} - -# Verify mc (MinIO Client) is available -if ! command -v "${MC_BIN}" >/dev/null 2>&1; then - echo "Error: 'mc' (MinIO Client) not found. MC_BIN=${MC_BIN}" >&2 - echo "" >&2 - echo "Install mc with one of the following:" >&2 - echo " macOS (Homebrew): brew install minio/stable/mc" >&2 - echo " Go: go install github.com/minio/mc@latest" >&2 - echo " Linux (amd64): curl -sSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2 - echo " Linux (arm64): curl -sSL https://dl.min.io/client/mc/release/linux-arm64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2 - echo "" >&2 - echo "Or set MC_BIN=/path/to/mc before running this script." >&2 - exit 1 -fi - -# Verify warp is available -if ! command -v "${WARP_BIN}" >/dev/null 2>&1; then - echo "Error: 'warp' benchmark tool not found. WARP_BIN=${WARP_BIN}" >&2 - echo "" >&2 - echo "Install warp with one of the following:" >&2 - echo " Go: go install github.com/minio/warp@latest" >&2 - echo " Linux (amd64): curl -sSL https://github.com/minio/warp/releases/latest/download/warp_Linux_x86_64.tar.gz | tar -xz -C /usr/local/bin warp" >&2 - echo " macOS (amd64): curl -sSL https://github.com/minio/warp/releases/latest/download/warp_Darwin_x86_64.tar.gz | tar -xz -C /usr/local/bin warp" >&2 - echo " macOS (arm64): curl -sSL https://github.com/minio/warp/releases/latest/download/warp_Darwin_arm64.tar.gz | tar -xz -C /usr/local/bin warp" >&2 - echo "" >&2 - echo "Or set WARP_BIN=/path/to/warp before running this script." >&2 - exit 1 -fi - -SIZES=( - "4KiB" - "16KiB" - "64KiB" - "256KiB" - "1MiB" -) - -cleanup() { - if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" >/dev/null 2>&1; then - kill "${SERVER_PID}" >/dev/null 2>&1 || true - wait "${SERVER_PID}" >/dev/null 2>&1 || true - fi -} - -trap cleanup EXIT - -echo "Starting local RustFS benchmark server..." -( - cd "${ROOT_DIR}" - export RUST_LOG="${RUST_LOG:-warn}" - export RUSTFS_VOLUMES="${VOLUME_DIR}/disk{1...4}" - export RUSTFS_ADDRESS=":${PORT}" - export RUSTFS_CONSOLE_ENABLE=false - export RUSTFS_CONSOLE_ADDRESS=":${CONSOLE_PORT}" - export RUSTFS_ROOT_USER="${ACCESS_KEY}" - export RUSTFS_ROOT_PASSWORD="${SECRET_KEY}" - export RUSTFS_OBJECT_CACHE_ENABLE=false - "${RUSTFS_BIN}" server >"${SERVER_LOG}" 2>&1 -) & -SERVER_PID=$! - -echo "Waiting for RustFS server to become ready..." -for _ in $(seq 1 60); do - if curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 || curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then - break - fi - sleep 1 -done - -if ! curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 && ! curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then - echo "RustFS did not become ready in time. See ${SERVER_LOG}" >&2 - exit 1 -fi - -echo "Preparing mc alias..." -MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" alias set bench "http://${HOST}" "${ACCESS_KEY}" "${SECRET_KEY}" >/dev/null -MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" mb --ignore-existing "bench/${WARP_BUCKET}" >/dev/null - -{ - echo "# Small PUT Benchmark Results" - echo - echo "- Run ID: \`${RUN_ID}\`" - echo "- Host: \`${HOST}\`" - echo "- Duration per size: \`${WARP_DURATION}\`" - echo "- Concurrency: \`${WARP_CONCURRENCY}\`" - echo "- Bucket: \`${WARP_BUCKET}\`" - echo -} >"${RESULTS_MD}" - -for size in "${SIZES[@]}"; do - echo - echo "Running warp PUT benchmark for size ${size}..." - size_slug="$(echo "${size}" | tr '[:upper:]' '[:lower:]')" - OUT_FILE="${LOG_DIR}/warp-put-${size}.log" - BENCHDATA_FILE="${RUN_DIR}/warp-put-${size}.csv.zst" - "${WARP_BIN}" put \ - --no-color \ - --host "${HOST}" \ - --access-key "${ACCESS_KEY}" \ - --secret-key "${SECRET_KEY}" \ - --bucket "${WARP_BUCKET}" \ - --obj.size "${size}" \ - --duration "${WARP_DURATION}" \ - --concurrent "${WARP_CONCURRENCY}" \ - --prefix "${RUN_ID}/${size_slug}" \ - --disable-multipart \ - --noclear \ - --benchdata "${BENCHDATA_FILE}" \ - --insecure \ - >"${OUT_FILE}" 2>&1 - - { - echo "## ${size}" - echo - echo '```text' - tail -n 20 "${OUT_FILE}" - echo '```' - echo - } >>"${RESULTS_MD}" -done - -echo -echo "Benchmark completed." -echo "Results: ${RESULTS_MD}" -echo "Server log: ${SERVER_LOG}" diff --git a/scripts/bench-small-put-mc-local.sh b/scripts/bench-small-put-mc-local.sh deleted file mode 100755 index d7f716a76..000000000 --- a/scripts/bench-small-put-mc-local.sh +++ /dev/null @@ -1,300 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" -RUSTFS_BIN="${RUSTFS_BIN:-${ROOT_DIR}/target/debug/rustfs}" -MC_BIN="${MC_BIN:-mc}" - -PORT="${RUSTFS_BENCH_PORT:-9910}" -CONSOLE_PORT="${RUSTFS_BENCH_CONSOLE_PORT:-9911}" -HOST="127.0.0.1:${PORT}" -ACCESS_KEY="${RUSTFS_BENCH_ACCESS_KEY:-rustfsadmin}" -SECRET_KEY="${RUSTFS_BENCH_SECRET_KEY:-rustfsadmin}" - -OPS_PER_SIZE="${RUSTFS_BENCH_OPS_PER_SIZE:-0}" -BENCH_SECONDS="${RUSTFS_BENCH_SECONDS:-10}" -CONCURRENCY="${RUSTFS_BENCH_CONCURRENCY:-32}" -PUT_TIMEOUT_SECS="${RUSTFS_BENCH_PUT_TIMEOUT_SECS:-15}" -RUN_ID="${RUSTFS_BENCH_RUN_ID:-small-put-mc-$(date +%Y%m%d-%H%M%S)}" -BENCH_ROOT="${ROOT_DIR}/target/benchmarks" -RUN_DIR="${BENCH_ROOT}/${RUN_ID}" -VOLUME_DIR="${RUN_DIR}/volumes" -LOG_DIR="${RUN_DIR}/logs" -FILE_DIR="${RUN_DIR}/files" -MC_CONFIG_DIR="${RUN_DIR}/mc-config" -RESULTS_MD="${RUN_DIR}/RESULTS.md" -SERVER_LOG="${LOG_DIR}/rustfs.log" -BUCKET="${RUSTFS_BENCH_BUCKET:-small-put-benchmark}" - -# Verify mc (MinIO Client) is available -if ! command -v "${MC_BIN}" >/dev/null 2>&1; then - echo "Error: 'mc' (MinIO Client) not found. MC_BIN=${MC_BIN}" >&2 - echo "" >&2 - echo "Install mc with one of the following:" >&2 - echo " macOS (Homebrew): brew install minio/stable/mc" >&2 - echo " Go: go install github.com/minio/mc@latest" >&2 - echo " Linux (amd64): curl -sSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2 - echo " Linux (arm64): curl -sSL https://dl.min.io/client/mc/release/linux-arm64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2 - echo "" >&2 - echo "Or set MC_BIN=/path/to/mc before running this script." >&2 - exit 1 -fi - -mkdir -p "${VOLUME_DIR}" "${LOG_DIR}" "${FILE_DIR}" "${MC_CONFIG_DIR}" -mkdir -p "${VOLUME_DIR}"/disk{1..4} - -SIZES=( - "4KiB" - "16KiB" - "64KiB" - "256KiB" - "1MiB" -) - -size_to_bytes() { - case "$1" in - 4KiB) echo 4096 ;; - 16KiB) echo 16384 ;; - 64KiB) echo 65536 ;; - 256KiB) echo 262144 ;; - 1MiB) echo 1048576 ;; - *) - echo "unsupported size: $1" >&2 - return 1 - ;; - esac -} - -now_secs() { - perl -MTime::HiRes=time -e 'printf "%.6f\n", time' -} - -prepare_sample_file() { - local size_label="$1" - local bytes="$2" - local file_path="${FILE_DIR}/${size_label}.bin" - if [[ ! -f "${file_path}" ]]; then - dd if=/dev/zero of="${file_path}" bs="${bytes}" count=1 status=none - fi - echo "${file_path}" -} - -run_mc_put_with_timeout() { - local sample_file="$1" - local key="$2" - perl -e ' - use strict; - use warnings; - - my $timeout = shift @ARGV; - my $pid = fork(); - die "fork failed: $!" unless defined $pid; - - if ($pid == 0) { - exec @ARGV; - die "exec failed: $!"; - } - - my $timed_out = 0; - local $SIG{ALRM} = sub { - $timed_out = 1; - kill "TERM", $pid; - select undef, undef, undef, 0.2; - kill "KILL", $pid; - }; - - alarm($timeout); - my $waited = waitpid($pid, 0); - alarm(0); - - exit(1) if $timed_out; - exit(($waited == $pid && $? == 0) ? 0 : 1); - ' "${PUT_TIMEOUT_SECS}" env MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" put --quiet --disable-multipart "${sample_file}" "${key}" >/dev/null 2>&1 -} - -export -f run_mc_put_with_timeout - -cleanup() { - if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" >/dev/null 2>&1; then - kill "${SERVER_PID}" >/dev/null 2>&1 || true - wait "${SERVER_PID}" >/dev/null 2>&1 || true - fi -} - -trap cleanup EXIT - -echo "Starting local RustFS benchmark server..." -( - cd "${ROOT_DIR}" - export RUST_LOG="${RUST_LOG:-warn}" - export RUSTFS_VOLUMES="${VOLUME_DIR}/disk{1...4}" - export RUSTFS_ADDRESS=":${PORT}" - export RUSTFS_CONSOLE_ENABLE=false - export RUSTFS_CONSOLE_ADDRESS=":${CONSOLE_PORT}" - export RUSTFS_ROOT_USER="${ACCESS_KEY}" - export RUSTFS_ROOT_PASSWORD="${SECRET_KEY}" - export RUSTFS_OBJECT_CACHE_ENABLE=false - "${RUSTFS_BIN}" server >"${SERVER_LOG}" 2>&1 -) & -SERVER_PID=$! - -echo "Waiting for RustFS server to become ready..." -for _ in $(seq 1 60); do - if curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 || curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then - break - fi - sleep 1 -done - -if ! curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 && ! curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then - echo "RustFS did not become ready in time. See ${SERVER_LOG}" >&2 - exit 1 -fi - -echo "Preparing mc alias..." -MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" alias set bench "http://${HOST}" "${ACCESS_KEY}" "${SECRET_KEY}" >/dev/null -MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" mb --ignore-existing "bench/${BUCKET}" >/dev/null - -{ - echo "# Small PUT Benchmark Results" - echo - echo "- Run ID: \`${RUN_ID}\`" - echo "- Host: \`${HOST}\`" - echo "- Bucket: \`${BUCKET}\`" - if [[ "${OPS_PER_SIZE}" -gt 0 ]]; then - echo "- Ops per size: \`${OPS_PER_SIZE}\`" - else - echo "- Duration per size: \`${BENCH_SECONDS}s\`" - fi - echo "- Concurrency: \`${CONCURRENCY}\`" - echo "- Per request timeout: \`${PUT_TIMEOUT_SECS}s\`" - echo -} >"${RESULTS_MD}" - -for size_label in "${SIZES[@]}"; do - bytes="$(size_to_bytes "${size_label}")" - sample_file="$(prepare_sample_file "${size_label}" "${bytes}")" - raw_json="${RUN_DIR}/${size_label}.jsonl" - prefix="$(echo "${size_label}" | tr '[:upper:]' '[:lower:]')" - worker_dir="${RUN_DIR}/${size_label}-workers" - mkdir -p "${worker_dir}" - - echo "Running mc PUT benchmark for size ${size_label}..." - batch_start="$(now_secs)" - if [[ "${OPS_PER_SIZE}" -gt 0 ]]; then - seq 1 "${OPS_PER_SIZE}" | xargs -n1 -P "${CONCURRENCY}" -I{} \ - env MC_CONFIG_DIR="${MC_CONFIG_DIR}" MC_BIN="${MC_BIN}" PUT_TIMEOUT_SECS="${PUT_TIMEOUT_SECS}" SAMPLE_FILE="${sample_file}" BUCKET="${BUCKET}" PREFIX="${prefix}" /bin/bash -lc ' - idx="$1" - start=$(perl -MTime::HiRes=time -e '"'"'printf "%.6f\n", time'"'"') - if run_mc_put_with_timeout "${SAMPLE_FILE}" "bench/${BUCKET}/${PREFIX}/obj-${idx}.bin"; then - finish=$(perl -MTime::HiRes=time -e '"'"'printf "%.6f\n", time'"'"') - perl -e '"'"' - use strict; - use warnings; - my ($idx, $start, $finish) = @ARGV; - my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000); - print "{\"idx\":${idx},\"ok\":true,\"duration_ms\":${duration_ms}}\n"; - '"'"' -- "${idx}" "${start}" "${finish}" - else - finish=$(perl -MTime::HiRes=time -e '"'"'printf "%.6f\n", time'"'"') - perl -e '"'"' - use strict; - use warnings; - my ($idx, $start, $finish) = @ARGV; - my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000); - print "{\"idx\":${idx},\"ok\":false,\"duration_ms\":${duration_ms}}\n"; - '"'"' -- "${idx}" "${start}" "${finish}" - fi - ' _ {} >"${raw_json}" - else - bench_deadline="$(perl -e 'use strict; use warnings; my ($start, $secs) = @ARGV; printf "%.6f", $start + $secs;' -- "${batch_start}" "${BENCH_SECONDS}")" - pids=() - for worker in $(seq 1 "${CONCURRENCY}"); do - ( - out_file="${worker_dir}/worker-${worker}.jsonl" - idx=0 - while true; do - now="$(now_secs)" - if perl -e 'use strict; use warnings; exit(($ARGV[0] >= $ARGV[1]) ? 0 : 1)' -- "${now}" "${bench_deadline}"; then - break - fi - start="${now}" - key="bench/${BUCKET}/${prefix}/worker-${worker}/obj-${idx}.bin" - if run_mc_put_with_timeout "${sample_file}" "${key}"; then - finish="$(now_secs)" - perl -e ' - use strict; - use warnings; - my ($idx, $start, $finish) = @ARGV; - my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000); - print "{\"idx\":${idx},\"ok\":true,\"duration_ms\":${duration_ms}}\n"; - ' -- "${idx}" "${start}" "${finish}" >>"${out_file}" - else - finish="$(now_secs)" - perl -e ' - use strict; - use warnings; - my ($idx, $start, $finish) = @ARGV; - my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000); - print "{\"idx\":${idx},\"ok\":false,\"duration_ms\":${duration_ms}}\n"; - ' -- "${idx}" "${start}" "${finish}" >>"${out_file}" - fi - idx=$((idx + 1)) - done - ) & - pids+=($!) - done - for pid in "${pids[@]}"; do - wait "${pid}" - done - cat "${worker_dir}"/worker-*.jsonl >"${raw_json}" - fi - batch_finish="$(now_secs)" - - jq_summary="$(jq -s ' - def pct(p): - if length == 0 then null - else (sort_by(.duration_ms) | .[((length - 1) * p | floor)].duration_ms) - end; - { - total: length, - succeeded: (map(select(.ok == true)) | length), - failed: (map(select(.ok == false)) | length), - avg_ms: (if length == 0 then null else (map(.duration_ms) | add / length) end), - p50_ms: pct(0.50), - p90_ms: pct(0.90), - p99_ms: pct(0.99) - } - ' "${raw_json}")" - - success_count="$(echo "${jq_summary}" | jq -r '.succeeded')" - avg_ms="$(echo "${jq_summary}" | jq -r '.avg_ms')" - p50_ms="$(echo "${jq_summary}" | jq -r '.p50_ms')" - p90_ms="$(echo "${jq_summary}" | jq -r '.p90_ms')" - p99_ms="$(echo "${jq_summary}" | jq -r '.p99_ms')" - failed_count="$(echo "${jq_summary}" | jq -r '.failed')" - - wall_secs="$(perl -e 'use strict; use warnings; my ($start, $finish) = @ARGV; printf "%.6f", ($finish - $start);' -- "${batch_start}" "${batch_finish}")" - mib_per_sec="$(perl -e 'use strict; use warnings; my ($bytes, $count, $secs) = @ARGV; printf "%.3f", (($bytes * $count) / (1024 * 1024)) / $secs;' -- "${bytes}" "${success_count}" "${wall_secs}")" - obj_per_sec="$(perl -e 'use strict; use warnings; my ($count, $secs) = @ARGV; printf "%.3f", $count / $secs;' -- "${success_count}" "${wall_secs}")" - - { - echo "## ${size_label}" - echo - echo "- Successful PUTs: \`${success_count}\`" - echo "- Failed PUTs: \`${failed_count}\`" - echo "- Wall time: \`${wall_secs}s\`" - echo "- Throughput: \`${mib_per_sec} MiB/s\`" - echo "- Object rate: \`${obj_per_sec} obj/s\`" - echo "- Avg latency: \`${avg_ms} ms\`" - echo "- p50 latency: \`${p50_ms} ms\`" - echo "- p90 latency: \`${p90_ms} ms\`" - echo "- p99 latency: \`${p99_ms} ms\`" - echo - } >>"${RESULTS_MD}" -done - -echo -echo "Benchmark completed." -echo "Results: ${RESULTS_MD}" -echo "Server log: ${SERVER_LOG}" diff --git a/scripts/run.sh b/scripts/run.sh index 4d3e94dce..36c4f788c 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -38,8 +38,7 @@ mkdir -p ./target/volume/test{1..4} if [ -z "$RUST_LOG" ]; then export RUST_BACKTRACE=1 -# export RUST_LOG="info,rustfs=debug,rustfs_ecstore=info,s3s=debug,rustfs_iam=info,rustfs_notify=info" - export RUST_LOG="error" + export RUST_LOG="info,rustfs=debug,rustfs_ecstore=info,s3s=debug,rustfs_iam=info,rustfs_notify=info" fi # export RUSTFS_ERASURE_SET_DRIVE_COUNT=5 @@ -72,7 +71,7 @@ export RUSTFS_OBS_SERVICE_NAME=rustfs # Service name export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version export RUSTFS_OBS_ENVIRONMENT=production # Environment name development, staging, production export RUSTFS_OBS_LOGGER_LEVEL=info # Log level, supports trace, debug, info, warn, error -export RUSTFS_OBS_LOG_STDOUT_ENABLED=false # Whether to enable local stdout logging +export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging export RUSTFS_OBS_LOG_DIRECTORY="$current_dir/deploy/logs" # Log directory export RUSTFS_OBS_LOG_ROTATION_TIME="minutely" # Log rotation time unit, can be "minutely", "hourly", "daily" export RUSTFS_OBS_LOG_KEEP_FILES=10 # Number of log files to keep