From 32bf8f5bf30c21323c155dbcb6b2fe69ee0d62aa Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 7 Apr 2026 08:33:46 +0800 Subject: [PATCH] feat(storage): add direct chunk GET fast path (#2351) Signed-off-by: houseme Co-authored-by: heihutu Co-authored-by: cxymds --- Cargo.lock | 467 +-- Cargo.toml | 4 +- _typos.toml | 1 + crates/config/src/constants/zero_copy.rs | 28 + crates/e2e_test/Cargo.toml | 8 + crates/e2e_test/src/bin/small_put_bench.rs | 441 +++ crates/e2e_test/src/checksum_upload_test.rs | 276 +- crates/e2e_test/src/kms/kms_local_test.rs | 96 + crates/e2e_test/src/lib.rs | 4 + crates/e2e_test/src/range_request_test.rs | 80 + crates/ecstore/Cargo.toml | 14 + crates/ecstore/README.md | 37 + .../ecstore/benches/bitrot_chunk_benchmark.rs | 106 + .../ecstore/benches/direct_chunk_benchmark.rs | 390 +++ .../benches/reconstructed_chunk_benchmark.rs | 393 +++ crates/ecstore/run_benchmarks.sh | 19 +- crates/ecstore/src/bitrot.rs | 790 ++++- crates/ecstore/src/bucket/migration.rs | 14 +- crates/ecstore/src/config/com.rs | 20 +- crates/ecstore/src/config/storageclass.rs | 51 +- crates/ecstore/src/data_movement.rs | 23 +- crates/ecstore/src/disk/disk_store.rs | 9 + crates/ecstore/src/disk/local.rs | 858 +++++- crates/ecstore/src/disk/mod.rs | 12 + crates/ecstore/src/erasure_coding/bitrot.rs | 68 + crates/ecstore/src/erasure_coding/decode.rs | 245 +- crates/ecstore/src/erasure_coding/encode.rs | 365 ++- crates/ecstore/src/erasure_coding/erasure.rs | 282 +- crates/ecstore/src/erasure_coding/heal.rs | 2 +- crates/ecstore/src/rpc/remote_disk.rs | 31 +- crates/ecstore/src/set_disk.rs | 222 +- crates/ecstore/src/set_disk/read.rs | 1009 ++++++- crates/ecstore/src/set_disk/write.rs | 132 + crates/ecstore/src/sets.rs | 33 +- crates/ecstore/src/store.rs | 31 +- crates/ecstore/src/store/multipart.rs | 2 +- crates/ecstore/src/store/object.rs | 31 +- crates/ecstore/src/store_api.rs | 1 + crates/ecstore/src/store_api/readers.rs | 227 +- crates/ecstore/src/store_api/traits.rs | 10 +- crates/ecstore/src/store_api/types.rs | 16 +- crates/ecstore/src/tier/tier.rs | 13 +- crates/filemeta/src/filemeta.rs | 6 +- crates/heal/src/heal/storage.rs | 2 +- crates/heal/tests/heal_integration_test.rs | 4 +- crates/io-core/Cargo.toml | 2 + crates/io-core/src/adapter.rs | 124 + crates/io-core/src/chunk.rs | 276 ++ crates/io-core/src/lib.rs | 4 + crates/io-core/src/pool.rs | 42 +- crates/io-metrics/src/lib.rs | 598 ++-- crates/io-metrics/src/metric_names.rs | 55 +- crates/object-io/Cargo.toml | 37 + crates/object-io/src/get.rs | 1703 +++++++++++ crates/object-io/src/lib.rs | 16 + crates/object-io/src/put.rs | 1564 ++++++++++ crates/protocols/src/swift/object.rs | 10 +- crates/rio/src/checksum.rs | 40 +- crates/rio/src/compress_reader.rs | 94 + crates/rio/src/encrypt_reader.rs | 182 +- crates/rio/src/etag_reader.rs | 52 +- crates/rio/src/hardlimit_reader.rs | 57 + crates/rio/src/hash_reader.rs | 251 +- crates/rio/src/http_reader.rs | 104 +- crates/rio/src/lib.rs | 140 +- .../tests/lifecycle_integration_test.rs | 14 +- rustfs/Cargo.toml | 1 + .../src/app/lifecycle_transition_api_test.rs | 8 +- rustfs/src/app/multipart_usecase.rs | 173 +- rustfs/src/app/object_usecase.rs | 2623 +---------------- rustfs/src/app/object_usecase/app_adapters.rs | 616 ++++ .../src/app/object_usecase/get_object_flow.rs | 280 ++ .../object_usecase/get_object_zero_copy.rs | 338 +++ .../app/object_usecase/put_object_extract.rs | 499 ++++ .../src/app/object_usecase/put_object_flow.rs | 868 ++++++ rustfs/src/app/object_usecase/types.rs | 52 + .../src/app/object_usecase/zero_copy_tests.rs | 1222 ++++++++ rustfs/src/error.rs | 14 + rustfs/src/server/http.rs | 2 +- .../src/storage/concurrency/object_cache.rs | 94 + rustfs/src/storage/ecfs.rs | 39 - scripts/bench-small-put-local.sh | 152 + scripts/bench-small-put-mc-local.sh | 300 ++ scripts/run.sh | 5 +- 84 files changed, 15932 insertions(+), 3592 deletions(-) create mode 100644 crates/e2e_test/src/bin/small_put_bench.rs create mode 100644 crates/e2e_test/src/range_request_test.rs create mode 100644 crates/ecstore/benches/bitrot_chunk_benchmark.rs create mode 100644 crates/ecstore/benches/direct_chunk_benchmark.rs create mode 100644 crates/ecstore/benches/reconstructed_chunk_benchmark.rs create mode 100644 crates/io-core/src/adapter.rs create mode 100644 crates/io-core/src/chunk.rs create mode 100644 crates/object-io/Cargo.toml create mode 100644 crates/object-io/src/get.rs create mode 100644 crates/object-io/src/lib.rs create mode 100644 crates/object-io/src/put.rs create mode 100644 rustfs/src/app/object_usecase/app_adapters.rs create mode 100644 rustfs/src/app/object_usecase/get_object_flow.rs create mode 100644 rustfs/src/app/object_usecase/get_object_zero_copy.rs create mode 100644 rustfs/src/app/object_usecase/put_object_extract.rs create mode 100644 rustfs/src/app/object_usecase/put_object_flow.rs create mode 100644 rustfs/src/app/object_usecase/types.rs create mode 100644 rustfs/src/app/object_usecase/zero_copy_tests.rs create mode 100755 scripts/bench-small-put-local.sh create mode 100755 scripts/bench-small-put-mc-local.sh diff --git a/Cargo.lock b/Cargo.lock index 7199861d6..456df56fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -410,7 +410,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.13.0", + "indexmap 2.13.1", "itoa", "lexical-core", "memchr", @@ -687,9 +687,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.0" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" dependencies = [ "cc", "cmake", @@ -1227,16 +1227,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.2.17", + "cpufeatures 0.3.0", ] [[package]] @@ -1405,9 +1405,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.57" +version = "1.2.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" dependencies = [ "find-msvc-tools", "jobserver", @@ -1514,7 +1514,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.7", + "crypto-common 0.1.6", "inout 0.1.4", ] @@ -1582,18 +1582,18 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] [[package]] name = "cmov" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0758edba32d61d1fd9f4d69491b47604b91ee2f7e6b33de7e54ca4ebe55dc3" +checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" [[package]] name = "colorchoice" @@ -1971,9 +1971,9 @@ dependencies = [ [[package]] name = "crypto-bigint" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fde2467e74147f492aebb834985186b2c74761927b8b9b3bd303bcb2e72199d" +checksum = "42a0d26b245348befa0c121944541476763dcc46ede886c88f9d12e1697d27c3" dependencies = [ "cpubits", "ctutils", @@ -1985,9 +1985,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", "typenum", @@ -2010,7 +2010,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21f41f23de7d24cdbda7f0c4d9c0351f99a4ceb258ef30e5c1927af8987ffe5a" dependencies = [ - "crypto-bigint 0.7.1", + "crypto-bigint 0.7.3", "libm", "rand_core 0.10.0", ] @@ -2047,9 +2047,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1005a6d4446f5120ef475ad3d2af2b30c49c2c9c6904258e3bb30219bebed5e4" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" dependencies = [ "cmov", ] @@ -2325,7 +2325,7 @@ dependencies = [ "chrono", "half", "hashbrown 0.16.1", - "indexmap 2.13.0", + "indexmap 2.13.1", "itertools 0.14.0", "libc", "log", @@ -2529,7 +2529,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.13.0", + "indexmap 2.13.1", "itertools 0.14.0", "paste", "recursive", @@ -2545,7 +2545,7 @@ checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.13.0", + "indexmap 2.13.1", "itertools 0.14.0", "paste", ] @@ -2709,7 +2709,7 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap 2.13.0", + "indexmap 2.13.1", "itertools 0.14.0", "log", "recursive", @@ -2732,7 +2732,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.16.1", - "indexmap 2.13.0", + "indexmap 2.13.1", "itertools 0.14.0", "parking_lot 0.12.5", "paste", @@ -2768,7 +2768,7 @@ dependencies = [ "datafusion-common", "datafusion-expr-common", "hashbrown 0.16.1", - "indexmap 2.13.0", + "indexmap 2.13.1", "itertools 0.14.0", "parking_lot 0.12.5", ] @@ -2815,7 +2815,7 @@ dependencies = [ "futures", "half", "hashbrown 0.16.1", - "indexmap 2.13.0", + "indexmap 2.13.1", "itertools 0.14.0", "log", "num-traits", @@ -2867,7 +2867,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-functions-nested", - "indexmap 2.13.0", + "indexmap 2.13.1", "log", "recursive", "regex", @@ -2917,9 +2917,9 @@ dependencies = [ [[package]] name = "deflate64" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "807800ff3288b621186fe0a8f3392c4652068257302709c24efd918c3dffcdc2" +checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" [[package]] name = "der" @@ -3045,8 +3045,7 @@ dependencies = [ [[package]] name = "dial9-tokio-telemetry" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fab5b5b736126e4a4a3ed06e15389ac199c2ac4f72395197addb305e6ba1759" +source = "git+https://github.com/dial9-rs/dial9-tokio-telemetry.git?rev=60502082601b647c4a51962595721f631b7bbce1#60502082601b647c4a51962595721f631b7bbce1" dependencies = [ "arc-swap", "bon", @@ -3070,8 +3069,7 @@ dependencies = [ [[package]] name = "dial9-trace-format" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e0ee560b05f09bf817602d57644947e31e83c521d4e0277f723a6e64d44f92" +source = "git+https://github.com/dial9-rs/dial9-tokio-telemetry.git?rev=60502082601b647c4a51962595721f631b7bbce1#60502082601b647c4a51962595721f631b7bbce1" dependencies = [ "dial9-trace-format-derive", "serde", @@ -3080,8 +3078,7 @@ dependencies = [ [[package]] name = "dial9-trace-format-derive" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dbbd8126d4d6613931317cfe2a7275c1cd487e41c961e42456ab5f956570030" +source = "git+https://github.com/dial9-rs/dial9-tokio-telemetry.git?rev=60502082601b647c4a51962595721f631b7bbce1#60502082601b647c4a51962595721f631b7bbce1" dependencies = [ "proc-macro2", "quote", @@ -3102,7 +3099,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.7", + "crypto-common 0.1.6", "subtle", ] @@ -3182,10 +3179,13 @@ dependencies = [ "base64 0.22.1", "bytes", "chrono", + "clap", "flatbuffers", "flate2", "futures", "http 1.4.0", + "http-body 1.0.1", + "http-body-util", "md5", "rand 0.10.0", "rcgen", @@ -3353,18 +3353,18 @@ dependencies = [ [[package]] name = "env_filter" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" dependencies = [ "log", ] [[package]] name = "env_logger" -version = "0.11.9" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" dependencies = [ "env_filter", "log", @@ -3454,9 +3454,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "a043dc74da1e37d6afe657061213aa6f425f855399a11d3463c6ecccc4dfda1f" [[package]] name = "ff" @@ -3701,9 +3701,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -4051,7 +4051,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap 2.13.0", + "indexmap 2.13.1", "slab", "tokio", "tokio-util", @@ -4313,9 +4313,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hybrid-array" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8655f91cd07f2b9d0c24137bd650fe69617773435ee5ec83022377777ce65ef1" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" dependencies = [ "typenum", ] @@ -4425,12 +4425,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -4438,9 +4439,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -4451,9 +4452,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -4465,15 +4466,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -4485,15 +4486,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -4550,9 +4551,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" dependencies = [ "equivalent", "hashbrown 0.16.1", @@ -4567,7 +4568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash 0.8.12", - "indexmap 2.13.0", + "indexmap 2.13.1", "is-terminal", "itoa", "log", @@ -4590,7 +4591,7 @@ dependencies = [ "crossbeam-utils", "dashmap", "env_logger", - "indexmap 2.13.0", + "indexmap 2.13.1", "itoa", "log", "num-format", @@ -4650,9 +4651,9 @@ dependencies = [ [[package]] name = "iri-string" -version = "0.7.10" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" dependencies = [ "memchr", "serde", @@ -4781,7 +4782,7 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "jni-sys", + "jni-sys 0.3.1", "log", "thiserror 1.0.69", "walkdir", @@ -4790,9 +4791,31 @@ dependencies = [ [[package]] name = "jni-sys" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] [[package]] name = "jobserver" @@ -4806,10 +4829,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -4983,9 +5008,9 @@ dependencies = [ [[package]] name = "liblzma-sys" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186" +checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6" dependencies = [ "cc", "libc", @@ -5010,9 +5035,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" dependencies = [ "bitflags 2.11.0", "libc", @@ -5084,9 +5109,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "local-ip-address" @@ -5299,9 +5324,9 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.16.1", - "indexmap 2.13.0", + "indexmap 2.13.1", "metrics", - "ordered-float 5.1.0", + "ordered-float 5.3.0", "quanta", "radix_trie", "rand 0.9.2", @@ -5311,9 +5336,9 @@ dependencies = [ [[package]] name = "metrique" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3e5ecbbefec32dafed0fd98ef23768aaade6de35b8434fc3e44f6346b73cd6" +checksum = "c41212ded0b2ba808836b36db163067eb6da41447e7f224267a7331b670009b9" dependencies = [ "itoa", "jiff", @@ -5331,9 +5356,9 @@ dependencies = [ [[package]] name = "metrique-core" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad6478374c256ffbb0d2de67b7d93e43ac94e35a083f40bd5f72a9770f6110bb" +checksum = "a5f67f8bf383a36ddb563846a9b143228505fd1f833aa936fdefbc9cb1e46855" dependencies = [ "itertools 0.14.0", "metrique-writer-core", @@ -5341,9 +5366,9 @@ dependencies = [ [[package]] name = "metrique-macro" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83adb8929ae9b2f7a4ec07a04c3af569ffe22f96f02c89063e4a78895d6af760" +checksum = "07c50c41313eaa762e16c251aa3aa7f1eff170ab403fac8fc688840e93f39b18" dependencies = [ "Inflector", "darling 0.23.0", @@ -5354,24 +5379,24 @@ dependencies = [ [[package]] name = "metrique-service-metrics" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d01f36f47452cd6e33f66fc8185bb32f320aaa5721b6ad7230776442d3cf180" +checksum = "07a742784bddd4a8636cf4e952ec4bbd49d4dab34b7dd24d11026ab16adbfe19" dependencies = [ "metrique-writer", ] [[package]] name = "metrique-timesource" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60fb3f2836dffc05146f0dfe7bf2e0789909f3fefd72c729491adaef01acc1a" +checksum = "d607939211e4eaaa8cd35394fa5e57faffb7390d0ac513b39992edcaf3cc526c" [[package]] name = "metrique-writer" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d9ba4f5a6b5dd821f78315095840e88d244fafbdda3cf1688835cd2a56aec" +checksum = "8ea8f6776cef2fed8ebaea2044971e4bfec96268eb47ccccb0e93797a198b025" dependencies = [ "ahash 0.8.12", "crossbeam-queue", @@ -5390,9 +5415,9 @@ dependencies = [ [[package]] name = "metrique-writer-core" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642989d2c349dfcd705a0b6b63887459f71c8b8deb6dc79e39e12eaa17400aba" +checksum = "5399135c49a096ba9565872ddfef24627a4129501e930cbffe84d66897d4b12c" dependencies = [ "derive-where", "itertools 0.14.0", @@ -5402,9 +5427,9 @@ dependencies = [ [[package]] name = "metrique-writer-macro" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12edafee41e67f90ab2efe2b850e10751f0da3da4aeb61b8eb7e6c31666e8da8" +checksum = "7417c002f7b01c3d96792ff553b0b7e333059048a9e40d4f8d4bab23f570773e" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -5673,9 +5698,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-format" @@ -6046,9 +6071,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "5.1.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" dependencies = [ "num-traits", ] @@ -6315,7 +6340,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.13.0", + "indexmap 2.13.1", "serde", ] @@ -6398,7 +6423,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e" dependencies = [ "der 0.8.0", - "spki 0.8.0-rc.4", + "spki 0.8.0", ] [[package]] @@ -6428,7 +6453,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12922b6296c06eb741b02d7b5161e3aaa22864af38dfa025a1a3ba3f68c84577" dependencies = [ "der 0.8.0", - "spki 0.8.0-rc.4", + "spki 0.8.0", ] [[package]] @@ -6515,9 +6540,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -6745,7 +6770,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973" dependencies = [ "anyhow", - "indexmap 2.13.0", + "indexmap 2.13.1", "log", "protobuf", "protobuf-support", @@ -6785,9 +6810,9 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14104c5a24d9bcf7eb2c24753e0f49fe14555d8bd565ea3d38e4b4303267259d" +checksum = "7c3a14896dfa883796f1cb410461aef38810ea05f2b2c33c5aded3649095fdad" dependencies = [ "bitflags 2.11.0", "memchr", @@ -6922,7 +6947,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7471,14 +7496,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87ed3e93fc7e473e464b9726f4759659e72bc8665e4b8ea227547024f416d905" dependencies = [ "const-oid 0.10.2", - "crypto-bigint 0.7.1", + "crypto-bigint 0.7.3", "crypto-primes", "digest 0.11.2", "pkcs1 0.8.0-rc.4", "pkcs8 0.11.0-rc.11", "rand_core 0.10.0", "signature 3.0.0-rc.10", - "spki 0.8.0-rc.4", + "spki 0.8.0", "zeroize", ] @@ -7627,6 +7652,7 @@ dependencies = [ "rustfs-madmin", "rustfs-metrics", "rustfs-notify", + "rustfs-object-io", "rustfs-obs", "rustfs-policy", "rustfs-protocols", @@ -7827,6 +7853,7 @@ dependencies = [ "hyper-rustls", "hyper-util", "lazy_static", + "libc", "md-5 0.11.0", "memmap2 0.9.10", "metrics", @@ -7848,6 +7875,7 @@ dependencies = [ "rustfs-config", "rustfs-credentials", "rustfs-filemeta", + "rustfs-io-core", "rustfs-io-metrics", "rustfs-lock", "rustfs-madmin", @@ -7970,6 +7998,8 @@ name = "rustfs-io-core" version = "0.0.5" dependencies = [ "bytes", + "futures-core", + "futures-util", "memmap2 0.9.10", "rustfs-io-metrics", "thiserror 2.0.18", @@ -8139,6 +8169,31 @@ dependencies = [ "wildmatch", ] +[[package]] +name = "rustfs-object-io" +version = "0.0.5" +dependencies = [ + "astral-tokio-tar", + "atoi", + "bytes", + "futures-util", + "http 1.4.0", + "rustfs-concurrency", + "rustfs-ecstore", + "rustfs-io-core", + "rustfs-io-metrics", + "rustfs-rio", + "rustfs-s3select-api", + "rustfs-utils", + "s3s", + "serial_test", + "thiserror 2.0.18", + "time", + "tokio", + "tokio-util", + "uuid", +] + [[package]] name = "rustfs-obs" version = "0.0.5" @@ -8863,9 +8918,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ "serde", "serde_core", @@ -8983,7 +9038,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.13.0", + "indexmap 2.13.1", "schemars 0.9.0", "schemars 1.2.1", "serde_core", @@ -9164,9 +9219,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "simdutf8" @@ -9357,9 +9412,9 @@ dependencies = [ [[package]] name = "spki" -version = "0.8.0-rc.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8baeff88f34ed0691978ec34440140e1572b68c7dd4a495fd14a3dc1944daa80" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", "der 0.8.0", @@ -9507,9 +9562,9 @@ dependencies = [ [[package]] name = "symbolic-common" -version = "12.17.2" +version = "12.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751a2823d606b5d0a7616499e4130a516ebd01a44f39811be2b9600936509c23" +checksum = "52ca086c1eb5c7ee74b151ba83c6487d5d33f8c08ad991b86f3f58f6629e68d5" dependencies = [ "debugid", "memmap2 0.9.10", @@ -9519,9 +9574,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.17.2" +version = "12.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b237cfbe320601dd24b4ac817a5b68bb28f5508e33f08d42be0682cadc8ac9" +checksum = "baa911a28a62823aaf2cc2e074212492a3ee69d0d926cc8f5b12b4a108ff5c0c" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -9815,9 +9870,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -10002,7 +10057,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.13.0", + "indexmap 2.13.1", "pin-project-lite", "slab", "sync_wrapper", @@ -10253,9 +10308,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-width" @@ -10417,9 +10472,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" dependencies = [ "cfg-if", "once_cell", @@ -10430,23 +10485,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.64" +version = "0.4.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -10454,9 +10505,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" dependencies = [ "bumpalo", "proc-macro2", @@ -10467,9 +10518,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" dependencies = [ "unicode-ident", ] @@ -10491,7 +10542,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.0", + "indexmap 2.13.1", "wasm-encoder", "wasmparser", ] @@ -10517,15 +10568,15 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags 2.11.0", "hashbrown 0.15.5", - "indexmap 2.13.0", + "indexmap 2.13.1", "semver", ] [[package]] name = "web-sys" -version = "0.3.91" +version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" dependencies = [ "js-sys", "wasm-bindgen", @@ -10750,6 +10801,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -10783,13 +10843,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows-threading" version = "0.2.1" @@ -10811,6 +10888,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -10823,6 +10906,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -10835,12 +10924,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -10853,6 +10954,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -10865,6 +10972,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -10877,6 +10990,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -10889,6 +11008,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -10917,7 +11042,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.13.0", + "indexmap 2.13.1", "prettyplease", "syn 2.0.117", "wasm-metadata", @@ -10948,7 +11073,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags 2.11.0", - "indexmap 2.13.0", + "indexmap 2.13.1", "log", "serde", "serde_derive", @@ -10967,7 +11092,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.0", + "indexmap 2.13.1", "log", "semver", "serde", @@ -10991,9 +11116,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "x509-parser" @@ -11076,9 +11201,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -11087,9 +11212,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -11099,18 +11224,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.47" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.47" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", @@ -11119,18 +11244,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -11160,9 +11285,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -11171,9 +11296,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -11182,9 +11307,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", @@ -11205,7 +11330,7 @@ dependencies = [ "flate2", "getrandom 0.4.2", "hmac 0.12.1", - "indexmap 2.13.0", + "indexmap 2.13.1", "lzma-rust2", "memchr", "pbkdf2 0.12.2", diff --git a/Cargo.toml b/Cargo.toml index 1bc8933b2..3cd70f6c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ members = [ "crates/workers", # Worker thread pools and task scheduling "crates/io-metrics", # Zero-copy metrics collection for performance analysis "crates/io-core", # Zero-copy core reader and writer implementations + "crates/object-io", # Object I/O policy and zero-copy helper primitives "crates/zip", # ZIP file handling and compression ] resolver = "3" @@ -97,6 +98,7 @@ rustfs-metrics = { path = "crates/metrics", version = "0.0.5" } rustfs-notify = { path = "crates/notify", version = "0.0.5" } rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" } rustfs-io-core = { path = "crates/io-core", version = "0.0.5" } +rustfs-object-io = { path = "crates/object-io", version = "0.0.5" } rustfs-obs = { path = "crates/obs", version = "0.0.5" } rustfs-policy = { path = "crates/policy", version = "0.0.5" } rustfs-protos = { path = "crates/protos", version = "0.0.5" } @@ -284,7 +286,7 @@ zstd = "0.13.3" # Observability and Metrics metrics = "0.24.3" -dial9-tokio-telemetry = "0.2" +dial9-tokio-telemetry = { version = "0.2", git = "https://github.com/dial9-rs/dial9-tokio-telemetry.git", rev = "60502082601b647c4a51962595721f631b7bbce1" } opentelemetry = { version = "0.31.0" } opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] } opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] } diff --git a/_typos.toml b/_typos.toml index 2d1aa7e51..6ddfb7ab1 100644 --- a/_typos.toml +++ b/_typos.toml @@ -39,6 +39,7 @@ abd = "abd" mak = "mak" gae = "gae" GAE = "GAE" +writeable = "writeable" # s3-tests original test names (cannot be changed) nonexisted = "nonexisted" consts = "consts" diff --git a/crates/config/src/constants/zero_copy.rs b/crates/config/src/constants/zero_copy.rs index e931bd02c..ff23d6059 100644 --- a/crates/config/src/constants/zero_copy.rs +++ b/crates/config/src/constants/zero_copy.rs @@ -49,6 +49,34 @@ pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE"; /// to regular I/O without errors. pub const DEFAULT_OBJECT_ZERO_COPY_ENABLE: bool = true; +/// Environment variable for zero-copy read operating mode. +/// +/// Supported values: +/// - `off`: disable mmap-backed chunk fast path and always use the compatibility path +/// - `conservative`: allow a single mmap window per request +/// - `balanced`: allow multiple mmap windows with the default size guardrails +/// - `aggressive`: allow multi-window mmap and relax the small-object cutoff +pub const ENV_OBJECT_ZERO_COPY_MODE: &str = "RUSTFS_OBJECT_ZERO_COPY_MODE"; + +/// Default zero-copy read mode. +pub const DEFAULT_OBJECT_ZERO_COPY_MODE: &str = "balanced"; + +/// Environment variable for the maximum mmap window size used by the chunk fast path. +/// +/// This controls the visible bytes per mapped chunk before the implementation emits a new window. +pub const ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES"; + +/// Default mmap window size for chunk fast path reads: 8 MiB. +pub const DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: usize = 8 * 1024 * 1024; + +/// Environment variable for the maximum total active mmap bytes. +/// +/// Requests that would exceed this active window budget fall back to the compatibility path. +pub const ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES"; + +/// Default maximum active mmap bytes across concurrent local chunk fast-path reads: 256 MiB. +pub const DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: usize = 256 * 1024 * 1024; + // ============================================================================= // Direct I/O Configuration // ============================================================================= diff --git a/crates/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml index bc4d27721..cea060a54 100644 --- a/crates/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -20,6 +20,11 @@ license.workspace = true repository.workspace = true rust-version.workspace = true +[[bin]] +name = "small_put_bench" +path = "src/bin/small_put_bench.rs" +test = false + [lints] workspace = true @@ -41,6 +46,7 @@ serde_json.workspace = true tonic = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } +clap = { workspace = true } rustfs-madmin.workspace = true rustfs-filemeta.workspace = true bytes.workspace = true @@ -52,6 +58,8 @@ async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] } async-trait = { workspace = true } flate2.workspace = true http.workspace = true +http-body.workspace = true +http-body-util.workspace = true reqwest = { workspace = true } rustfs-signer.workspace = true tracing = { workspace = true } diff --git a/crates/e2e_test/src/bin/small_put_bench.rs b/crates/e2e_test/src/bin/small_put_bench.rs new file mode 100644 index 000000000..75b5864c5 --- /dev/null +++ b/crates/e2e_test/src/bin/small_put_bench.rs @@ -0,0 +1,441 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, anyhow, bail}; +use aws_sdk_s3::config::{Credentials, Region}; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{Delete, ObjectIdentifier}; +use aws_sdk_s3::{Client, Config}; +use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; +use bytes::Bytes; +use clap::Parser; +use serde::Serialize; +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +#[derive(Parser, Debug)] +#[command(name = "small_put_bench")] +#[command(about = "Rust-native small PUT benchmark for RustFS-compatible S3 endpoints")] +struct Args { + #[arg(long, env = "RUSTFS_BENCH_ENDPOINT")] + endpoint: String, + + #[arg(long, env = "RUSTFS_BENCH_ACCESS_KEY", default_value = "rustfsadmin")] + access_key: String, + + #[arg(long, env = "RUSTFS_BENCH_SECRET_KEY", default_value = "rustfsadmin")] + secret_key: String, + + #[arg(long, env = "RUSTFS_BENCH_REGION", default_value = "us-east-1")] + region: String, + + #[arg(long, env = "RUSTFS_BENCH_BUCKET", default_value = "small-put-benchmark")] + bucket: String, + + #[arg(long, env = "RUSTFS_BENCH_SIZES", default_value = "4KiB,16KiB,64KiB,256KiB,1MiB")] + sizes: String, + + #[arg(long, env = "RUSTFS_BENCH_CONCURRENCY", default_value_t = 8)] + concurrency: usize, + + #[arg(long, env = "RUSTFS_BENCH_DURATION_SECS", default_value_t = 10)] + duration_secs: u64, + + #[arg(long, env = "RUSTFS_BENCH_TIMEOUT_SECS", default_value_t = 15)] + timeout_secs: u64, + + #[arg(long, env = "RUSTFS_BENCH_PREFIX")] + prefix: Option, + + #[arg(long)] + output_json: Option, + + #[arg(long, default_value_t = false)] + cleanup: bool, +} + +#[derive(Clone, Debug)] +struct SizeSpec { + label: String, + slug: String, + bytes: usize, +} + +#[derive(Debug)] +struct Sample { + ok: bool, + duration_ms: f64, +} + +#[derive(Debug, Serialize)] +struct SizeSummary { + label: String, + bytes: usize, + total: usize, + succeeded: usize, + failed: usize, + wall_secs: f64, + object_rate: f64, + throughput_mib_per_sec: f64, + avg_ms: Option, + p50_ms: Option, + p90_ms: Option, + p99_ms: Option, +} + +#[derive(Debug, Serialize)] +struct RunSummary { + run_id: String, + endpoint: String, + bucket: String, + concurrency: usize, + duration_secs: u64, + timeout_secs: u64, + sizes: Vec, +} + +fn main() -> Result<()> { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .context("failed to build tokio runtime")?; + runtime.block_on(async_main()) +} + +async fn async_main() -> Result<()> { + let args = Args::parse(); + validate_args(&args)?; + + let sizes = parse_size_list(&args.sizes)?; + let run_id = args.prefix.clone().unwrap_or_else(default_run_id); + let client = build_s3_client(&args.endpoint, &args.access_key, &args.secret_key, &args.region); + + ensure_bucket(&client, &args.bucket).await?; + + let mut size_summaries = Vec::with_capacity(sizes.len()); + for size in &sizes { + let summary = run_size_benchmark( + client.clone(), + args.bucket.clone(), + run_id.clone(), + size.clone(), + args.concurrency, + Duration::from_secs(args.duration_secs), + Duration::from_secs(args.timeout_secs), + ) + .await?; + print_size_summary(&summary); + size_summaries.push(summary); + } + + if args.cleanup { + cleanup_prefix(&client, &args.bucket, &run_id).await?; + } + + let summary = RunSummary { + run_id, + endpoint: args.endpoint, + bucket: args.bucket, + concurrency: args.concurrency, + duration_secs: args.duration_secs, + timeout_secs: args.timeout_secs, + sizes: size_summaries, + }; + + if let Some(path) = args.output_json { + let json = serde_json::to_vec_pretty(&summary).context("failed to serialize benchmark summary")?; + std::fs::write(&path, json).with_context(|| format!("failed to write benchmark summary to {}", path.display()))?; + println!("Wrote summary to {}", path.display()); + } + + Ok(()) +} + +fn validate_args(args: &Args) -> Result<()> { + if args.concurrency == 0 { + bail!("--concurrency must be greater than zero"); + } + if args.duration_secs == 0 { + bail!("--duration-secs must be greater than zero"); + } + if args.timeout_secs == 0 { + bail!("--timeout-secs must be greater than zero"); + } + Ok(()) +} + +fn build_s3_client(endpoint: &str, access_key: &str, secret_key: &str, region: &str) -> Client { + let credentials = Credentials::new(access_key, secret_key, None, None, "small-put-bench"); + let mut config = Config::builder() + .credentials_provider(credentials) + .region(Region::new(region.to_string())) + .endpoint_url(endpoint) + .force_path_style(true) + .behavior_version_latest(); + + if endpoint.starts_with("http://") { + config = config.http_client(SmithyHttpClientBuilder::new().build_http()); + } + + Client::from_conf(config.build()) +} + +async fn ensure_bucket(client: &Client, bucket: &str) -> Result<()> { + if client.head_bucket().bucket(bucket).send().await.is_ok() { + return Ok(()); + } + + match client.create_bucket().bucket(bucket).send().await { + Ok(_) => Ok(()), + Err(err) => { + let rendered = err.to_string(); + if rendered.contains("BucketAlreadyOwnedByYou") || rendered.contains("BucketAlreadyExists") { + Ok(()) + } else { + Err(err).with_context(|| format!("failed to create benchmark bucket {bucket}")) + } + } + } +} + +async fn run_size_benchmark( + client: Client, + bucket: String, + run_id: String, + size: SizeSpec, + concurrency: usize, + duration: Duration, + timeout: Duration, +) -> Result { + let payload = Bytes::from(vec![0_u8; size.bytes]); + let deadline = Instant::now() + duration; + let wall_start = Instant::now(); + + let mut handles = Vec::with_capacity(concurrency); + for worker in 0..concurrency { + let client = client.clone(); + let bucket = bucket.clone(); + let payload = payload.clone(); + let prefix = format!("{run_id}/{}/worker-{worker}", size.slug); + handles.push(tokio::spawn(async move { + let mut samples = Vec::new(); + let mut idx = 0usize; + + while Instant::now() < deadline { + let key = format!("{prefix}/obj-{idx}.bin"); + let started_at = Instant::now(); + let request = client + .put_object() + .bucket(&bucket) + .key(key) + .body(ByteStream::from(payload.clone())) + .content_type("application/octet-stream"); + + let ok = matches!(tokio::time::timeout(timeout, request.send()).await, Ok(Ok(_))); + samples.push(Sample { + ok, + duration_ms: started_at.elapsed().as_secs_f64() * 1000.0, + }); + idx += 1; + } + + samples + })); + } + + let mut samples = Vec::new(); + for handle in handles { + samples.extend(handle.await.map_err(|err| anyhow!("benchmark worker join error: {err}"))?); + } + + Ok(build_size_summary(&size, samples, wall_start.elapsed())) +} + +fn build_size_summary(size: &SizeSpec, mut samples: Vec, wall_elapsed: Duration) -> SizeSummary { + let total = samples.len(); + let succeeded = samples.iter().filter(|sample| sample.ok).count(); + let failed = total.saturating_sub(succeeded); + let wall_secs = wall_elapsed.as_secs_f64(); + let object_rate = if wall_secs > 0.0 { succeeded as f64 / wall_secs } else { 0.0 }; + let throughput_mib_per_sec = if wall_secs > 0.0 { + ((size.bytes * succeeded) as f64 / (1024.0 * 1024.0)) / wall_secs + } else { + 0.0 + }; + + let avg_ms = if total > 0 { + Some(samples.iter().map(|sample| sample.duration_ms).sum::() / total as f64) + } else { + None + }; + + samples.sort_by(|lhs, rhs| lhs.duration_ms.total_cmp(&rhs.duration_ms)); + let durations: Vec = samples.into_iter().map(|sample| sample.duration_ms).collect(); + + SizeSummary { + label: size.label.clone(), + bytes: size.bytes, + total, + succeeded, + failed, + wall_secs, + object_rate, + throughput_mib_per_sec, + avg_ms, + p50_ms: percentile(&durations, 0.50), + p90_ms: percentile(&durations, 0.90), + p99_ms: percentile(&durations, 0.99), + } +} + +async fn cleanup_prefix(client: &Client, bucket: &str, prefix: &str) -> Result<()> { + let mut continuation_token = None; + loop { + let response = client + .list_objects_v2() + .bucket(bucket) + .prefix(prefix) + .set_continuation_token(continuation_token.clone()) + .send() + .await + .with_context(|| format!("failed to list objects for cleanup under {bucket}/{prefix}"))?; + + let objects: Vec = response + .contents + .unwrap_or_default() + .into_iter() + .filter_map(|object| object.key.map(|key| ObjectIdentifier::builder().key(key).build().ok())) + .flatten() + .collect(); + + for chunk in objects.chunks(1_000) { + if chunk.is_empty() { + continue; + } + + client + .delete_objects() + .bucket(bucket) + .delete( + Delete::builder() + .set_objects(Some(chunk.to_vec())) + .quiet(true) + .build() + .context("failed to build delete request")?, + ) + .send() + .await + .with_context(|| format!("failed to delete cleanup batch under {bucket}/{prefix}"))?; + } + + if response.is_truncated.unwrap_or(false) { + continuation_token = response.next_continuation_token; + } else { + break; + } + } + + Ok(()) +} + +fn parse_size_list(input: &str) -> Result> { + input + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(parse_size_spec) + .collect() +} + +fn parse_size_spec(input: &str) -> Result { + let normalized = input.trim(); + let lower = normalized.to_ascii_lowercase(); + + let (number_part, multiplier) = if let Some(value) = lower.strip_suffix("kib") { + (value, 1024usize) + } else if let Some(value) = lower.strip_suffix("mib") { + (value, 1024usize * 1024usize) + } else if let Some(value) = lower.strip_suffix('b') { + (value, 1usize) + } else { + (lower.as_str(), 1usize) + }; + + let value = number_part + .trim() + .parse::() + .with_context(|| format!("invalid size component: {input}"))?; + let bytes = value + .checked_mul(multiplier) + .ok_or_else(|| anyhow!("size overflow for {input}"))?; + + Ok(SizeSpec { + label: normalized.to_string(), + slug: normalized + .chars() + .filter(|ch| ch.is_ascii_alphanumeric()) + .collect::() + .to_ascii_lowercase(), + bytes, + }) +} + +fn percentile(values: &[f64], percentile: f64) -> Option { + if values.is_empty() { + return None; + } + + let index = ((values.len() - 1) as f64 * percentile).floor() as usize; + values.get(index).copied() +} + +fn default_run_id() -> String { + format!("small-put-bench-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S")) +} + +fn print_size_summary(summary: &SizeSummary) { + println!( + "{}: success={} failed={} obj/s={:.3} MiB/s={:.3} avg={:.3?} p50={:.3?} p90={:.3?} p99={:.3?}", + summary.label, + summary.succeeded, + summary.failed, + summary.object_rate, + summary.throughput_mib_per_sec, + summary.avg_ms, + summary.p50_ms, + summary.p90_ms, + summary.p99_ms, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_size_spec_supports_binary_units() { + let four_kib = parse_size_spec("4KiB").expect("4KiB should parse"); + assert_eq!(four_kib.bytes, 4 * 1024); + + let one_mib = parse_size_spec("1MiB").expect("1MiB should parse"); + assert_eq!(one_mib.bytes, 1024 * 1024); + } + + #[test] + fn percentile_returns_expected_bucket() { + let values = vec![10.0, 20.0, 30.0, 40.0, 50.0]; + assert_eq!(percentile(&values, 0.50), Some(30.0)); + assert_eq!(percentile(&values, 0.90), Some(40.0)); + } +} diff --git a/crates/e2e_test/src/checksum_upload_test.rs b/crates/e2e_test/src/checksum_upload_test.rs index 6be69df85..c15237a15 100644 --- a/crates/e2e_test/src/checksum_upload_test.rs +++ b/crates/e2e_test/src/checksum_upload_test.rs @@ -19,9 +19,13 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; - use aws_sdk_s3::primitives::ByteStream; + use aws_sdk_s3::primitives::{ByteStream, SdkBody}; use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart}; use base64::Engine; + use bytes::Bytes; + use futures::StreamExt; + use http_body::Frame; + use http_body_util::StreamBody; use rustfs_rio::{Checksum, ChecksumType as RioChecksumType}; use serial_test::serial; use sha2::{Digest, Sha256}; @@ -64,6 +68,53 @@ mod tests { .encoded } + fn streamed_body_70kib_of_a() -> ByteStream { + let bytes = Bytes::from_static(&[b'a'; 1024]); + let stream = futures::stream::repeat_with(move || { + let frame = Frame::data(bytes.clone()); + Ok::<_, std::io::Error>(frame) + }); + let body = WithSizeHint::new(StreamBody::new(stream.take(70)), 70 * 1024); + ByteStream::new(SdkBody::from_body_1_x(body)) + } + + struct WithSizeHint { + inner: T, + size_hint: usize, + } + + impl WithSizeHint { + fn new(inner: T, size_hint: usize) -> Self { + Self { inner, size_hint } + } + } + + impl http_body::Body for WithSizeHint + where + T: http_body::Body + Unpin, + { + type Data = T::Data; + type Error = T::Error; + + fn poll_frame( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll, Self::Error>>> { + let this = self.get_mut(); + std::pin::Pin::new(&mut this.inner).poll_frame(cx) + } + + fn is_end_stream(&self) -> bool { + self.inner.is_end_stream() + } + + fn size_hint(&self) -> http_body::SizeHint { + let mut hint = self.inner.size_hint(); + hint.set_exact(self.size_hint as u64); + hint + } + } + /// PutObject with Content-MD5: upload succeeds and GetObject returns same content. #[tokio::test] #[serial] @@ -136,6 +187,121 @@ mod tests { info!("PASSED: PutObject with checksum_sha256 and GetObject content match"); } + /// Mirrors `s3s-e2e` behavior: only request `checksum_algorithm`, then expect + /// both PutObject and GetObject(checksum_mode=enabled) to expose the same checksum. + #[tokio::test] + #[serial] + async fn test_put_object_with_checksum_algorithm_only() { + init_logging(); + info!("TEST: PutObject with checksum_algorithm only"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client(&env); + let bucket = "test-checksum-algorithm-only"; + create_bucket(&client, bucket).await.expect("Failed to create bucket"); + + let key = "obj-with-checksum-algorithm-only.txt"; + let content = vec![b'a'; 70 * 1024]; + + let put_resp = client + .put_object() + .bucket(bucket) + .key(key) + .checksum_algorithm(ChecksumAlgorithm::Crc32) + .body(ByteStream::from(content.clone())) + .send() + .await + .expect("PutObject with checksum_algorithm should succeed"); + + let put_checksum = put_resp + .checksum_crc32() + .expect("PutObject should return checksum_crc32 when checksum_algorithm is used") + .to_string(); + + let mut get_resp = client + .get_object() + .bucket(bucket) + .key(key) + .checksum_mode(ChecksumMode::Enabled) + .send() + .await + .expect("GetObject should succeed"); + + let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(aws_sdk_s3::primitives::SdkBody::empty())) + .collect() + .await + .expect("collect body") + .into_bytes(); + + assert_eq!(body_bytes.as_ref(), content.as_slice(), "GetObject body must match uploaded content"); + assert_eq!( + get_resp.checksum_crc32().map(str::to_string), + Some(put_checksum), + "GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum" + ); + } + + /// Matches the `s3s-e2e` streaming upload shape more closely than `ByteStream::from(Vec)`. + #[tokio::test] + #[serial] + async fn test_put_object_with_checksum_algorithm_only_streaming_body() { + init_logging(); + info!("TEST: PutObject with checksum_algorithm only using streaming body"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client(&env); + let bucket = "test-checksum-algorithm-streaming"; + create_bucket(&client, bucket).await.expect("Failed to create bucket"); + + let key = "obj-with-checksum-algorithm-streaming.txt"; + let expected_content = vec![b'a'; 70 * 1024]; + + let put_resp = client + .put_object() + .bucket(bucket) + .key(key) + .checksum_algorithm(ChecksumAlgorithm::Crc32) + .body(streamed_body_70kib_of_a()) + .send() + .await + .expect("PutObject with streaming checksum_algorithm should succeed"); + + let put_checksum = put_resp + .checksum_crc32() + .expect("PutObject should return checksum_crc32 for streaming checksum_algorithm uploads") + .to_string(); + + let mut get_resp = client + .get_object() + .bucket(bucket) + .key(key) + .checksum_mode(ChecksumMode::Enabled) + .send() + .await + .expect("GetObject should succeed"); + + let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(SdkBody::empty())) + .collect() + .await + .expect("collect body") + .into_bytes(); + + assert_eq!( + body_bytes.as_ref(), + expected_content.as_slice(), + "GetObject body must match uploaded content" + ); + assert_eq!( + get_resp.checksum_crc32().map(str::to_string), + Some(put_checksum), + "GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum for streaming uploads" + ); + } + /// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content. /// Uses part size >= 5MB (server minimum) for two parts. #[tokio::test] @@ -234,6 +400,114 @@ mod tests { info!("PASSED: MultipartUpload with checksum and GetObject content match"); } + /// Mirrors `s3s-e2e` multipart behavior: request checksum algorithm at MPU creation, + /// rely on auto checksum handling during UploadPart, and expect CompleteMultipartUpload to succeed. + #[tokio::test] + #[serial] + async fn test_multipart_upload_with_crc32_algorithm_only() { + init_logging(); + info!("TEST: MultipartUpload with checksum_algorithm only (CRC32)"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client(&env); + let bucket = "test-multipart-checksum-crc32-auto"; + create_bucket(&client, bucket).await.expect("Failed to create bucket"); + + let key = "multipart-with-crc32-auto.bin"; + let part1_content = "a".repeat(5 * 1024 * 1024 + 1); + let part2_content = "b".repeat(1024); + + let create_resp = client + .create_multipart_upload() + .bucket(bucket) + .key(key) + .checksum_algorithm(ChecksumAlgorithm::Crc32) + .send() + .await + .expect("CreateMultipartUpload should succeed"); + + let upload_id = create_resp.upload_id().expect("upload_id should be present"); + + let part1_resp = client + .upload_part() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .part_number(1) + .body(ByteStream::from(part1_content.clone().into_bytes())) + .send() + .await + .expect("UploadPart 1 should succeed"); + let part1_checksum = part1_resp + .checksum_crc32() + .expect("UploadPart 1 should return checksum_crc32") + .to_string(); + + let part2_resp = client + .upload_part() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .part_number(2) + .body(ByteStream::from(part2_content.clone().into_bytes())) + .send() + .await + .expect("UploadPart 2 should succeed"); + let part2_checksum = part2_resp + .checksum_crc32() + .expect("UploadPart 2 should return checksum_crc32") + .to_string(); + + let completed_upload = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(part1_resp.e_tag().expect("etag part 1")) + .checksum_crc32(part1_checksum) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(part2_resp.e_tag().expect("etag part 2")) + .checksum_crc32(part2_checksum) + .build(), + ) + .build(); + + client + .complete_multipart_upload() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .multipart_upload(completed_upload) + .send() + .await + .expect("CompleteMultipartUpload should succeed"); + + let body_bytes = client + .get_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect("GetObject should succeed") + .body + .collect() + .await + .expect("collect body") + .into_bytes(); + + let expected_content = format!("{part1_content}{part2_content}"); + assert_eq!( + body_bytes.as_ref(), + expected_content.as_bytes(), + "completed multipart object must match concatenated parts" + ); + } + /// Regression test for issue #2282: /// CRC64NVME full-object checksum should match between direct PutObject and multipart upload. #[tokio::test] diff --git a/crates/e2e_test/src/kms/kms_local_test.rs b/crates/e2e_test/src/kms/kms_local_test.rs index d3a24404b..208dbc6a2 100644 --- a/crates/e2e_test/src/kms/kms_local_test.rs +++ b/crates/e2e_test/src/kms/kms_local_test.rs @@ -344,6 +344,9 @@ async fn test_local_kms_multipart_upload() { test_multipart_upload_with_sse_c(&s3_client, TEST_BUCKET) .await .expect("SSE-C multipart upload test failed"); + test_multipart_download_with_wrong_sse_c_key_fails(&s3_client, TEST_BUCKET) + .await + .expect("SSE-C multipart wrong-key download test failed"); // Test 4: Large multipart upload (test streaming encryption with multiple blocks) // TODO: Re-enable after fixing streaming encryption issues with large files @@ -648,6 +651,99 @@ async fn test_multipart_upload_with_sse_c( Ok(()) } +async fn test_multipart_download_with_wrong_sse_c_key_fails( + s3_client: &aws_sdk_s3::Client, + bucket: &str, +) -> Result<(), Box> { + let object_key = "multipart-sse-c-bad-download-test"; + let part_size = 5 * 1024 * 1024; + let total_parts = 2; + let total_size = part_size * total_parts; + + let encryption_key = "01234567890123456789012345678901"; + let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key); + let key_md5 = sse_customer_key_md5_base64(encryption_key); + + let wrong_key = "abcdefghijklmnopqrstuvwxyz012345"; + let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key); + let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key); + + let test_data: Vec = (0..total_size).map(|i| ((i * 5) % 256) as u8).collect(); + + let create_multipart_output = s3_client + .create_multipart_upload() + .bucket(bucket) + .key(object_key) + .sse_customer_algorithm("AES256") + .sse_customer_key(&key_b64) + .sse_customer_key_md5(&key_md5) + .send() + .await?; + + let upload_id = create_multipart_output.upload_id().unwrap(); + let mut completed_parts = Vec::new(); + + for part_number in 1..=total_parts { + let start = (part_number - 1) * part_size; + let end = std::cmp::min(start + part_size, total_size); + let part_data = &test_data[start..end]; + + let upload_part_output = s3_client + .upload_part() + .bucket(bucket) + .key(object_key) + .upload_id(upload_id) + .part_number(part_number as i32) + .body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec())) + .sse_customer_algorithm("AES256") + .sse_customer_key(&key_b64) + .sse_customer_key_md5(&key_md5) + .send() + .await?; + + completed_parts.push( + aws_sdk_s3::types::CompletedPart::builder() + .part_number(part_number as i32) + .e_tag(upload_part_output.e_tag().unwrap()) + .build(), + ); + } + + let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder() + .set_parts(Some(completed_parts)) + .build(); + + s3_client + .complete_multipart_upload() + .bucket(bucket) + .key(object_key) + .upload_id(upload_id) + .multipart_upload(completed_multipart_upload) + .send() + .await?; + + let err = s3_client + .get_object() + .bucket(bucket) + .key(object_key) + .sse_customer_algorithm("AES256") + .sse_customer_key(&wrong_key_b64) + .sse_customer_key_md5(&wrong_key_md5) + .send() + .await + .expect_err("multipart SSE-C download with the wrong key should fail"); + + let service_err = err.into_service_error(); + assert_eq!( + service_err.meta().code(), + Some("InvalidRequest"), + "wrong-key multipart SSE-C download should return InvalidRequest, got {:?}", + service_err.meta().code() + ); + + Ok(()) +} + /// Test large multipart upload to verify streaming encryption works correctly #[allow(dead_code)] async fn test_large_multipart_upload( diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index be3f3758f..37b73d888 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -97,6 +97,10 @@ mod cluster_concurrency_test; #[cfg(test)] mod checksum_upload_test; +// Range request regression tests +#[cfg(test)] +mod range_request_test; + // Group deletion tests #[cfg(test)] mod group_delete_test; diff --git a/crates/e2e_test/src/range_request_test.rs b/crates/e2e_test/src/range_request_test.rs new file mode 100644 index 000000000..b5283129c --- /dev/null +++ b/crates/e2e_test/src/range_request_test.rs @@ -0,0 +1,80 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! End-to-end regression test for invalid GET object ranges. + +#[cfg(test)] +mod tests { + use crate::common::{RustFSTestEnvironment, init_logging}; + use aws_sdk_s3::Client; + use aws_sdk_s3::error::SdkError; + use aws_sdk_s3::primitives::ByteStream; + use serial_test::serial; + use tracing::info; + + fn create_s3_client(env: &RustFSTestEnvironment) -> Client { + env.create_s3_client() + } + + async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box> { + match client.create_bucket().bucket(bucket).send().await { + Ok(_) => Ok(()), + Err(err) => { + if err.to_string().contains("BucketAlreadyOwnedByYou") || err.to_string().contains("BucketAlreadyExists") { + Ok(()) + } else { + Err(Box::new(err)) + } + } + } + } + + #[tokio::test] + #[serial] + async fn test_get_object_invalid_range_returns_416_issue_s3_implemented_tests() { + init_logging(); + info!("TEST: GetObject invalid range should return InvalidRange/416"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client(&env); + let bucket = "test-invalid-range"; + let key = "range.txt"; + let content = b"testcontent"; + + create_bucket(&client, bucket).await.expect("Failed to create bucket"); + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(content)) + .send() + .await + .expect("PutObject should succeed"); + + let result = client.get_object().bucket(bucket).key(key).range("bytes=40-50").send().await; + + let err = result.expect_err("GetObject with an unsatisfiable range should fail"); + match err { + SdkError::ServiceError(service_err) => { + assert_eq!(service_err.raw().status().as_u16(), 416, "invalid range should return HTTP 416"); + + let s3_err = service_err.into_err(); + assert_eq!(s3_err.meta().code(), Some("InvalidRange"), "invalid range should map to InvalidRange"); + } + other_err => panic!("Expected S3 service error, got: {other_err:?}"), + } + } +} diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index e76563b09..1de8e70b3 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -70,6 +70,7 @@ reed-solomon-erasure = { workspace = true } reed-solomon-simd = { workspace = true } lazy_static.workspace = true rustfs-lock.workspace = true +rustfs-io-core.workspace = true rustfs-io-metrics.workspace = true regex = { workspace = true } path-absolutize = { workspace = true } @@ -97,6 +98,7 @@ rand.workspace = true pin-project-lite.workspace = true md-5.workspace = true memmap2 = { workspace = true } +libc.workspace = true rustfs-madmin.workspace = true rustfs-workers.workspace = true reqwest = { workspace = true } @@ -138,5 +140,17 @@ harness = false name = "comparison_benchmark" harness = false +[[bench]] +name = "direct_chunk_benchmark" +harness = false + +[[bench]] +name = "reconstructed_chunk_benchmark" +harness = false + +[[bench]] +name = "bitrot_chunk_benchmark" +harness = false + [lib] doctest = false diff --git a/crates/ecstore/README.md b/crates/ecstore/README.md index bcca1fa8a..4ced3a106 100644 --- a/crates/ecstore/README.md +++ b/crates/ecstore/README.md @@ -32,6 +32,43 @@ For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs). +## 📈 Benchmarks + +ECStore ships several Criterion benchmarks under [`crates/ecstore/benches/`](./benches/). + +### Direct Chunk Path + +Use the direct chunk benchmark to compare the current slice-forwarding path against the previous assembled-copy path: + +```bash +cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark +``` + +To run only the end-to-end ECStore range-read benchmark: + +```bash +cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark ecstore_get_object_chunks +``` + +To run the reconstructed multi-disk range-read benchmark: + +```bash +cargo bench -p rustfs-ecstore --bench reconstructed_chunk_benchmark +``` + +### Saved Comparison Points + +Latest local measurements on this branch: + +- `direct_chunk_path/slice_forwarding/single_block_aligned`: about `477 ns` +- `direct_chunk_path/assembled_copy/single_block_aligned`: about `3.25 us` +- `direct_chunk_path/slice_forwarding/multi_block_unaligned`: about `963 ns` +- `direct_chunk_path/assembled_copy/multi_block_unaligned`: about `7.25 us` +- `ecstore_get_object_chunks/drain/multi_disk_range`: about `644-654 us`, throughput about `2.86-2.90 GiB/s` +- `reconstructed_chunk_path/drain/multi_disk_missing_shard`: about `1.292-1.304 ms`, throughput about `1.43-1.45 GiB/s` + +These numbers are intended as branch-local reference points. Re-run the benchmark on your target machine before treating them as a regression baseline. + ## 📄 License This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details. diff --git a/crates/ecstore/benches/bitrot_chunk_benchmark.rs b/crates/ecstore/benches/bitrot_chunk_benchmark.rs new file mode 100644 index 000000000..76165c260 --- /dev/null +++ b/crates/ecstore/benches/bitrot_chunk_benchmark.rs @@ -0,0 +1,106 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytes::Bytes; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use rustfs_ecstore::bitrot::decode_bitrot_chunk_source_for_bench; +use rustfs_io_core::IoChunk; +use rustfs_utils::HashAlgorithm; +use std::hint::black_box; + +struct BitrotChunkBenchCase { + name: &'static str, + source_chunks: Vec, + shard_size: usize, + expected_decoded_len: usize, + expected_copied: bool, +} + +fn encode_shard(checksum_algo: HashAlgorithm, shard: &[u8]) -> Vec { + let mut encoded = Vec::with_capacity(checksum_algo.size() + shard.len()); + encoded.extend_from_slice(checksum_algo.hash_encode(shard).as_ref()); + encoded.extend_from_slice(shard); + encoded +} + +fn bitrot_chunk_bench_cases() -> [BitrotChunkBenchCase; 2] { + let checksum_algo = HashAlgorithm::Md5; + let shard_one = b"abcd"; + let shard_two = b"efgh"; + let encoded_one = encode_shard(checksum_algo.clone(), shard_one); + let encoded_two = encode_shard(checksum_algo.clone(), shard_two); + + let mut cross_chunk = Vec::with_capacity(encoded_one.len() + encoded_two.len()); + cross_chunk.extend_from_slice(&encoded_one); + cross_chunk.extend_from_slice(&encoded_two); + let split = checksum_algo.size() + 2; + + [ + BitrotChunkBenchCase { + name: "aligned_multi_chunk_no_copy", + source_chunks: vec![ + IoChunk::Shared(Bytes::from(encoded_one)), + IoChunk::Shared(Bytes::from(encoded_two)), + ], + shard_size: shard_one.len(), + expected_decoded_len: shard_one.len() + shard_two.len(), + expected_copied: false, + }, + BitrotChunkBenchCase { + name: "cross_chunk_frame_copy", + source_chunks: vec![ + IoChunk::Shared(Bytes::copy_from_slice(&cross_chunk[..split])), + IoChunk::Shared(Bytes::copy_from_slice(&cross_chunk[split..])), + ], + shard_size: shard_one.len(), + expected_decoded_len: shard_one.len() + shard_two.len(), + expected_copied: true, + }, + ] +} + +fn bench_bitrot_chunk_decode(c: &mut Criterion) { + let checksum_algo = HashAlgorithm::Md5; + let mut group = c.benchmark_group("bitrot_chunk_decode"); + group.sample_size(20); + + for case in bitrot_chunk_bench_cases() { + let (decoded, copied) = + decode_bitrot_chunk_source_for_bench(&case.source_chunks, case.shard_size, checksum_algo.clone(), false) + .expect("decode bitrot source"); + let decoded_len: usize = decoded.iter().map(IoChunk::len).sum(); + + assert_eq!(decoded_len, case.expected_decoded_len); + assert_eq!(copied, case.expected_copied); + + group.throughput(Throughput::Bytes(case.expected_decoded_len as u64)); + group.bench_with_input(BenchmarkId::new("decode", case.name), &case, |b, case| { + b.iter(|| { + let result = decode_bitrot_chunk_source_for_bench( + black_box(&case.source_chunks), + black_box(case.shard_size), + checksum_algo.clone(), + false, + ) + .expect("decode bitrot source"); + black_box(result); + }); + }); + } + + group.finish(); +} + +criterion_group!(benches, bench_bitrot_chunk_decode); +criterion_main!(benches); diff --git a/crates/ecstore/benches/direct_chunk_benchmark.rs b/crates/ecstore/benches/direct_chunk_benchmark.rs new file mode 100644 index 000000000..7d45841f3 --- /dev/null +++ b/crates/ecstore/benches/direct_chunk_benchmark.rs @@ -0,0 +1,390 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytes::{Bytes, BytesMut}; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use futures_util::StreamExt; +use futures_util::stream; +use http::HeaderMap; +use rustfs_ecstore::bucket::metadata_sys; +use rustfs_ecstore::disk::endpoint::Endpoint; +use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; +use rustfs_ecstore::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES}; +use rustfs_ecstore::set_disk::collect_direct_data_shard_chunks_for_benchmark; +use rustfs_ecstore::store::{ECStore, init_local_disks}; +use rustfs_ecstore::store_api::{ + BucketOperations, BucketOptions, ChunkNativePutData, GetObjectChunkCopyMode, HTTPRangeSpec, MakeBucketOptions, ObjectIO, + ObjectOptions, +}; +use rustfs_io_core::{BoxChunkStream, IoChunk, MappedChunk}; +use std::hint::black_box; +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU16, Ordering}; +use tempfile::TempDir; +use tokio::runtime::Runtime; +use tokio_util::sync::CancellationToken; + +#[derive(Clone)] +struct BenchCase { + name: &'static str, + data_shards: usize, + block_size: usize, + blocks: usize, + offset: usize, + length: usize, +} + +fn bench_cases() -> [BenchCase; 2] { + [ + BenchCase { + name: "single_block_aligned", + data_shards: 4, + block_size: 256 * 1024, + blocks: 1, + offset: 0, + length: 256 * 1024, + }, + BenchCase { + name: "multi_block_unaligned", + data_shards: 4, + block_size: 256 * 1024, + blocks: 8, + offset: 123_457, + length: 2 * 256 * 1024 + 33_333, + }, + ] +} + +#[derive(Clone)] +struct EcstoreBenchCase { + name: &'static str, + disk_count: usize, + payload_len: usize, + range: HTTPRangeSpec, + expected_copy_mode: GetObjectChunkCopyMode, +} + +struct EcstoreBenchEnv { + _temp_dir: TempDir, + store: Arc, + bucket: String, + key: String, + range: HTTPRangeSpec, + opts: ObjectOptions, + expected_len: usize, + expected_copy_mode: GetObjectChunkCopyMode, +} + +fn ecstore_bench_cases() -> [EcstoreBenchCase; 1] { + [EcstoreBenchCase { + name: "multi_disk_range", + disk_count: 4, + payload_len: 3 * 1024 * 1024 + 137, + range: HTTPRangeSpec { + is_suffix_length: false, + start: 123_457, + end: 2 * 1024 * 1024 + 33_333, + }, + expected_copy_mode: expected_direct_copy_mode(), + }] +} + +#[cfg(unix)] +const fn expected_direct_copy_mode() -> GetObjectChunkCopyMode { + GetObjectChunkCopyMode::TrueZeroCopy +} + +#[cfg(not(unix))] +const fn expected_direct_copy_mode() -> GetObjectChunkCopyMode { + GetObjectChunkCopyMode::SharedBytes +} + +fn clone_range_spec(range: &HTTPRangeSpec) -> HTTPRangeSpec { + HTTPRangeSpec { + is_suffix_length: range.is_suffix_length, + start: range.start, + end: range.end, + } +} + +fn next_loopback_addr() -> SocketAddr { + static NEXT_PORT: AtomicU16 = AtomicU16::new(39013); + let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); + SocketAddr::from(([127, 0, 0, 1], port)) +} + +fn build_endpoint_pools(paths: &[std::path::PathBuf]) -> EndpointServerPools { + let mut endpoints = Vec::with_capacity(paths.len()); + for (idx, disk_path) in paths.iter().enumerate() { + let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf8 path")).expect("endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(idx); + endpoints.push(endpoint); + } + + EndpointServerPools(vec![PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: paths.len(), + endpoints: Endpoints::from(endpoints), + cmd_line: "bench".to_string(), + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }]) +} + +async fn build_ecstore_bench_env(case: &EcstoreBenchCase) -> EcstoreBenchEnv { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let mut disk_paths = Vec::with_capacity(case.disk_count); + for idx in 0..case.disk_count { + let path = temp_dir.path().join(format!("disk{}", idx + 1)); + tokio::fs::create_dir_all(&path).await.expect("create disk dir"); + disk_paths.push(path); + } + + let endpoint_pools = build_endpoint_pools(&disk_paths); + GLOBAL_LOCAL_DISK_MAP.write().await.clear(); + GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear(); + GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear(); + init_local_disks(endpoint_pools.clone()).await.expect("init local disks"); + + let store = ECStore::new(next_loopback_addr(), endpoint_pools, CancellationToken::new()) + .await + .expect("create ecstore"); + + let buckets = store + .list_bucket(&BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + .expect("list buckets") + .into_iter() + .map(|bucket| bucket.name) + .collect(); + metadata_sys::init_bucket_metadata_sys(store.clone(), buckets).await; + + let object_id = case.name.replace('_', "-"); + let bucket = format!("bench-direct-{object_id}"); + let key = format!("objects/{object_id}.bin"); + store + .make_bucket( + &bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("make bucket"); + + let payload: Vec = (0..case.payload_len).map(|idx| (idx % 251) as u8).collect(); + let mut reader = ChunkNativePutData::from_vec(payload); + let put_info = store + .put_object(&bucket, &key, &mut reader, &ObjectOptions::default()) + .await + .expect("put object"); + if case.disk_count > 1 { + assert!(put_info.data_blocks > 1, "expected multi-data-shard object"); + } + + let (_, expected_len) = case.range.get_offset_length(case.payload_len as i64).expect("range length"); + + EcstoreBenchEnv { + _temp_dir: temp_dir, + store, + bucket, + key, + range: clone_range_spec(&case.range), + opts: ObjectOptions::default(), + expected_len: expected_len as usize, + expected_copy_mode: case.expected_copy_mode, + } +} + +async fn run_ecstore_get_object_chunks_bench(env: &EcstoreBenchEnv) -> (GetObjectChunkCopyMode, usize, usize) { + let mut result = env + .store + .get_object_chunks(&env.bucket, &env.key, Some(clone_range_spec(&env.range)), HeaderMap::new(), &env.opts) + .await + .expect("get object chunks"); + let copy_mode = result.copy_mode; + let mut total_len = 0usize; + let mut chunk_count = 0usize; + while let Some(chunk) = result.stream.next().await { + let chunk = chunk.expect("chunk"); + total_len += chunk.len(); + chunk_count += 1; + } + + (copy_mode, total_len, chunk_count) +} + +fn build_shard_bytes(case: &BenchCase) -> Vec> { + let total_len = case.block_size * case.blocks; + let payload: Vec = (0..total_len).map(|idx| (idx % 251) as u8).collect(); + let mut shards = vec![Vec::with_capacity(case.blocks); case.data_shards]; + + for block in 0..case.blocks { + let block_start = block * case.block_size; + let block_slice = &payload[block_start..block_start + case.block_size]; + let shard_width = case.block_size / case.data_shards; + for (shard_idx, shard) in shards.iter_mut().enumerate().take(case.data_shards) { + let shard_start = shard_idx * shard_width; + let shard_end = shard_start + shard_width; + shard.push(Bytes::copy_from_slice(&block_slice[shard_start..shard_end])); + } + } + + shards +} + +fn build_mapped_streams(shards: &[Vec]) -> Vec { + shards + .iter() + .map(|shard_chunks| { + let chunks: Vec<_> = shard_chunks + .iter() + .map(|chunk| { + let mapped = MappedChunk::new(chunk.clone(), 0, chunk.len()).expect("mapped chunk"); + Ok::(IoChunk::Mapped(mapped)) + }) + .collect(); + Box::pin(stream::iter(chunks)) as BoxChunkStream + }) + .collect() +} + +fn collect_old_assembly( + shards: &[Vec], + data_shards: usize, + block_size: usize, + offset: usize, + length: usize, +) -> Vec { + if length == 0 { + return Vec::new(); + } + + let start_block = offset / block_size; + let end_block = offset.saturating_add(length - 1) / block_size; + let mut result = Vec::with_capacity(end_block - start_block + 1); + + for block_index in start_block..=end_block { + let block_offset = if block_index == start_block { offset % block_size } else { 0 }; + let block_length = if start_block == end_block { + length + } else if block_index == start_block { + block_size - (offset % block_size) + } else if block_index == end_block { + (offset + length) % block_size + } else { + block_size + }; + + if block_length == 0 { + break; + } + + let mut block = BytesMut::with_capacity(block_length); + let mut write_left = block_length; + let mut skip = block_offset; + + for shard in shards.iter().take(data_shards) { + let shard_chunk = &shard[block_index]; + if skip >= shard_chunk.len() { + skip -= shard_chunk.len(); + continue; + } + + let available = &shard_chunk[skip..]; + skip = 0; + let take = available.len().min(write_left); + block.extend_from_slice(&available[..take]); + write_left -= take; + + if write_left == 0 { + break; + } + } + + result.push(IoChunk::Shared(block.freeze())); + } + + result +} + +fn bench_direct_chunk_path(c: &mut Criterion) { + let runtime = Runtime::new().expect("tokio runtime"); + let mut group = c.benchmark_group("direct_chunk_path"); + + for case in bench_cases() { + let shard_bytes = build_shard_bytes(&case); + group.throughput(Throughput::Bytes(case.length as u64)); + + group.bench_with_input(BenchmarkId::new("slice_forwarding", case.name), &case, |b, case| { + b.iter(|| { + let streams = build_mapped_streams(&shard_bytes); + let chunks = runtime + .block_on(collect_direct_data_shard_chunks_for_benchmark( + streams, + case.data_shards, + case.block_size, + case.blocks * case.block_size, + false, + case.offset, + case.length, + )) + .expect("collect direct chunks"); + black_box(chunks); + }); + }); + + group.bench_with_input(BenchmarkId::new("assembled_copy", case.name), &case, |b, case| { + b.iter(|| { + let chunks = collect_old_assembly(&shard_bytes, case.data_shards, case.block_size, case.offset, case.length); + black_box(chunks); + }); + }); + } + + group.finish(); +} + +fn bench_ecstore_get_object_chunks(c: &mut Criterion) { + let runtime = Runtime::new().expect("tokio runtime"); + let mut group = c.benchmark_group("ecstore_get_object_chunks"); + group.sample_size(10); + + for case in ecstore_bench_cases() { + let env = runtime.block_on(build_ecstore_bench_env(&case)); + let (copy_mode, total_len, _) = runtime.block_on(run_ecstore_get_object_chunks_bench(&env)); + assert_eq!(copy_mode, env.expected_copy_mode); + assert_eq!(total_len, env.expected_len); + + group.throughput(Throughput::Bytes(env.expected_len as u64)); + group.bench_with_input(BenchmarkId::new("drain", case.name), &env, |b, env| { + b.iter(|| { + let result = runtime.block_on(run_ecstore_get_object_chunks_bench(env)); + black_box(result); + }); + }); + } + + group.finish(); +} + +criterion_group!(benches, bench_direct_chunk_path, bench_ecstore_get_object_chunks); +criterion_main!(benches); diff --git a/crates/ecstore/benches/reconstructed_chunk_benchmark.rs b/crates/ecstore/benches/reconstructed_chunk_benchmark.rs new file mode 100644 index 000000000..d77b0d9f3 --- /dev/null +++ b/crates/ecstore/benches/reconstructed_chunk_benchmark.rs @@ -0,0 +1,393 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use futures_util::StreamExt; +use http::HeaderMap; +use rustfs_ecstore::bucket::metadata_sys; +use rustfs_ecstore::disk::endpoint::Endpoint; +use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; +use rustfs_ecstore::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES}; +use rustfs_ecstore::store::{ECStore, init_local_disks}; +use rustfs_ecstore::store_api::{ + BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, GetObjectChunkCopyMode, HTTPRangeSpec, MakeBucketOptions, + MultipartOperations, ObjectOperations, ObjectOptions, +}; +use std::hint::black_box; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU16, Ordering}; +use tempfile::TempDir; +use tokio::runtime::Runtime; +use tokio_util::sync::CancellationToken; + +#[derive(Clone)] +enum ReconstructedReadSpec { + PartNumber { + part_number: usize, + missing_part_name: &'static str, + }, + Range { + start: u64, + end: u64, + missing_part_name: &'static str, + }, +} + +#[derive(Clone)] +struct ReconstructedBenchCase { + object_id: &'static str, + name: &'static str, + read_spec: ReconstructedReadSpec, +} + +struct ReconstructedBenchEnv { + store: Arc, + bucket: String, + key: String, + range: HTTPRangeSpec, + opts: ObjectOptions, + expected_len: usize, +} + +struct ReconstructedBenchSuite { + _temp_dir: TempDir, + store: Arc, + disk_paths: Vec, +} + +const MULTIPART_PART_ONE_LEN: usize = 5 * 1024 * 1024; +const MULTIPART_PART_TWO_LEN: usize = 5 * 1024 * 1024 + 137; +const MULTIPART_PART_THREE_LEN: usize = 1024 * 1024 + 77; + +fn reconstructed_bench_cases() -> [ReconstructedBenchCase; 4] { + [ + ReconstructedBenchCase { + object_id: "mp-part2", + name: "multi_disk_missing_shard_multipart_part2", + read_spec: ReconstructedReadSpec::PartNumber { + part_number: 2, + missing_part_name: "part.2", + }, + }, + ReconstructedBenchCase { + object_id: "mp-cross-range", + name: "multi_disk_missing_shard_multipart_cross_part_range", + read_spec: ReconstructedReadSpec::Range { + start: (MULTIPART_PART_ONE_LEN - 32 * 1024) as u64, + end: (MULTIPART_PART_ONE_LEN + 96 * 1024) as u64, + missing_part_name: "part.2", + }, + }, + ReconstructedBenchCase { + object_id: "mp-part3", + name: "multi_disk_missing_shard_multipart_part3", + read_spec: ReconstructedReadSpec::PartNumber { + part_number: 3, + missing_part_name: "part.3", + }, + }, + ReconstructedBenchCase { + object_id: "mp-cross-final-range", + name: "multi_disk_missing_shard_multipart_cross_final_part_range", + read_spec: ReconstructedReadSpec::Range { + start: (MULTIPART_PART_ONE_LEN + MULTIPART_PART_TWO_LEN - 32 * 1024) as u64, + end: (MULTIPART_PART_ONE_LEN + MULTIPART_PART_TWO_LEN + 96 * 1024) as u64, + missing_part_name: "part.3", + }, + }, + ] +} + +fn next_loopback_addr() -> SocketAddr { + static NEXT_PORT: AtomicU16 = AtomicU16::new(39113); + let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); + SocketAddr::from(([127, 0, 0, 1], port)) +} + +fn clone_range_spec(range: &HTTPRangeSpec) -> HTTPRangeSpec { + HTTPRangeSpec { + is_suffix_length: range.is_suffix_length, + start: range.start, + end: range.end, + } +} + +fn build_endpoint_pools(paths: &[PathBuf]) -> EndpointServerPools { + let mut endpoints = Vec::with_capacity(paths.len()); + for (idx, disk_path) in paths.iter().enumerate() { + let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf8 path")).expect("endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(idx); + endpoints.push(endpoint); + } + + EndpointServerPools(vec![PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: paths.len(), + endpoints: Endpoints::from(endpoints), + cmd_line: "bench".to_string(), + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }]) +} + +fn find_part_files(root: &Path, part_name: &str, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(root) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + find_part_files(&path, part_name, out); + continue; + } + + if path.file_name().and_then(|name| name.to_str()) == Some(part_name) { + out.push(path); + } + } +} + +async fn remove_part_files(root: &Path, part_name: &str) -> Vec<(PathBuf, Vec)> { + let mut paths = Vec::new(); + find_part_files(root, part_name, &mut paths); + + let mut removed = Vec::with_capacity(paths.len()); + for path in paths { + let content = tokio::fs::read(&path).await.expect("read part file before removal"); + tokio::fs::remove_file(&path).await.expect("remove part file"); + removed.push((path, content)); + } + + removed +} + +async fn restore_part_files(files: Vec<(PathBuf, Vec)>) { + for (path, content) in files { + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent) + .await + .expect("ensure parent for part restore"); + } + tokio::fs::write(&path, content).await.expect("restore part file"); + } +} + +async fn run_reconstructed_get_object_chunks( + store: &Arc, + bucket: &str, + key: &str, + range: &HTTPRangeSpec, + opts: &ObjectOptions, +) -> (GetObjectChunkCopyMode, usize, usize) { + let mut result = store + .get_object_chunks(bucket, key, Some(clone_range_spec(range)), HeaderMap::new(), opts) + .await + .expect("get object chunks"); + + let copy_mode = result.copy_mode; + let mut total_len = 0usize; + let mut chunk_count = 0usize; + while let Some(chunk) = result.stream.next().await { + let chunk = chunk.expect("chunk"); + total_len += chunk.len(); + chunk_count += 1; + } + + (copy_mode, total_len, chunk_count) +} + +async fn create_multipart_object(store: &Arc, bucket: &str, key: &str, parts: &[Vec]) -> usize { + let upload = store + .new_multipart_upload(bucket, key, &ObjectOptions::default()) + .await + .expect("new multipart upload"); + + let mut completed_parts = Vec::with_capacity(parts.len()); + for (idx, part) in parts.iter().enumerate() { + let mut reader = ChunkNativePutData::from_vec(part.clone()); + let part_info = store + .put_object_part(bucket, key, &upload.upload_id, idx + 1, &mut reader, &ObjectOptions::default()) + .await + .expect("put object part"); + completed_parts.push(CompletePart { + part_num: idx + 1, + etag: part_info.etag, + ..Default::default() + }); + } + + store + .clone() + .complete_multipart_upload(bucket, key, &upload.upload_id, completed_parts, &ObjectOptions::default()) + .await + .expect("complete multipart upload"); + + parts.iter().map(Vec::len).sum() +} + +async fn build_reconstructed_bench_suite() -> ReconstructedBenchSuite { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let disk_paths: Vec<_> = (1..=4).map(|idx| temp_dir.path().join(format!("disk{idx}"))).collect(); + for disk_path in &disk_paths { + tokio::fs::create_dir_all(disk_path).await.expect("create disk dir"); + } + + let endpoint_pools = build_endpoint_pools(&disk_paths); + GLOBAL_LOCAL_DISK_MAP.write().await.clear(); + GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear(); + GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear(); + init_local_disks(endpoint_pools.clone()).await.expect("init local disks"); + + let store = ECStore::new(next_loopback_addr(), endpoint_pools, CancellationToken::new()) + .await + .expect("create ecstore"); + + let buckets = store + .list_bucket(&BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + .expect("list buckets") + .into_iter() + .map(|bucket| bucket.name) + .collect(); + metadata_sys::init_bucket_metadata_sys(store.clone(), buckets).await; + + ReconstructedBenchSuite { + _temp_dir: temp_dir, + store, + disk_paths, + } +} + +async fn build_reconstructed_bench_env(suite: &ReconstructedBenchSuite, case: &ReconstructedBenchCase) -> ReconstructedBenchEnv { + let bucket = format!("bench-r-{}", case.object_id); + let key = format!("objects/{}.bin", case.object_id); + suite + .store + .make_bucket( + &bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("make bucket"); + + let part_one: Vec = (0..MULTIPART_PART_ONE_LEN).map(|idx| (idx % 251) as u8).collect(); + let part_two: Vec = (0..MULTIPART_PART_TWO_LEN).map(|idx| ((idx + 11) % 251) as u8).collect(); + let part_three: Vec = (0..MULTIPART_PART_THREE_LEN).map(|idx| ((idx + 29) % 251) as u8).collect(); + let payload_len = create_multipart_object(&suite.store, &bucket, &key, &[part_one, part_two, part_three]).await; + let info = suite + .store + .get_object_info(&bucket, &key, &ObjectOptions::default()) + .await + .expect("get object info"); + let (range, opts, missing_part_name) = match case.read_spec.clone() { + ReconstructedReadSpec::PartNumber { + part_number, + missing_part_name, + } => { + let range = HTTPRangeSpec::from_object_info(&info, part_number).expect("part_number range"); + let opts = ObjectOptions { + part_number: Some(part_number), + ..Default::default() + }; + (range, opts, missing_part_name) + } + ReconstructedReadSpec::Range { + start, + end, + missing_part_name, + } => ( + HTTPRangeSpec { + is_suffix_length: false, + start: start as i64, + end: end as i64, + }, + ObjectOptions::default(), + missing_part_name, + ), + }; + + let (_, expected_len) = range.get_offset_length(payload_len as i64).expect("range length"); + let mut selected_reconstructed = false; + for disk_path in &suite.disk_paths { + let object_root = disk_path + .join(&bucket) + .join("objects") + .join(format!("{}.bin", case.object_id)); + let removed_parts = remove_part_files(&object_root, missing_part_name).await; + if removed_parts.is_empty() { + continue; + } + + let (copy_mode, total_len, _) = run_reconstructed_get_object_chunks(&suite.store, &bucket, &key, &range, &opts).await; + if copy_mode == GetObjectChunkCopyMode::Reconstructed && total_len == expected_len as usize { + selected_reconstructed = true; + break; + } + + restore_part_files(removed_parts).await; + } + assert!( + selected_reconstructed, + "failed to select a missing shard placement that triggers reconstructed copy mode for benchmark case {}", + case.name + ); + + ReconstructedBenchEnv { + store: suite.store.clone(), + bucket, + key, + range: clone_range_spec(&range), + opts, + expected_len: expected_len as usize, + } +} + +async fn run_reconstructed_get_object_chunks_bench(env: &ReconstructedBenchEnv) -> (GetObjectChunkCopyMode, usize, usize) { + run_reconstructed_get_object_chunks(&env.store, &env.bucket, &env.key, &env.range, &env.opts).await +} + +fn bench_reconstructed_chunk_path(c: &mut Criterion) { + let runtime = Runtime::new().expect("tokio runtime"); + let suite = runtime.block_on(build_reconstructed_bench_suite()); + let mut group = c.benchmark_group("reconstructed_chunk_path"); + group.sample_size(10); + for case in reconstructed_bench_cases() { + let env = runtime.block_on(build_reconstructed_bench_env(&suite, &case)); + let (copy_mode, total_len, _) = runtime.block_on(run_reconstructed_get_object_chunks_bench(&env)); + assert_eq!(copy_mode, GetObjectChunkCopyMode::Reconstructed); + assert_eq!(total_len, env.expected_len); + + group.throughput(Throughput::Bytes(env.expected_len as u64)); + group.bench_with_input(BenchmarkId::new("drain", case.name), &env, |b, env| { + b.iter(|| { + let result = runtime.block_on(run_reconstructed_get_object_chunks_bench(env)); + black_box(result); + }); + }); + } + group.finish(); +} + +criterion_group!(benches, bench_reconstructed_chunk_path); +criterion_main!(benches); diff --git a/crates/ecstore/run_benchmarks.sh b/crates/ecstore/run_benchmarks.sh index 7e5266c3e..8a2e86d9b 100755 --- a/crates/ecstore/run_benchmarks.sh +++ b/crates/ecstore/run_benchmarks.sh @@ -119,6 +119,16 @@ run_large_data_test() { print_success "Large-dataset tests completed" } +# Run direct chunk path benchmarks +run_direct_chunk_benchmark() { + print_info "📦 Starting direct chunk path benchmarks..." + echo "================================================" + + cargo bench --bench direct_chunk_benchmark + + print_success "Direct chunk path benchmarks completed" +} + # Generate comparison report generate_comparison_report() { print_info "📊 Generating performance report..." @@ -168,6 +178,7 @@ show_help() { echo " full Run the full benchmark suite" echo " performance Run detailed performance tests" echo " simd Run the SIMD-only tests" + echo " direct Run the direct chunk path benchmarks" echo " large Run large-dataset tests" echo " clean Remove previous results" echo " help Show this help message" @@ -177,6 +188,7 @@ show_help() { echo " $0 performance # Detailed performance test" echo " $0 full # Full benchmark suite" echo " $0 simd # SIMD-only benchmark" + echo " $0 direct # Direct chunk path benchmark" echo " $0 large # Large-dataset benchmark" echo "" echo "Features:" @@ -240,6 +252,11 @@ main() { run_simd_benchmark generate_comparison_report ;; + "direct") + cleanup + run_direct_chunk_benchmark + generate_comparison_report + ;; "large") cleanup run_large_data_test @@ -263,4 +280,4 @@ main() { } # Launch script -main "$@" \ No newline at end of file +main "$@" diff --git a/crates/ecstore/src/bitrot.rs b/crates/ecstore/src/bitrot.rs index 6969edf73..eb52009cf 100644 --- a/crates/ecstore/src/bitrot.rs +++ b/crates/ecstore/src/bitrot.rs @@ -14,13 +14,328 @@ use crate::disk::{self, DiskAPI as _, DiskStore, error::DiskError}; use crate::erasure_coding::{BitrotReader, BitrotWriterWrapper, CustomWriter}; -use bytes::Bytes; +use crate::store_api::{GetObjectChunkCopyMode, GetObjectChunkPath, GetObjectChunkResult}; +use bytes::{Bytes, BytesMut}; +use futures_util::{StreamExt, stream}; +use rustfs_io_core::{BoxChunkStream, IoChunk}; use rustfs_utils::HashAlgorithm; +use std::collections::VecDeque; use std::io::Cursor; use std::time::Instant; use tokio::io::AsyncRead; use tracing::debug; +const BITROT_READ_OPERATION: &str = "bitrot_read"; + +fn classify_chunk_copy_mode(source_direct: bool, copied: bool) -> GetObjectChunkCopyMode { + if copied { + GetObjectChunkCopyMode::SingleCopy + } else if source_direct { + GetObjectChunkCopyMode::TrueZeroCopy + } else { + GetObjectChunkCopyMode::SharedBytes + } +} + +struct ChunkSpan { + bytes: Bytes, + chunk: IoChunk, + copied: bool, +} + +fn take_contiguous_chunk_span(chunk: &IoChunk, offset: usize, len: usize) -> std::io::Result { + match chunk { + IoChunk::Shared(bytes) => { + let bytes = bytes.slice(offset..offset + len); + Ok(ChunkSpan { + bytes: bytes.clone(), + chunk: IoChunk::Shared(bytes), + copied: false, + }) + } + IoChunk::Mapped(mapped) => { + let chunk = IoChunk::Mapped(mapped.slice(offset, len)?); + let bytes = chunk.as_bytes(); + Ok(ChunkSpan { + bytes, + chunk, + copied: false, + }) + } + IoChunk::Pooled(pooled) => { + let chunk = IoChunk::Pooled(pooled.slice(offset, len)?); + let bytes = chunk.as_bytes(); + Ok(ChunkSpan { + bytes, + chunk, + copied: false, + }) + } + } +} + +struct BitrotChunkSource { + source_stream: BoxChunkStream, + source_chunks: VecDeque, + source_chunk_offset: usize, + source_buffered_bytes: usize, + source_done: bool, +} + +struct BitrotChunkStreamState { + source: BitrotChunkSource, + decoded_remaining: usize, + trim_prefix: usize, + output_remaining: usize, + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify: bool, +} + +struct ChunkCursor<'a> { + chunks: &'a [IoChunk], + chunk_index: usize, + chunk_offset: usize, + consumed: usize, + total_len: usize, +} + +impl<'a> ChunkCursor<'a> { + fn new(chunks: &'a [IoChunk]) -> Self { + Self { + chunks, + chunk_index: 0, + chunk_offset: 0, + consumed: 0, + total_len: chunks.iter().map(IoChunk::len).sum(), + } + } + + fn remaining(&self) -> usize { + self.total_len.saturating_sub(self.consumed) + } + + fn skip_empty_chunks(&mut self) { + while let Some(chunk) = self.chunks.get(self.chunk_index) { + if self.chunk_offset < chunk.len() { + break; + } + self.chunk_index += 1; + self.chunk_offset = 0; + } + } + + fn advance(&mut self, len: usize) { + self.consumed += len; + self.chunk_offset += len; + self.skip_empty_chunks(); + } + + fn take_span(&mut self, len: usize) -> std::io::Result { + self.skip_empty_chunks(); + if self.remaining() < len { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source")); + } + + let Some(chunk) = self.chunks.get(self.chunk_index) else { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "missing bitrot chunk source")); + }; + let available = chunk.len().saturating_sub(self.chunk_offset); + + if len <= available { + let span = take_contiguous_chunk_span(chunk, self.chunk_offset, len)?; + self.advance(len); + return Ok(span); + } + + let mut aggregate = BytesMut::with_capacity(len); + let mut remaining = len; + while remaining > 0 { + self.skip_empty_chunks(); + let Some(chunk) = self.chunks.get(self.chunk_index) else { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source")); + }; + let available = chunk.len().saturating_sub(self.chunk_offset); + let take = available.min(remaining); + aggregate.extend_from_slice(&chunk.as_bytes()[self.chunk_offset..self.chunk_offset + take]); + self.advance(take); + remaining -= take; + } + + let bytes = aggregate.freeze(); + Ok(ChunkSpan { + bytes: bytes.clone(), + chunk: IoChunk::Shared(bytes), + copied: true, + }) + } +} + +impl BitrotChunkSource { + fn new(source_stream: BoxChunkStream, source_chunks: VecDeque, source_done: bool) -> Self { + let source_buffered_bytes = source_chunks.iter().map(IoChunk::len).sum(); + Self { + source_stream, + source_chunks, + source_chunk_offset: 0, + source_buffered_bytes, + source_done, + } + } + + fn skip_empty_chunks(&mut self) { + while let Some(chunk) = self.source_chunks.front() { + if self.source_chunk_offset < chunk.len() { + break; + } + self.source_chunks.pop_front(); + self.source_chunk_offset = 0; + } + } + + async fn fill(&mut self, min_bytes: usize) -> std::io::Result<()> { + while self.source_buffered_bytes < min_bytes && !self.source_done { + match self.source_stream.next().await { + Some(Ok(chunk)) => { + self.source_buffered_bytes += chunk.len(); + self.source_chunks.push_back(chunk); + } + Some(Err(err)) => return Err(err), + None => self.source_done = true, + } + } + + if self.source_buffered_bytes < min_bytes { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source")); + } + + Ok(()) + } + + fn advance(&mut self, len: usize) { + self.source_buffered_bytes = self.source_buffered_bytes.saturating_sub(len); + self.source_chunk_offset += len; + self.skip_empty_chunks(); + } + + fn take_span(&mut self, len: usize) -> std::io::Result { + self.skip_empty_chunks(); + if self.source_buffered_bytes < len { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source")); + } + + let Some(chunk) = self.source_chunks.front() else { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "missing bitrot chunk source")); + }; + let available = chunk.len().saturating_sub(self.source_chunk_offset); + + if len <= available { + let span = take_contiguous_chunk_span(chunk, self.source_chunk_offset, len)?; + self.advance(len); + return Ok(span); + } + + let mut aggregate = BytesMut::with_capacity(len); + let mut remaining = len; + while remaining > 0 { + self.skip_empty_chunks(); + let Some(chunk) = self.source_chunks.front() else { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source")); + }; + let available = chunk.len().saturating_sub(self.source_chunk_offset); + let take = available.min(remaining); + aggregate.extend_from_slice(&chunk.as_bytes()[self.source_chunk_offset..self.source_chunk_offset + take]); + self.advance(take); + remaining -= take; + } + + let bytes = aggregate.freeze(); + Ok(ChunkSpan { + bytes: bytes.clone(), + chunk: IoChunk::Shared(bytes), + copied: true, + }) + } +} + +impl BitrotChunkStreamState { + #[allow(clippy::too_many_arguments)] + fn new( + source_stream: BoxChunkStream, + source_chunks: VecDeque, + source_done: bool, + decoded_remaining: usize, + trim_prefix: usize, + output_remaining: usize, + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify: bool, + ) -> Self { + Self { + source: BitrotChunkSource::new(source_stream, source_chunks, source_done), + decoded_remaining, + trim_prefix, + output_remaining, + shard_size, + checksum_algo, + skip_verify, + } + } + + fn hash_size(&self) -> usize { + self.checksum_algo.size() + } + + async fn next_verified_chunk(&mut self) -> std::io::Result> { + let hash_size = self.hash_size(); + + while self.output_remaining > 0 && self.decoded_remaining > 0 { + let data_len = self.shard_size.min(self.decoded_remaining); + + let expected_hash = if hash_size > 0 { + self.source.fill(hash_size).await?; + Some(self.source.take_span(hash_size)?) + } else { + None + }; + + self.source.fill(data_len).await?; + let data_span = self.source.take_span(data_len)?; + + if let Some(expected_hash) = expected_hash + && !self.skip_verify + && self.checksum_algo.hash_encode(data_span.bytes.as_ref()).as_ref() != expected_hash.bytes.as_ref() + { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch")); + } + + self.decoded_remaining -= data_len; + + if self.trim_prefix >= data_len { + self.trim_prefix -= data_len; + continue; + } + + let start = self.trim_prefix; + self.trim_prefix = 0; + let take = (data_len - start).min(self.output_remaining); + self.output_remaining -= take; + + let chunk = if start == 0 && take == data_len { + data_span.chunk + } else { + data_span.chunk.slice(start, take)? + }; + + if !chunk.is_empty() { + return Ok(Some(chunk)); + } + } + + Ok(None) + } +} + /// Create a BitrotReader from either inline data or disk file stream /// /// # Parameters @@ -65,20 +380,39 @@ pub async fn create_bitrot_reader( } else if let Some(disk) = disk { // Read from disk if use_zero_copy { + if !disk.is_local() { + rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy); + rustfs_io_metrics::record_io_fallback( + rustfs_io_metrics::IoStage::ReadSetup, + rustfs_io_metrics::FallbackReason::NonLocalBackend, + ); + + let rd = disk.read_file_stream(bucket, path, offset, length).await?; + let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify); + return Ok(Some(reader)); + } + // Try zero-copy read first (uses mmap on Unix) let start = Instant::now(); match disk.read_file_zero_copy(bucket, path, offset, length).await { Ok(bytes) => { let duration_ms = start.elapsed().as_secs_f64() * 1000.0; - // Record zero-copy metrics - rustfs_io_metrics::record_zero_copy_read(bytes.len(), duration_ms); + rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Fast); + // `read_file_zero_copy()` returns a shared `Bytes` view, but it may still + // internally aggregate multiple chunk windows. The exact chunk-native copy + // mode is only preserved by `create_bitrot_chunk_stream()`. + rustfs_io_metrics::record_io_copy_mode( + BITROT_READ_OPERATION, + rustfs_io_metrics::CopyMode::SharedBytes, + bytes.len(), + ); - // Log successful zero-copy read debug!( size = bytes.len(), + duration_ms, path = %path, - "zero_copy_read_success" + "bitrot_fast_read_success" ); // Wrap Bytes in Cursor for AsyncRead @@ -93,14 +427,16 @@ pub async fn create_bitrot_reader( Ok(Some(reader)) } Err(e) => { - // Record zero-copy fallback - rustfs_io_metrics::record_zero_copy_fallback(&format!("{:?}", e)); + rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy); + rustfs_io_metrics::record_io_fallback( + rustfs_io_metrics::IoStage::ReadSetup, + rustfs_io_metrics::FallbackReason::Unknown, + ); - // Log zero-copy fallback debug!( - reason = %format!("{:?}", e), + reason = %e, path = %path, - "zero_copy_fallback" + "bitrot_fast_read_fallback" ); // Fall back to regular stream read on error @@ -117,6 +453,7 @@ pub async fn create_bitrot_reader( } } } else { + rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy); // Use regular stream read match disk.read_file_stream(bucket, path, offset, length).await { Ok(rd) => { @@ -132,6 +469,198 @@ pub async fn create_bitrot_reader( } } +/// Create a chunk stream from bitrot-encoded data, preserving source chunk provenance when possible. +#[allow(clippy::too_many_arguments)] +pub async fn create_bitrot_chunk_stream( + inline_data: Option<&[u8]>, + disk: Option<&DiskStore>, + bucket: &str, + path: &str, + offset: usize, + length: usize, + total_data_size: usize, + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify: bool, + use_zero_copy: bool, +) -> disk::error::Result> { + let fetch_start = (offset / shard_size) * shard_size; + let fetch_end = (offset + length).div_ceil(shard_size) * shard_size; + let fetch_end = fetch_end.min(total_data_size); + let fetch_length = fetch_end.saturating_sub(fetch_start); + let trim_prefix = offset.saturating_sub(fetch_start); + let hash_size = checksum_algo.size(); + let encoded_length = fetch_length.div_ceil(shard_size) * hash_size + fetch_length; + let encoded_offset = fetch_start.div_ceil(shard_size) * hash_size + fetch_start; + + let mut source_done = false; + let (source_stream, mut prefetched_chunks, source_direct) = if let Some(data) = inline_data { + source_done = true; + let mut chunks = VecDeque::new(); + chunks.push_back(IoChunk::Shared( + Bytes::copy_from_slice(data).slice(encoded_offset..encoded_offset + encoded_length), + )); + let source_stream: BoxChunkStream = Box::pin(stream::empty::>()); + (source_stream, chunks, false) + } else if let Some(disk) = disk { + if use_zero_copy { + let mut source_stream = disk.read_file_chunks(bucket, path, encoded_offset, encoded_length).await?; + let mut prefetched_chunks = VecDeque::new(); + let mut direct = true; + while prefetched_chunks.len() < 2 { + let Some(chunk) = source_stream.next().await else { + source_done = true; + break; + }; + let chunk = chunk?; + direct &= matches!(chunk, IoChunk::Mapped(_)); + prefetched_chunks.push_back(chunk); + } + (source_stream, prefetched_chunks, direct) + } else { + source_done = true; + let bytes = disk.read_file_zero_copy(bucket, path, encoded_offset, encoded_length).await?; + let mut chunks = VecDeque::new(); + chunks.push_back(IoChunk::Shared(bytes)); + let source_stream: BoxChunkStream = Box::pin(stream::empty::>()); + (source_stream, chunks, false) + } + } else { + return Ok(None); + }; + + let copied = predicted_stream_copy(encoded_length, shard_size, checksum_algo.size(), &prefetched_chunks, source_done); + let state = BitrotChunkStreamState::new( + source_stream, + std::mem::take(&mut prefetched_chunks), + source_done, + fetch_length, + trim_prefix, + length, + shard_size, + checksum_algo, + skip_verify, + ); + let stream = stream::unfold(Some(state), |state| async move { + let mut state = match state { + Some(state) => state, + None => return None, + }; + + match state.next_verified_chunk().await { + Ok(Some(chunk)) => { + let next_state = if state.output_remaining == 0 { None } else { Some(state) }; + Some((Ok::(chunk), next_state)) + } + Ok(None) => None, + Err(err) => Some((Err(err), None)), + } + }); + Ok(Some(GetObjectChunkResult { + stream: Box::pin(stream), + path: GetObjectChunkPath::Direct, + copy_mode: classify_chunk_copy_mode(source_direct, copied), + })) +} + +fn predicted_stream_copy( + encoded_length: usize, + shard_size: usize, + hash_size: usize, + prefetched_chunks: &VecDeque, + source_done: bool, +) -> bool { + if prefetched_chunks.is_empty() { + return false; + } + + if source_done && prefetched_chunks.len() == 1 { + return false; + } + + let full_frame_len = hash_size + shard_size; + if full_frame_len == 0 { + return false; + } + + let first_window_len = prefetched_chunks.front().map(IoChunk::len).unwrap_or(encoded_length); + encoded_length > first_window_len && !first_window_len.is_multiple_of(full_frame_len) +} + +fn trim_chunk_vec(chunks: Vec, offset: usize, length: usize) -> std::io::Result> { + let mut skip = offset; + let mut remaining = length; + let mut result = Vec::new(); + + for chunk in chunks { + if remaining == 0 { + break; + } + + let chunk_len = chunk.len(); + if skip >= chunk_len { + skip -= chunk_len; + continue; + } + + let start = skip; + let take = (chunk_len - start).min(remaining); + result.push(chunk.slice(start, take)?); + remaining -= take; + skip = 0; + } + + Ok(result) +} + +fn decode_bitrot_chunk_source( + source_chunks: &[IoChunk], + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify: bool, +) -> std::io::Result<(Vec, bool)> { + let hash_size = checksum_algo.size(); + let mut cursor = ChunkCursor::new(source_chunks); + let mut result = Vec::new(); + let mut copied = false; + + while cursor.remaining() > 0 { + let expected_hash = if hash_size > 0 { + Some(cursor.take_span(hash_size)?) + } else { + None + }; + + let data_len = shard_size.min(cursor.remaining()); + if data_len == 0 { + break; + } + + let data_span = cursor.take_span(data_len)?; + copied |= data_span.copied; + if let Some(expected_hash) = expected_hash { + copied |= expected_hash.copied; + if !skip_verify && checksum_algo.hash_encode(data_span.bytes.as_ref()).as_ref() != expected_hash.bytes.as_ref() { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch")); + } + } + + result.push(data_span.chunk); + } + + Ok((result, copied)) +} + +#[doc(hidden)] +pub fn decode_bitrot_chunk_source_for_bench( + source_chunks: &[IoChunk], + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify: bool, +) -> std::io::Result<(Vec, bool)> { + decode_bitrot_chunk_source(source_chunks, shard_size, checksum_algo, skip_verify) +} + /// Create a new BitrotWriterWrapper based on the provided parameters /// /// # Parameters @@ -176,6 +705,7 @@ pub async fn create_bitrot_writer( #[cfg(test)] mod tests { use super::*; + use futures_util::StreamExt; #[tokio::test] async fn test_create_bitrot_reader_with_inline_data() { @@ -226,6 +756,246 @@ mod tests { assert!(result.unwrap().is_some()); } + #[tokio::test] + async fn test_create_bitrot_chunk_stream_with_inline_data() { + let shard_size = 4; + let checksum_algo = HashAlgorithm::HighwayHash256S; + let shard1 = b"abcd"; + let shard2 = b"ef"; + + let mut encoded = Vec::new(); + encoded.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref()); + encoded.extend_from_slice(shard1); + encoded.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref()); + encoded.extend_from_slice(shard2); + + let mut stream = create_bitrot_chunk_stream( + Some(&encoded), + None, + "test-bucket", + "test-path", + 0, + shard1.len() + shard2.len(), + shard1.len() + shard2.len(), + shard_size, + checksum_algo, + false, + false, + ) + .await + .unwrap() + .unwrap() + .stream; + + let mut collected = Vec::new(); + while let Some(chunk) = stream.next().await { + collected.extend_from_slice(&chunk.unwrap().as_bytes()); + } + + assert_eq!(collected, b"abcdef"); + } + + #[tokio::test] + async fn test_create_bitrot_chunk_stream_detects_hash_mismatch() { + let shard_size = 4; + let checksum_algo = HashAlgorithm::HighwayHash256S; + let shard = b"abcd"; + + let mut encoded = Vec::new(); + let mut bad_hash = checksum_algo.hash_encode(shard).as_ref().to_vec(); + bad_hash[0] ^= 0xFF; + encoded.extend_from_slice(&bad_hash); + encoded.extend_from_slice(shard); + + let result = create_bitrot_chunk_stream( + Some(&encoded), + None, + "test-bucket", + "test-path", + 0, + shard.len(), + shard.len(), + shard_size, + checksum_algo, + false, + false, + ) + .await; + + let mut stream = result.unwrap().unwrap().stream; + let err = stream.next().await.unwrap().unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!(err.to_string().contains("bitrot hash mismatch")); + } + + #[tokio::test] + async fn test_create_bitrot_chunk_stream_trims_range_after_decode() { + let shard_size = 4; + let checksum_algo = HashAlgorithm::HighwayHash256S; + let shard1 = b"abcd"; + let shard2 = b"efgh"; + + let mut encoded = Vec::new(); + encoded.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref()); + encoded.extend_from_slice(shard1); + encoded.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref()); + encoded.extend_from_slice(shard2); + + let mut stream = create_bitrot_chunk_stream( + Some(&encoded), + None, + "test-bucket", + "test-path", + 1, + 5, + shard1.len() + shard2.len(), + shard_size, + checksum_algo, + false, + false, + ) + .await + .unwrap() + .unwrap() + .stream; + + let mut collected = Vec::new(); + while let Some(chunk) = stream.next().await { + collected.extend_from_slice(&chunk.unwrap().as_bytes()); + } + + assert_eq!(collected, b"bcdef"); + } + + #[test] + fn test_decode_bitrot_chunk_source_preserves_aligned_multi_chunk_slices() { + let shard_size = 4; + let checksum_algo = HashAlgorithm::Md5; + let shard1 = b"abcd"; + let shard2 = b"efgh"; + + let mut encoded_chunk_one = Vec::new(); + encoded_chunk_one.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref()); + encoded_chunk_one.extend_from_slice(shard1); + + let mut encoded_chunk_two = Vec::new(); + encoded_chunk_two.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref()); + encoded_chunk_two.extend_from_slice(shard2); + + let source_chunks = vec![ + IoChunk::Shared(Bytes::from(encoded_chunk_one)), + IoChunk::Shared(Bytes::from(encoded_chunk_two)), + ]; + let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap(); + + assert!(!copied, "frame-aligned multi-chunk source should not require aggregate copies"); + assert_eq!(decoded.len(), 2); + assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd")); + assert_eq!(decoded[1].as_bytes(), Bytes::from_static(b"efgh")); + } + + #[test] + fn test_decode_bitrot_chunk_source_marks_cross_chunk_frame_as_copied() { + let shard_size = 4; + let checksum_algo = HashAlgorithm::Md5; + let shard1 = b"abcd"; + let shard2 = b"efgh"; + + let hash1 = checksum_algo.hash_encode(shard1).as_ref().to_vec(); + let hash2 = checksum_algo.hash_encode(shard2).as_ref().to_vec(); + let mut encoded = Vec::new(); + encoded.extend_from_slice(&hash1); + encoded.extend_from_slice(shard1); + encoded.extend_from_slice(&hash2); + encoded.extend_from_slice(shard2); + + let split = hash1.len() + 2; + let source_chunks = vec![ + IoChunk::Shared(Bytes::copy_from_slice(&encoded[..split])), + IoChunk::Shared(Bytes::copy_from_slice(&encoded[split..])), + ]; + let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap(); + + assert!(copied, "cross-chunk frame should be classified as requiring a copy"); + assert_eq!(decoded.len(), 2); + assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd")); + assert_eq!(decoded[1].as_bytes(), Bytes::from_static(b"efgh")); + } + + #[test] + fn test_decode_bitrot_chunk_source_preserves_pooled_single_chunk_slice() { + let shard_size = 4; + let checksum_algo = HashAlgorithm::Md5; + let shard = b"abcd"; + + let mut encoded = Vec::new(); + encoded.extend_from_slice(checksum_algo.hash_encode(shard).as_ref()); + encoded.extend_from_slice(shard); + + let source_chunks = vec![IoChunk::Pooled(rustfs_io_core::PooledChunk::from_vec(encoded))]; + let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap(); + + assert!(!copied, "single pooled chunk slice should preserve provenance without copy"); + assert_eq!(decoded.len(), 1); + assert!(matches!(&decoded[0], IoChunk::Pooled(_))); + assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd")); + } + + #[tokio::test] + async fn test_bitrot_chunk_source_marks_cross_chunk_take_as_copied() { + let source_stream: BoxChunkStream = Box::pin(stream::iter(vec![ + Ok(IoChunk::Shared(Bytes::from_static(b"ab"))), + Ok(IoChunk::Shared(Bytes::from_static(b"cd"))), + ])); + let mut source = BitrotChunkSource::new(source_stream, VecDeque::new(), false); + + source.fill(4).await.expect("source fill should succeed"); + let span = source.take_span(4).expect("cross-chunk take should succeed"); + + assert!(span.copied, "cross-chunk take should be classified as copied"); + assert_eq!(span.bytes, Bytes::from_static(b"abcd")); + assert_eq!(span.chunk.as_bytes(), Bytes::from_static(b"abcd")); + } + + #[tokio::test] + async fn test_bitrot_chunk_stream_state_yields_verified_prefix_before_later_truncation() { + let shard_size = 4; + let checksum_algo = HashAlgorithm::Md5; + let shard1 = b"abcd"; + let shard2 = b"efgh"; + + let mut first_frame = Vec::new(); + first_frame.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref()); + first_frame.extend_from_slice(shard1); + + let mut second_frame_prefix = Vec::new(); + second_frame_prefix.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref()); + second_frame_prefix.extend_from_slice(&shard2[..2]); + + let source_stream: BoxChunkStream = Box::pin(stream::iter(vec![ + Ok(IoChunk::Shared(Bytes::from(first_frame))), + Ok(IoChunk::Shared(Bytes::from(second_frame_prefix))), + ])); + let mut state = BitrotChunkStreamState::new( + source_stream, + VecDeque::new(), + false, + shard1.len() + shard2.len(), + 0, + shard1.len() + shard2.len(), + shard_size, + checksum_algo, + false, + ); + + let first = state.next_verified_chunk().await.unwrap().unwrap(); + assert_eq!(first.as_bytes(), Bytes::from_static(b"abcd")); + + let err = state.next_verified_chunk().await.unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof); + assert!(err.to_string().contains("truncated bitrot chunk source")); + } + #[tokio::test] async fn test_create_bitrot_reader_with_inline_offset_starts_at_requested_shard() { let shard_size = 4; diff --git a/crates/ecstore/src/bucket/migration.rs b/crates/ecstore/src/bucket/migration.rs index bf7809bc8..15d01a45b 100644 --- a/crates/ecstore/src/bucket/migration.rs +++ b/crates/ecstore/src/bucket/migration.rs @@ -17,7 +17,7 @@ use crate::bucket::metadata::BUCKET_METADATA_FILE; use crate::bucket::replication::{decode_resync_file, encode_resync_file}; use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; -use crate::store_api::{BucketOptions, ObjectOptions, PutObjReader, StorageAPI}; +use crate::store_api::{BucketOptions, ChunkNativePutData, ObjectOptions, StorageAPI}; use http::HeaderMap; use rustfs_policy::auth::UserIdentity; use rustfs_policy::policy::PolicyDoc; @@ -263,10 +263,8 @@ async fn migrate_one_if_missing( } }; - if let Err(e) = store - .put_object(RUSTFS_META_BUCKET, path, &mut PutObjReader::from_vec(data), opts) - .await - { + let mut put_data = ChunkNativePutData::from_vec(data); + if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await { warn!("write {label}: {e}"); } else { info!("Migrated {label}"); @@ -343,10 +341,8 @@ pub async fn try_migrate_iam_config(store: Arc) { continue; } }; - if let Err(e) = store - .put_object(RUSTFS_META_BUCKET, path, &mut PutObjReader::from_vec(data), &opts) - .await - { + let mut put_data = ChunkNativePutData::from_vec(data); + if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await { warn!("write IAM config {path}: {e}"); } else { info!("Migrated IAM config: {path}"); diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 12a534cbd..f2e48c2fe 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -16,7 +16,7 @@ use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, audit, notify, oidc, stor use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; use crate::error::{Error, Result}; use crate::global::is_first_cluster_node_local; -use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI}; +use crate::store_api::{ChunkNativePutData, ObjectInfo, ObjectOptions, StorageAPI}; use http::HeaderMap; use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS}; use rustfs_config::notify::{NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS}; @@ -128,10 +128,8 @@ pub async fn delete_config(api: Arc, file: &str) -> Result<()> } pub async fn save_config_with_opts(api: Arc, file: &str, data: Vec, opts: &ObjectOptions) -> Result<()> { - if let Err(err) = api - .put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data), opts) - .await - { + let mut put_data = ChunkNativePutData::from_vec(data); + if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await { error!("save_config_with_opts: err: {:?}, file: {}", err, file); return Err(err); } @@ -1078,10 +1076,10 @@ mod tests { use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type}; use crate::set_disk::SetDisks; use crate::store_api::{ - BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, - HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, - MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations, - ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions, + BucketInfo, BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, DeleteBucketOptions, DeletedObject, + GetObjectReader, HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, + ListOperations, MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, + ObjectOperations, ObjectOptions, ObjectToDelete, PartInfo, StorageAPI, WalkOptions, }; use http::HeaderMap; use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS}; @@ -1304,7 +1302,7 @@ mod tests { &self, _bucket: &str, _object: &str, - _data: &mut PutObjReader, + _data: &mut ChunkNativePutData, _opts: &ObjectOptions, ) -> Result { panic!("unused in test") @@ -1491,7 +1489,7 @@ mod tests { _object: &str, _upload_id: &str, _part_id: usize, - _data: &mut PutObjReader, + _data: &mut ChunkNativePutData, _opts: &ObjectOptions, ) -> Result { panic!("unused in test") diff --git a/crates/ecstore/src/config/storageclass.rs b/crates/ecstore/src/config/storageclass.rs index 5fe70f4a0..2f6f92147 100644 --- a/crates/ecstore/src/config/storageclass.rs +++ b/crates/ecstore/src/config/storageclass.rs @@ -150,18 +150,7 @@ impl Config { return false; } - let shard_size = shard_size as usize; - - let mut inline_block = DEFAULT_INLINE_BLOCK; - if self.initialized { - inline_block = self.inline_block; - } - - if versioned { - shard_size <= inline_block / 8 - } else { - shard_size <= inline_block - } + shard_size as usize <= self.inline_shard_limit_bytes(versioned) } pub fn inline_block(&self) -> usize { @@ -172,6 +161,15 @@ impl Config { } } + pub fn inline_shard_limit_bytes(&self, versioned: bool) -> usize { + let inline_block = self.inline_block(); + if versioned { inline_block / 8 } else { inline_block } + } + + pub fn inline_object_limit_bytes(&self, data_shards: usize, versioned: bool) -> usize { + self.inline_shard_limit_bytes(versioned).saturating_mul(data_shards.max(1)) + } + pub fn capacity_optimized(&self) -> bool { if !self.initialized { false @@ -336,3 +334,32 @@ pub fn validate_parity_inner(ss_parity: usize, rrs_parity: usize, set_drive_coun } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn inline_object_limit_matches_default_non_versioned_budget() { + let cfg = Config { + initialized: true, + inline_block: DEFAULT_INLINE_BLOCK, + ..Default::default() + }; + + assert_eq!(cfg.inline_shard_limit_bytes(false), DEFAULT_INLINE_BLOCK); + assert_eq!(cfg.inline_object_limit_bytes(8, false), DEFAULT_INLINE_BLOCK * 8); + } + + #[test] + fn inline_object_limit_scales_down_for_versioned_objects() { + let cfg = Config { + initialized: true, + inline_block: DEFAULT_INLINE_BLOCK, + ..Default::default() + }; + + assert_eq!(cfg.inline_shard_limit_bytes(true), DEFAULT_INLINE_BLOCK / 8); + assert_eq!(cfg.inline_object_limit_bytes(8, true), DEFAULT_INLINE_BLOCK); + } +} diff --git a/crates/ecstore/src/data_movement.rs b/crates/ecstore/src/data_movement.rs index f40840624..7d2ef7d5f 100644 --- a/crates/ecstore/src/data_movement.rs +++ b/crates/ecstore/src/data_movement.rs @@ -14,9 +14,11 @@ use crate::error::{Error, Result}; use crate::store::ECStore; -use crate::store_api::{CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader}; +use crate::store_api::{ + ChunkNativePutData, CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions, +}; use bytes::Bytes; -use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; +use rustfs_rio::{BlockReadable, BoxReadBlockFuture, EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; use std::io::Cursor; use std::pin::Pin; use std::sync::{ @@ -54,6 +56,11 @@ impl TryGetIndex for IndexedDataMovementRead } } +impl BlockReadable for IndexedDataMovementReader { + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + Box::pin(rustfs_utils::read_full(self, buf)) + } +} pub fn decode_part_index(index: Option<&Bytes>) -> Option { let bytes = index?; let mut decoded = Index::new(); @@ -64,7 +71,7 @@ pub fn decode_part_index(index: Option<&Bytes>) -> Option { } } -pub fn put_obj_reader_from_chunk(chunk: Vec, size: i64, actual_size: i64, index: Option) -> Result { +pub fn put_data_from_chunk(chunk: Vec, size: i64, actual_size: i64, index: Option) -> Result { use sha2::{Digest, Sha256}; let sha256hex = if !chunk.is_empty() { @@ -74,8 +81,8 @@ pub fn put_obj_reader_from_chunk(chunk: Vec, size: i64, actual_size: i64, in }; let reader = IndexedDataMovementReader::new(Cursor::new(chunk), index); - let hash_reader = HashReader::from_stream(reader, size, actual_size, None, sha256hex, false)?; - Ok(PutObjReader::new(hash_reader)) + let hash_reader = HashReader::from_reader(reader, size, actual_size, None, sha256hex, false)?; + Ok(ChunkNativePutData::new(hash_reader)) } pub fn new_multipart_abort_flag() -> Arc { @@ -172,7 +179,7 @@ pub(crate) async fn migrate_object( let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?; let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size }; let index = decode_part_index(part.index.as_ref()); - let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?; + let mut data = put_data_from_chunk(chunk, part_size, part_actual_size, index)?; let pi = match store .put_object_part( @@ -254,8 +261,8 @@ pub(crate) async fn migrate_object( .first() .and_then(|part| decode_part_index(part.index.as_ref())); let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index); - let hrd = HashReader::from_stream(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?; - let mut data = PutObjReader::new(hrd); + let hrd = HashReader::from_reader(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?; + let mut data = ChunkNativePutData::new(hrd); if let Err(err) = store .put_object( diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 958d2c07a..347e79c72 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -21,6 +21,7 @@ use crate::disk::{ use crate::global::GLOBAL_LOCAL_DISK_ID_MAP; use bytes::Bytes; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; +use rustfs_io_core::BoxChunkStream; use std::{ path::PathBuf, sync::{ @@ -738,6 +739,14 @@ impl DiskAPI for LocalDiskWrapper { .await } + async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result { + self.track_disk_health( + || async { self.disk.read_file_chunks(volume, path, offset, length).await }, + get_max_timeout_duration(), + ) + .await + } + async fn append_file(&self, volume: &str, path: &str) -> Result { self.track_disk_health(|| async { self.disk.append_file(volume, path).await }, Duration::ZERO) .await diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 868ab6cd6..d023a121c 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -30,12 +30,19 @@ use crate::disk::{ }; use crate::erasure_coding::bitrot_verify; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; -use bytes::Bytes; +use bytes::{Bytes, BytesMut}; +use futures_util::{StreamExt, stream}; use parking_lot::RwLock as ParkingLotRwLock; +use rustfs_config::{ + DEFAULT_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES, DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES, + DEFAULT_OBJECT_ZERO_COPY_MODE, ENV_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES, + ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES, ENV_OBJECT_ZERO_COPY_MODE, +}; use rustfs_filemeta::{ Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn, get_file_info, read_xl_meta_no_data, }; +use rustfs_io_core::{BoxChunkStream, BytesPool, IoChunk, MappedChunk, PooledChunk}; use rustfs_utils::HashAlgorithm; use rustfs_utils::os::get_info; use rustfs_utils::path::{ @@ -46,7 +53,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::fmt::Debug; use std::io::SeekFrom; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::Duration; use std::{ @@ -61,6 +68,11 @@ use tokio::time::interval; use tracing::{debug, error, info, warn}; use uuid::Uuid; +#[cfg(test)] +use serial_test::serial; +#[cfg(test)] +use temp_env::with_var; + #[derive(Debug, Clone)] pub struct FormatInfo { pub id: Option, @@ -97,6 +109,410 @@ pub struct LocalDisk { exit_signal: Option>, } +const LOCAL_CHUNK_FAST_PATH_MIN_BYTES: usize = 64 * 1024; +const LOCAL_DISK_POOLED_SOURCE_FALLBACK: &str = "fallback"; +const LOCAL_DISK_POOLED_SOURCE_COMPAT_COLLECT: &str = "compat_collect"; +const LOCAL_DISK_POOLED_SOURCE_COMPAT_DIRECT: &str = "compat_direct"; +const ACTIVE_MMAP_WINDOW_BUDGET_EXCEEDED_MESSAGE: &str = "active mmap window budget exceeded"; + +#[cfg(unix)] +const LOCAL_CHUNK_COMPAT_MAX_MAPPED_WINDOWS: usize = 1; + +static LOCAL_CHUNK_FALLBACK_POOL: OnceLock = OnceLock::new(); + +fn local_chunk_fallback_pool() -> &'static BytesPool { + LOCAL_CHUNK_FALLBACK_POOL.get_or_init(BytesPool::new_tiered) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LocalChunkZeroCopyMode { + Off, + Conservative, + Balanced, + Aggressive, +} + +impl LocalChunkZeroCopyMode { + fn from_env() -> Self { + match rustfs_utils::get_env_str(ENV_OBJECT_ZERO_COPY_MODE, DEFAULT_OBJECT_ZERO_COPY_MODE) + .trim() + .to_ascii_lowercase() + .as_str() + { + "off" => Self::Off, + "conservative" => Self::Conservative, + "aggressive" => Self::Aggressive, + _ => Self::Balanced, + } + } + + fn effective() -> Self { + if !rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE) { + return Self::Off; + } + + Self::from_env() + } + + const fn fast_path_min_bytes(self) -> usize { + match self { + Self::Aggressive => 1, + Self::Off | Self::Conservative | Self::Balanced => LOCAL_CHUNK_FAST_PATH_MIN_BYTES, + } + } + + const fn allows_multi_window(self) -> bool { + matches!(self, Self::Balanced | Self::Aggressive) + } + + const fn is_disabled(self) -> bool { + matches!(self, Self::Off) + } +} + +#[cfg(unix)] +static ACTIVE_LOCAL_MMAP_BYTES: AtomicUsize = AtomicUsize::new(0); + +#[cfg(unix)] +#[derive(Debug)] +struct ActiveMmapWindow { + mmap: memmap2::Mmap, + accounted_len: usize, +} + +#[cfg(unix)] +impl AsRef<[u8]> for ActiveMmapWindow { + fn as_ref(&self) -> &[u8] { + &self.mmap[..] + } +} + +#[cfg(unix)] +impl Drop for ActiveMmapWindow { + fn drop(&mut self) { + let remaining = ACTIVE_LOCAL_MMAP_BYTES + .fetch_sub(self.accounted_len, Ordering::AcqRel) + .saturating_sub(self.accounted_len); + rustfs_io_metrics::record_local_disk_active_mmap_bytes(remaining); + } +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn mmap_page_size() -> usize { + static PAGE_SIZE: OnceLock = OnceLock::new(); + + *PAGE_SIZE.get_or_init(|| { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page_size <= 0 { 4096 } else { page_size as usize } + }) +} + +#[cfg(unix)] +fn configured_local_chunk_window_bytes() -> usize { + let page_size = mmap_page_size(); + rustfs_utils::get_env_usize(ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES, DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES) + .max(page_size) + .div_ceil(page_size) + * page_size +} + +#[cfg(unix)] +fn configured_local_chunk_max_active_mmap_bytes() -> usize { + rustfs_utils::get_env_usize(ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES, DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES) + .max(configured_local_chunk_window_bytes()) +} + +#[cfg(unix)] +fn should_prefer_pooled_zero_copy_compat(mode: LocalChunkZeroCopyMode, length: usize, window_bytes: usize) -> bool { + if mode.is_disabled() || !mode.allows_multi_window() || length < mode.fast_path_min_bytes() || window_bytes == 0 { + return false; + } + + length.div_ceil(window_bytes) > LOCAL_CHUNK_COMPAT_MAX_MAPPED_WINDOWS +} + +fn fallback_reason_for_local_mmap_error(err: &DiskError) -> rustfs_io_metrics::FallbackReason { + match err { + DiskError::Io(io_error) if io_error.to_string().contains(ACTIVE_MMAP_WINDOW_BUDGET_EXCEEDED_MESSAGE) => { + rustfs_io_metrics::FallbackReason::WindowLimitExceeded + } + _ => rustfs_io_metrics::FallbackReason::MmapUnavailable, + } +} + +#[cfg(unix)] +fn try_reserve_active_mmap_bytes(accounted_len: usize, max_active_bytes: usize) -> bool { + loop { + let current = ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire); + let Some(next) = current.checked_add(accounted_len) else { + return false; + }; + if next > max_active_bytes { + return false; + } + + if ACTIVE_LOCAL_MMAP_BYTES + .compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + rustfs_io_metrics::record_local_disk_active_mmap_bytes(next); + return true; + } + } +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn map_file_region_bytes(file_path: &Path, offset: usize, length: usize, max_active_bytes: usize) -> Result { + use memmap2::MmapOptions; + + let aligned_offset = offset / mmap_page_size() * mmap_page_size(); + let logical_offset = offset - aligned_offset; + let map_length = logical_offset.checked_add(length).ok_or(DiskError::FileCorrupt)?; + let visible_end = logical_offset.checked_add(length).ok_or(DiskError::FileCorrupt)?; + if !try_reserve_active_mmap_bytes(map_length, max_active_bytes) { + return Err(DiskError::other(ACTIVE_MMAP_WINDOW_BUDGET_EXCEEDED_MESSAGE)); + } + let file = std::fs::File::open(file_path).map_err(DiskError::from)?; + + let mmap_result = + unsafe { MmapOptions::new().offset(aligned_offset as u64).len(map_length).map(&file) }.map_err(DiskError::other); + let mmap = match mmap_result { + Ok(mmap) => mmap, + Err(err) => { + let remaining = ACTIVE_LOCAL_MMAP_BYTES + .fetch_sub(map_length, Ordering::AcqRel) + .saturating_sub(map_length); + rustfs_io_metrics::record_local_disk_active_mmap_bytes(remaining); + return Err(err); + } + }; + let bytes = Bytes::from_owner(ActiveMmapWindow { + mmap, + accounted_len: map_length, + }); + + Ok(bytes.slice(logical_offset..visible_end)) +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn map_file_region_chunk(file_path: &Path, offset: usize, length: usize, max_active_bytes: usize) -> Result { + let bytes = map_file_region_bytes(file_path, offset, length, max_active_bytes)?; + MappedChunk::new(bytes, 0, length).map_err(DiskError::other) +} + +#[cfg(unix)] +#[derive(Debug)] +struct LocalMappedChunkStreamState { + file_path: PathBuf, + next_offset: usize, + remaining: usize, + window_bytes: usize, +} + +async fn read_file_pooled_chunk_from_path(file_path: PathBuf, offset: usize, length: usize) -> std::io::Result { + read_file_pooled_chunk_from_path_with_source(file_path, offset, length, LOCAL_DISK_POOLED_SOURCE_FALLBACK).await +} + +async fn read_file_pooled_chunk_from_path_with_source( + file_path: PathBuf, + offset: usize, + length: usize, + metric_source: &'static str, +) -> std::io::Result { + let mut file = File::open(file_path).await?; + if offset > 0 { + file.seek(SeekFrom::Start(offset as u64)).await?; + } + + let mut buffer = local_chunk_fallback_pool().acquire_buffer(length).await; + buffer.resize(length, 0); + file.read_exact(&mut buffer[..length]).await?; + rustfs_io_metrics::record_local_disk_pooled_chunk(metric_source, length); + Ok(IoChunk::Pooled(PooledChunk::new(buffer, length).map_err(std::io::Error::other)?)) +} + +async fn prepare_read_file_request(disk: &LocalDisk, volume: &str, path: &str) -> Result<(PathBuf, PathBuf, Metadata)> { + let volume_dir = disk.get_bucket_path(volume)?; + if !skip_access_checks(volume) { + access(&volume_dir) + .await + .map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?; + } + + let file_path = disk.get_object_path(volume, path)?; + check_path_length(file_path.to_string_lossy().as_ref())?; + + let file_path_clone = file_path.clone(); + let meta = tokio::task::spawn_blocking(move || std::fs::metadata(&file_path_clone).map_err(DiskError::from)) + .await + .map_err(DiskError::from)??; + + Ok((volume_dir, file_path, meta)) +} + +fn validate_read_file_bounds(meta: &Metadata, offset: usize, length: usize) -> Result<()> { + let end_offset = offset.checked_add(length).ok_or(DiskError::FileCorrupt)?; + if meta.len() < end_offset as u64 { + error!( + "read_file: file size is less than offset + length {} + {} = {}", + offset, + length, + meta.len() + ); + return Err(DiskError::FileCorrupt); + } + + Ok(()) +} + +#[cfg(unix)] +fn build_lazy_mapped_chunk_stream( + file_path: PathBuf, + offset: usize, + length: usize, + window_bytes: usize, + max_active_bytes: usize, +) -> BoxChunkStream { + let state = LocalMappedChunkStreamState { + file_path, + next_offset: offset, + remaining: length, + window_bytes, + }; + + Box::pin(stream::unfold(Some(state), move |state| async move { + let mut state = match state { + Some(state) => state, + None => return None, + }; + + if state.remaining == 0 { + return None; + } + + let visible_len = state.remaining.min(state.window_bytes); + let window_offset = state.next_offset; + let file_path = state.file_path.clone(); + let mmap_result = + tokio::task::spawn_blocking(move || map_file_region_chunk(&file_path, window_offset, visible_len, max_active_bytes)) + .await; + + match mmap_result { + Ok(Ok(chunk)) => { + state.next_offset += visible_len; + state.remaining -= visible_len; + let next_state = if state.remaining == 0 { None } else { Some(state) }; + Some((Ok(IoChunk::Mapped(chunk)), next_state)) + } + Ok(Err(err)) => { + rustfs_io_metrics::record_io_fallback( + rustfs_io_metrics::IoStage::LocalDiskChunk, + fallback_reason_for_local_mmap_error(&err), + ); + debug!( + error = %err, + offset = window_offset, + len = visible_len, + "local disk lazy mmap window failed, falling back to buffered remainder" + ); + let fallback = + read_file_pooled_chunk_from_path(state.file_path.clone(), state.next_offset, state.remaining).await; + Some((fallback, None)) + } + Err(err) => { + rustfs_io_metrics::record_io_fallback( + rustfs_io_metrics::IoStage::LocalDiskChunk, + rustfs_io_metrics::FallbackReason::MmapUnavailable, + ); + debug!( + error = %err, + offset = window_offset, + len = visible_len, + "local disk lazy mmap task failed, falling back to buffered remainder" + ); + let fallback = + read_file_pooled_chunk_from_path(state.file_path.clone(), state.next_offset, state.remaining).await; + Some((fallback, None)) + } + } + })) +} + +async fn read_file_pooled_chunk_fallback( + disk: &LocalDisk, + volume_dir: &Path, + file_path: PathBuf, + offset: usize, + length: usize, +) -> Result { + read_file_pooled_chunk_fallback_with_source(disk, volume_dir, file_path, offset, length, LOCAL_DISK_POOLED_SOURCE_FALLBACK) + .await +} + +async fn read_file_pooled_chunk_fallback_with_source( + disk: &LocalDisk, + volume_dir: &Path, + file_path: PathBuf, + offset: usize, + length: usize, + metric_source: &'static str, +) -> Result { + let mut f = disk.open_file(file_path, O_RDONLY, volume_dir).await?; + + if offset > 0 { + f.seek(SeekFrom::Start(offset as u64)).await?; + } + + let mut buffer = local_chunk_fallback_pool().acquire_buffer(length).await; + buffer.resize(length, 0); + f.read_exact(&mut buffer[..length]).await?; + rustfs_io_metrics::record_local_disk_pooled_chunk(metric_source, length); + Ok(IoChunk::Pooled(PooledChunk::new(buffer, length).map_err(DiskError::other)?)) +} + +async fn collect_chunk_stream_bytes(mut stream: BoxChunkStream, expected_len: usize) -> Result { + let Some(first) = stream.next().await else { + return Ok(Bytes::new()); + }; + let first = first.map_err(DiskError::from)?; + let first_len = first.len(); + if matches!(first, IoChunk::Pooled(_)) { + rustfs_io_metrics::record_local_disk_pooled_chunk(LOCAL_DISK_POOLED_SOURCE_COMPAT_COLLECT, first_len); + } + let first_bytes = first.as_bytes(); + + let Some(second) = stream.next().await else { + return Ok(first_bytes); + }; + let second = second.map_err(DiskError::from)?; + let second_len = second.len(); + if matches!(second, IoChunk::Pooled(_)) { + rustfs_io_metrics::record_local_disk_pooled_chunk(LOCAL_DISK_POOLED_SOURCE_COMPAT_COLLECT, second_len); + } + let mut chunk_count = 2usize; + let mut total_bytes = first_len + second_len; + let mut buffer = BytesMut::with_capacity(expected_len); + buffer.extend_from_slice(first_bytes.as_ref()); + buffer.extend_from_slice(second.as_bytes().as_ref()); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(DiskError::from)?; + let chunk_len = chunk.len(); + if matches!(chunk, IoChunk::Pooled(_)) { + rustfs_io_metrics::record_local_disk_pooled_chunk(LOCAL_DISK_POOLED_SOURCE_COMPAT_COLLECT, chunk_len); + } + chunk_count += 1; + total_bytes += chunk_len; + buffer.extend_from_slice(chunk.as_bytes().as_ref()); + } + + rustfs_io_metrics::record_local_disk_compat_collect(chunk_count, total_bytes); + Ok(buffer.freeze()) +} + impl Drop for LocalDisk { fn drop(&mut self) { if let Some(exit_signal) = self.exit_signal.take() { @@ -1835,87 +2251,96 @@ impl DiskAPI for LocalDisk { use std::time::Instant; let start = Instant::now(); - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - access(&volume_dir) - .await - .map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?; - } - - let file_path = self.get_object_path(volume, path)?; - check_path_length(file_path.to_string_lossy().as_ref())?; - - // Verify file exists and get metadata - let file_path_clone = file_path.clone(); - let meta = tokio::task::spawn_blocking(move || std::fs::metadata(&file_path_clone).map_err(DiskError::from)) - .await - .map_err(DiskError::from)??; - - let end_offset = offset.checked_add(length).ok_or(DiskError::FileCorrupt)?; - if meta.len() < end_offset as u64 { - error!( - "read_file_zero_copy: file size is less than offset + length {} + {} = {}", - offset, - length, - meta.len() - ); - return Err(DiskError::FileCorrupt); - } - - // Unix: use mmap to read the data (copies into Bytes for safe ownership) - // Non-Unix: fall back to efficient read #[cfg(unix)] { - use memmap2::MmapOptions; - let file_path_clone = file_path.clone(); - let offset_u64 = offset as u64; - - let bytes = tokio::task::spawn_blocking(move || { - let file = std::fs::File::open(&file_path_clone).map_err(DiskError::from)?; - - // Create memory map for the specified region - // SAFETY: The file is opened as read-only, and we're mapping a region - // that we've already verified exists and is within file bounds. - let mmap = unsafe { MmapOptions::new().offset(offset_u64).len(length).map(&file) }.map_err(DiskError::other)?; - - // Copy the mapped region into a Bytes buffer. This avoids undefined - // behavior from treating OS-managed mmap memory as allocator-managed - // Vec storage, at the cost of an extra copy. - Ok::(Bytes::copy_from_slice(&mmap)) - }) - .await - .map_err(DiskError::from)??; - - // Log successful mmap read metrics - let duration_ms = start.elapsed().as_secs_f64() * 1000.0; - - // Record mmap read metrics - rustfs_io_metrics::record_zero_copy_read(length, duration_ms); - - debug!(size = length, duration_ms = duration_ms, "mmap_read_success"); - - return Ok(bytes); + let zero_copy_mode = LocalChunkZeroCopyMode::effective(); + let window_bytes = configured_local_chunk_window_bytes(); + if should_prefer_pooled_zero_copy_compat(zero_copy_mode, length, window_bytes) { + let (volume_dir, file_path, meta) = prepare_read_file_request(self, volume, path).await?; + validate_read_file_bounds(&meta, offset, length)?; + let chunk = read_file_pooled_chunk_fallback_with_source( + self, + &volume_dir, + file_path, + offset, + length, + LOCAL_DISK_POOLED_SOURCE_COMPAT_DIRECT, + ) + .await?; + let bytes = collect_chunk_stream_bytes(Box::pin(stream::iter(vec![Ok(chunk)])), length).await?; + debug!( + size = bytes.len(), + duration_ms = start.elapsed().as_secs_f64() * 1000.0, + "chunk_compat_read_pooled_success" + ); + return Ok(bytes); + } } - // Non-Unix fallback: efficient read into Bytes - #[cfg(not(unix))] + let bytes = collect_chunk_stream_bytes(self.read_file_chunks(volume, path, offset, length).await?, length).await?; + debug!( + size = bytes.len(), + duration_ms = start.elapsed().as_secs_f64() * 1000.0, + "chunk_compat_read_success" + ); + Ok(bytes) + } + + #[allow(unsafe_code)] + #[tracing::instrument(level = "debug", skip(self))] + async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result { + let (volume_dir, file_path, meta) = prepare_read_file_request(self, volume, path).await?; + validate_read_file_bounds(&meta, offset, length)?; + + let zero_copy_mode = LocalChunkZeroCopyMode::effective(); + if zero_copy_mode.is_disabled() { + rustfs_io_metrics::record_io_fallback( + rustfs_io_metrics::IoStage::LocalDiskChunk, + rustfs_io_metrics::FallbackReason::MmapDisabled, + ); + let chunk = read_file_pooled_chunk_fallback(self, &volume_dir, file_path, offset, length).await?; + return Ok(Box::pin(stream::iter(vec![Ok(chunk)]))); + } + + if length < zero_copy_mode.fast_path_min_bytes() { + rustfs_io_metrics::record_io_fallback( + rustfs_io_metrics::IoStage::LocalDiskChunk, + rustfs_io_metrics::FallbackReason::SmallObject, + ); + let chunk = read_file_pooled_chunk_fallback(self, &volume_dir, file_path, offset, length).await?; + return Ok(Box::pin(stream::iter(vec![Ok(chunk)]))); + } + + #[cfg(unix)] { - // Record zero-copy fallback - rustfs_io_metrics::record_zero_copy_fallback("non_unix_platform"); + let window_bytes = configured_local_chunk_window_bytes(); - debug!(reason = "non_unix_platform", "zero_copy_fallback"); - - let mut f = self.open_file(file_path, O_RDONLY, volume_dir).await?; - - if offset > 0 { - f.seek(SeekFrom::Start(offset as u64)).await?; + if !zero_copy_mode.allows_multi_window() && length > window_bytes { + rustfs_io_metrics::record_io_fallback( + rustfs_io_metrics::IoStage::LocalDiskChunk, + rustfs_io_metrics::FallbackReason::WindowLimitExceeded, + ); + let chunk = read_file_pooled_chunk_fallback(self, &volume_dir, file_path, offset, length).await?; + return Ok(Box::pin(stream::iter(vec![Ok(chunk)]))); } - let mut buffer = Vec::with_capacity(length); - buffer.resize(length, 0); - f.read_exact(&mut buffer).await?; + return Ok(build_lazy_mapped_chunk_stream( + file_path, + offset, + length, + window_bytes, + configured_local_chunk_max_active_mmap_bytes(), + )); + } - Ok(Bytes::from(buffer)) + #[cfg(not(unix))] + { + rustfs_io_metrics::record_io_fallback( + rustfs_io_metrics::IoStage::LocalDiskChunk, + rustfs_io_metrics::FallbackReason::MmapUnavailable, + ); + let chunk = read_file_pooled_chunk_fallback(self, &volume_dir, file_path, offset, length).await?; + Ok(Box::pin(stream::iter(vec![Ok(chunk)]))) } } @@ -2674,6 +3099,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf #[cfg(test)] mod test { use super::*; + use futures_util::StreamExt; #[tokio::test] async fn test_skip_access_checks() { @@ -2960,6 +3386,22 @@ mod test { assert!(matches!(result, Err(DiskError::FileCorrupt))); } + #[tokio::test] + async fn test_read_file_zero_copy_supports_non_zero_offset() { + use tempfile::tempdir; + + let dir = tempdir().unwrap(); + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + + disk.make_volume("test-volume").await.unwrap(); + let content = Bytes::from_static(b"0123456789abcdef"); + disk.write_all("test-volume", "test-file.txt", content.clone()).await.unwrap(); + + let result = disk.read_file_zero_copy("test-volume", "test-file.txt", 3, 7).await.unwrap(); + assert_eq!(result, Bytes::from_static(b"3456789")); + } + #[test] fn test_is_valid_volname() { // Valid volume names (length >= 3) @@ -3057,6 +3499,274 @@ mod test { let _ = fs::remove_file(test_file).await; } + #[tokio::test] + #[serial] + async fn test_read_file_chunks_returns_pooled_chunk_for_local_fallback() { + let dir = tempfile::tempdir().unwrap(); + let bucket = "chunk-bucket"; + let object = "obj.txt"; + let content = b"chunk-data"; + + fs::create_dir_all(dir.path().join(bucket)).await.unwrap(); + fs::write(dir.path().join(bucket).join(object), content).await.unwrap(); + + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + + let mut stream = disk.read_file_chunks(bucket, object, 0, content.len()).await.unwrap(); + let first = stream.next().await.unwrap().unwrap(); + assert!(matches!(first, IoChunk::Pooled(_))); + assert_eq!(first.as_bytes(), Bytes::from_static(content)); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + #[serial] + async fn test_read_file_chunks_prefers_mapped_chunk_when_eligible() { + let dir = tempfile::tempdir().unwrap(); + let bucket = "chunk-bucket"; + let object = "obj-large.txt"; + let content = vec![7u8; LOCAL_CHUNK_FAST_PATH_MIN_BYTES]; + + fs::create_dir_all(dir.path().join(bucket)).await.unwrap(); + fs::write(dir.path().join(bucket).join(object), &content).await.unwrap(); + + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + + let mut stream = disk.read_file_chunks(bucket, object, 0, content.len()).await.unwrap(); + let first = stream.next().await.unwrap().unwrap(); + #[cfg(unix)] + assert!(matches!(first, IoChunk::Mapped(_))); + #[cfg(not(unix))] + assert!(matches!(first, IoChunk::Pooled(_))); + assert_eq!(first.as_bytes(), Bytes::from(content)); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + #[serial] + async fn test_read_file_chunks_falls_back_to_pooled_for_small_object() { + let dir = tempfile::tempdir().unwrap(); + let bucket = "chunk-bucket"; + let object = "obj-small.txt"; + let content = b"small-object"; + + fs::create_dir_all(dir.path().join(bucket)).await.unwrap(); + fs::write(dir.path().join(bucket).join(object), content).await.unwrap(); + + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + + let mut stream = disk.read_file_chunks(bucket, object, 0, content.len()).await.unwrap(); + let first = stream.next().await.unwrap().unwrap(); + assert!(matches!(first, IoChunk::Pooled(_))); + assert_eq!(first.as_bytes(), Bytes::from_static(content)); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + #[serial] + async fn test_read_file_chunks_supports_non_zero_offset() { + let dir = tempfile::tempdir().unwrap(); + let bucket = "chunk-bucket"; + let object = "obj-offset.txt"; + let content = vec![3u8; LOCAL_CHUNK_FAST_PATH_MIN_BYTES + 16]; + + fs::create_dir_all(dir.path().join(bucket)).await.unwrap(); + fs::write(dir.path().join(bucket).join(object), &content).await.unwrap(); + + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + + let mut stream = disk + .read_file_chunks(bucket, object, 1, LOCAL_CHUNK_FAST_PATH_MIN_BYTES) + .await + .unwrap(); + let first = stream.next().await.unwrap().unwrap(); + #[cfg(unix)] + assert!(matches!(first, IoChunk::Mapped(_))); + #[cfg(not(unix))] + assert!(matches!(first, IoChunk::Pooled(_))); + assert_eq!(first.as_bytes(), Bytes::copy_from_slice(&content[1..1 + LOCAL_CHUNK_FAST_PATH_MIN_BYTES])); + assert!(stream.next().await.is_none()); + } + + #[cfg(unix)] + #[tokio::test] + #[serial] + async fn test_read_file_chunks_splits_large_reads_into_multiple_windows() { + let dir = tempfile::tempdir().unwrap(); + let bucket = "chunk-bucket"; + let object = "obj-windowed.txt"; + let content = vec![5u8; DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES + 32]; + + fs::create_dir_all(dir.path().join(bucket)).await.unwrap(); + fs::write(dir.path().join(bucket).join(object), &content).await.unwrap(); + + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + + let mut stream = disk.read_file_chunks(bucket, object, 0, content.len()).await.unwrap(); + let first = stream.next().await.unwrap().unwrap(); + let second = stream.next().await.unwrap().unwrap(); + + assert!(matches!(first, IoChunk::Mapped(_))); + assert!(matches!(second, IoChunk::Mapped(_))); + assert_eq!(first.len(), DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES); + assert_eq!(second.len(), 32); + assert_eq!(first.as_bytes(), Bytes::from(vec![5u8; DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES])); + assert_eq!(second.as_bytes(), Bytes::from(vec![5u8; 32])); + assert!(stream.next().await.is_none()); + } + + #[cfg(unix)] + #[tokio::test] + #[serial] + async fn test_read_file_zero_copy_collects_multi_window_chunk_stream() { + let dir = tempfile::tempdir().unwrap(); + let bucket = "chunk-bucket"; + let object = "obj-zero-copy-compat.txt"; + let content = vec![6u8; DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES + 48]; + + fs::create_dir_all(dir.path().join(bucket)).await.unwrap(); + fs::write(dir.path().join(bucket).join(object), &content).await.unwrap(); + + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + + let bytes = disk.read_file_zero_copy(bucket, object, 0, content.len()).await.unwrap(); + assert_eq!(bytes, Bytes::from(content)); + } + + #[cfg(unix)] + #[test] + #[serial] + fn test_read_file_chunks_lazy_windows_reuse_single_window_budget() { + let page_size = mmap_page_size(); + let window_bytes = page_size.to_string(); + let max_active_bytes = page_size.to_string(); + + with_var(ENV_OBJECT_ZERO_COPY_ENABLE, Some("true"), || { + with_var(ENV_OBJECT_ZERO_COPY_MODE, Some("aggressive"), || { + with_var(ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES, Some(window_bytes.clone()), || { + with_var(ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES, Some(max_active_bytes.clone()), || { + ACTIVE_LOCAL_MMAP_BYTES.store(0, Ordering::Release); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let dir = tempfile::tempdir().unwrap(); + let bucket = "chunk-bucket"; + let object = "obj-budgeted.txt"; + let content = vec![9u8; page_size * 2]; + + fs::create_dir_all(dir.path().join(bucket)).await.unwrap(); + fs::write(dir.path().join(bucket).join(object), &content).await.unwrap(); + + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + + let mut stream = disk.read_file_chunks(bucket, object, 0, content.len()).await.unwrap(); + let first = stream.next().await.unwrap().unwrap(); + assert!(matches!(first, IoChunk::Mapped(_))); + assert_eq!(first.len(), page_size); + assert_eq!(ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire), page_size); + + drop(first); + assert_eq!(ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire), 0); + + let second = stream.next().await.unwrap().unwrap(); + assert!(matches!(second, IoChunk::Mapped(_))); + assert_eq!(second.len(), page_size); + assert_eq!(ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire), page_size); + + drop(second); + assert_eq!(ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire), 0); + assert!(stream.next().await.is_none()); + }); + + assert_eq!(ACTIVE_LOCAL_MMAP_BYTES.load(Ordering::Acquire), 0); + }); + }); + }); + }); + } + + #[test] + #[serial] + fn test_local_chunk_zero_copy_mode_respects_enable_and_mode_env() { + with_var(ENV_OBJECT_ZERO_COPY_ENABLE, Some("false"), || { + assert_eq!(LocalChunkZeroCopyMode::effective(), LocalChunkZeroCopyMode::Off); + }); + + with_var(ENV_OBJECT_ZERO_COPY_ENABLE, Some("true"), || { + with_var(ENV_OBJECT_ZERO_COPY_MODE, Some("aggressive"), || { + assert_eq!(LocalChunkZeroCopyMode::effective(), LocalChunkZeroCopyMode::Aggressive); + }); + }); + } + + #[cfg(unix)] + #[test] + #[serial] + fn test_should_prefer_pooled_zero_copy_compat_for_multi_window_requests() { + let window_bytes = 1024; + let balanced_multi_window_len = LOCAL_CHUNK_FAST_PATH_MIN_BYTES.max(window_bytes * 2); + + assert!(!should_prefer_pooled_zero_copy_compat( + LocalChunkZeroCopyMode::Off, + window_bytes * 2, + window_bytes + )); + assert!(!should_prefer_pooled_zero_copy_compat( + LocalChunkZeroCopyMode::Conservative, + window_bytes * 2, + window_bytes + )); + assert!(!should_prefer_pooled_zero_copy_compat( + LocalChunkZeroCopyMode::Balanced, + window_bytes, + window_bytes + )); + assert!(should_prefer_pooled_zero_copy_compat( + LocalChunkZeroCopyMode::Balanced, + balanced_multi_window_len, + window_bytes + )); + assert!(should_prefer_pooled_zero_copy_compat( + LocalChunkZeroCopyMode::Aggressive, + window_bytes * 2, + window_bytes + )); + } + + #[test] + fn test_fallback_reason_for_local_mmap_error_distinguishes_budget_limit() { + let budget_err = DiskError::other(ACTIVE_MMAP_WINDOW_BUDGET_EXCEEDED_MESSAGE); + let generic_err = DiskError::other("mmap failed"); + + assert_eq!( + fallback_reason_for_local_mmap_error(&budget_err), + rustfs_io_metrics::FallbackReason::WindowLimitExceeded + ); + assert_eq!( + fallback_reason_for_local_mmap_error(&generic_err), + rustfs_io_metrics::FallbackReason::MmapUnavailable + ); + } + + #[cfg(unix)] + #[test] + #[serial] + fn test_configured_local_chunk_window_bytes_aligns_to_page_size() { + with_var(ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES, Some("12345"), || { + let page_size = mmap_page_size(); + let window_bytes = configured_local_chunk_window_bytes(); + assert!(window_bytes >= 12345); + assert_eq!(window_bytes % page_size, 0); + }); + } + #[test] fn test_is_root_path() { // Unix root path diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index b00a036ce..3bd2857c6 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -41,6 +41,7 @@ use error::DiskError; use error::{Error, Result}; use local::LocalDisk; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; +use rustfs_io_core::BoxChunkStream; use rustfs_madmin::info_commands::DiskMetrics; use serde::{Deserialize, Serialize}; use std::{fmt::Debug, path::PathBuf, sync::Arc}; @@ -295,6 +296,14 @@ impl DiskAPI for Disk { } } + #[tracing::instrument(skip(self))] + async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result { + match self { + Disk::Local(local_disk) => local_disk.read_file_chunks(volume, path, offset, length).await, + Disk::Remote(remote_disk) => remote_disk.read_file_chunks(volume, path, offset, length).await, + } + } + #[tracing::instrument(skip(self))] async fn append_file(&self, volume: &str, path: &str) -> Result { match self { @@ -505,6 +514,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { /// On other platforms, falls back to efficient read operations. async fn read_file_zero_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result; + /// Chunk-based file read compatibility layer for the zero-copy data plane. + async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result; + async fn append_file(&self, volume: &str, path: &str) -> Result; async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result; // ReadFileStream diff --git a/crates/ecstore/src/erasure_coding/bitrot.rs b/crates/ecstore/src/erasure_coding/bitrot.rs index 01e2f4c3c..7a02abf81 100644 --- a/crates/ecstore/src/erasure_coding/bitrot.rs +++ b/crates/ecstore/src/erasure_coding/bitrot.rs @@ -172,6 +172,52 @@ where } } +impl BitrotWriter { + fn write_inline_sync(&mut self, buf: &[u8]) -> std::io::Result { + if buf.is_empty() { + return Ok(0); + } + + if self.finished { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "bitrot writer already finished")); + } + + if buf.len() > self.shard_size { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("data size {} exceeds shard size {}", buf.len(), self.shard_size), + )); + } + + if buf.len() < self.shard_size { + self.finished = true; + } + + match &mut self.inner { + CustomWriter::InlineBuffer(data) => { + if self.hash_algo.size() > 0 { + let hash = self.hash_algo.hash_encode(buf); + if hash.as_ref().is_empty() { + error!("bitrot writer write hash error: hash is empty"); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "hash is empty")); + } + data.extend_from_slice(hash.as_ref()); + } + data.extend_from_slice(buf); + Ok(buf.len()) + } + CustomWriter::Other(_) => Err(std::io::Error::other("inline sync write requires inline buffer writer")), + } + } + + fn shutdown_inline_sync(&mut self) -> std::io::Result<()> { + match self.inner { + CustomWriter::InlineBuffer(_) => Ok(()), + CustomWriter::Other(_) => Err(std::io::Error::other("inline sync shutdown requires inline buffer writer")), + } + } +} + async fn write_all_vectored(writer: &mut W, hash: &[u8], data: &[u8]) -> std::io::Result<()> where W: AsyncWrite + Unpin, @@ -280,6 +326,10 @@ impl CustomWriter { Self::Other(_) => None, } } + + pub fn is_inline_buffer(&self) -> bool { + matches!(self, Self::InlineBuffer(_)) + } } impl AsyncWrite for CustomWriter { @@ -397,6 +447,24 @@ impl BitrotWriterWrapper { self.bitrot_writer.shutdown().await } + pub fn is_inline_buffer(&self) -> bool { + matches!(self.writer_type, WriterType::InlineBuffer) + } + + pub fn write_inline_sync(&mut self, buf: &[u8]) -> std::io::Result { + if !self.is_inline_buffer() { + return Err(std::io::Error::other("inline sync write requires inline buffer writer")); + } + self.bitrot_writer.write_inline_sync(buf) + } + + pub fn shutdown_inline_sync(&mut self) -> std::io::Result<()> { + if !self.is_inline_buffer() { + return Err(std::io::Error::other("inline sync shutdown requires inline buffer writer")); + } + self.bitrot_writer.shutdown_inline_sync() + } + /// Extract the inline buffer data, consuming the wrapper pub fn into_inline_data(self) -> Option> { match self.writer_type { diff --git a/crates/ecstore/src/erasure_coding/decode.rs b/crates/ecstore/src/erasure_coding/decode.rs index 0e5d03ed4..e0a5790ae 100644 --- a/crates/ecstore/src/erasure_coding/decode.rs +++ b/crates/ecstore/src/erasure_coding/decode.rs @@ -17,6 +17,7 @@ use crate::disk::error_reduce::reduce_errs; use crate::erasure_coding::{BitrotReader, Erasure}; use futures::stream::{FuturesUnordered, StreamExt}; use pin_project_lite::pin_project; +use rustfs_io_core::{IoChunk, PooledChunk}; use std::io; use std::io::ErrorKind; use tokio::io::AsyncRead; @@ -155,6 +156,68 @@ fn get_data_block_len(shards: &[Option>], data_blocks: usize) -> usize { size } +fn block_window( + offset: usize, + length: usize, + block_size: usize, + block_index: usize, + start_block: usize, + end_block: usize, +) -> (usize, usize) { + let end_remainder = offset.saturating_add(length) % block_size; + if start_block == end_block { + (offset % block_size, length) + } else if block_index == start_block { + (offset % block_size, block_size - (offset % block_size)) + } else if block_index == end_block { + (0, if end_remainder == 0 { block_size } else { end_remainder }) + } else { + (0, block_size) + } +} + +fn take_data_blocks_as_chunks( + shards: &mut [Option>], + data_blocks: usize, + mut offset: usize, + length: usize, +) -> io::Result> { + if get_data_block_len(shards, data_blocks) < length { + error!("take_data_blocks_as_chunks get_data_block_len < length"); + return Err(io::Error::new(ErrorKind::UnexpectedEof, "Not enough data blocks to write")); + } + + let mut chunks = Vec::new(); + let mut remaining = length; + for block_op in shards.iter_mut().take(data_blocks) { + let Some(block) = block_op.take() else { + error!("take_data_blocks_as_chunks block_op.is_none()"); + return Err(io::Error::new(ErrorKind::UnexpectedEof, "Missing data block")); + }; + + if offset >= block.len() { + offset -= block.len(); + continue; + } + + let start = offset; + offset = 0; + let take = (block.len() - start).min(remaining); + let chunk = if start == 0 && take == block.len() { + IoChunk::Pooled(PooledChunk::from_vec(block)) + } else { + IoChunk::Pooled(PooledChunk::from_vec(block).slice(start, take)?) + }; + chunks.push(chunk); + remaining -= take; + if remaining == 0 { + break; + } + } + + Ok(chunks) +} + /// Write data blocks from encoded blocks to target, supporting offset and length async fn write_data_blocks( writer: &mut W, @@ -213,6 +276,134 @@ where Ok(total_written) } +pub(crate) struct ErasureChunkDecoder { + erasure: Erasure, + reader: ParallelReader, + offset: usize, + length: usize, + start_block: usize, + end_block: usize, + current_block: usize, + written: usize, + healable_error: Option, + finished: bool, +} + +impl ErasureChunkDecoder +where + R: AsyncRead + Unpin + Send + Sync, +{ + pub(crate) fn new( + erasure: Erasure, + readers: Vec>>, + offset: usize, + length: usize, + total_length: usize, + ) -> io::Result { + if readers.len() != erasure.data_shards + erasure.parity_shards { + return Err(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")); + } + + let end_offset = offset + .checked_add(length) + .ok_or_else(|| io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length"))?; + if end_offset > total_length { + return Err(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")); + } + + let start_block = offset / erasure.block_size; + let end_block = if length == 0 { + start_block + } else { + end_offset.saturating_sub(1) / erasure.block_size + }; + let reader = ParallelReader::new(readers, erasure.clone(), offset, total_length); + + Ok(Self { + erasure, + reader, + offset, + length, + start_block, + end_block, + current_block: start_block, + written: 0, + healable_error: None, + finished: length == 0, + }) + } + + pub(crate) async fn next_chunks(&mut self) -> io::Result>> { + if self.finished { + return Ok(None); + } + + if self.current_block > self.end_block { + self.finished = true; + return Ok(None); + } + + let block_index = self.current_block; + self.current_block += 1; + + let (block_offset, block_length) = block_window( + self.offset, + self.length, + self.erasure.block_size, + block_index, + self.start_block, + self.end_block, + ); + if block_length == 0 { + self.finished = true; + return Ok(None); + } + + let (mut shards, errs) = self.reader.read().await; + + if self.healable_error.is_none() + && let (_, Some(err)) = reduce_errs(&errs, &[]) + && (err == Error::FileNotFound || err == Error::FileCorrupt) + { + self.healable_error = Some(err); + } + + if !self.reader.can_decode(&shards) { + self.finished = true; + error!("reconstructed chunk decoder can_decode errs: {:?}", &errs); + return Err(Error::ErasureReadQuorum.into()); + } + + if let Err(err) = self.erasure.decode_data(&mut shards) { + self.finished = true; + error!("reconstructed chunk decoder decode_data err: {:?}", err); + return Err(err); + } + + let chunks = take_data_blocks_as_chunks(&mut shards, self.erasure.data_shards, block_offset, block_length)?; + self.written += chunks.iter().map(IoChunk::len).sum::(); + Ok(Some(chunks)) + } + + pub(crate) fn written(&self) -> usize { + self.written + } + + pub(crate) fn finish_error(&self) -> Option { + if self.written < self.length { + Some(Error::LessData.into()) + } else { + None + } + } + + pub(crate) fn take_healable_error(&mut self) -> Option { + self.healable_error.take() + } +} + +pub(crate) type ReconstructedChunkDecoder = ErasureChunkDecoder; + impl Erasure { pub async fn decode( &self, @@ -230,7 +421,10 @@ impl Erasure { return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers"))); } - if offset + length > total_length { + let Some(end_offset) = offset.checked_add(length) else { + return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length"))); + }; + if end_offset > total_length { return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length"))); } @@ -245,7 +439,7 @@ impl Erasure { let mut reader = ParallelReader::new(readers, self.clone(), offset, total_length); let start = offset / self.block_size; - let end = (offset + length) / self.block_size; + let end = end_offset.saturating_sub(1) / self.block_size; for i in start..=end { let (block_offset, block_length) = if start == end { @@ -253,7 +447,8 @@ impl Erasure { } else if i == start { (offset % self.block_size, self.block_size - (offset % self.block_size)) } else if i == end { - (0, (offset + length) % self.block_size) + let end_remainder = end_offset % self.block_size; + (0, if end_remainder == 0 { self.block_size } else { end_remainder }) } else { (0, self.block_size) }; @@ -316,6 +511,7 @@ mod tests { disk::error::DiskError, erasure_coding::{BitrotReader, BitrotWriter}, }; + use bytes::Bytes; use rustfs_utils::HashAlgorithm; use std::io::Cursor; @@ -456,4 +652,47 @@ mod tests { let reader_cursor = Cursor::new(buf); BitrotReader::new(reader_cursor, shard_size, hash_algo.clone(), false) } + + async fn create_bitrot_reader_from_shard( + shard: Bytes, + shard_size: usize, + hash_algo: &HashAlgorithm, + ) -> BitrotReader>> { + let writer = Cursor::new(Vec::new()); + let mut writer = BitrotWriter::new(writer, shard_size, hash_algo.clone()); + writer.write(shard.as_ref()).await.unwrap(); + let reader_cursor = Cursor::new(writer.into_inner().into_inner()); + BitrotReader::new(reader_cursor, shard_size, hash_algo.clone(), false) + } + + #[tokio::test] + async fn test_erasure_chunk_decoder_reconstructs_missing_data_shard_as_pooled_chunks() { + let erasure = Erasure::new(2, 1, 4); + let original = b"abcd"; + let encoded = erasure.encode_data(original).unwrap(); + let shard_size = erasure.shard_size(); + let hash_algo = HashAlgorithm::None; + + let readers = vec![ + None, + Some(create_bitrot_reader_from_shard(encoded[1].clone(), shard_size, &hash_algo).await), + Some(create_bitrot_reader_from_shard(encoded[2].clone(), shard_size, &hash_algo).await), + ]; + + let mut decoder = ErasureChunkDecoder::new(erasure, readers, 0, original.len(), original.len()).unwrap(); + let first_batch = decoder.next_chunks().await.unwrap().unwrap(); + assert!( + first_batch.iter().all(|chunk| matches!(chunk, IoChunk::Pooled(_))), + "reconstructed decoder should produce pooled chunks" + ); + let collected = first_batch + .into_iter() + .flat_map(|chunk| chunk.as_bytes().to_vec()) + .collect::>(); + + assert_eq!(collected, original); + assert!(decoder.next_chunks().await.unwrap().is_none()); + assert_eq!(decoder.written(), original.len()); + assert!(decoder.finish_error().is_none()); + } } diff --git a/crates/ecstore/src/erasure_coding/encode.rs b/crates/ecstore/src/erasure_coding/encode.rs index e029f64a4..86b4a6930 100644 --- a/crates/ecstore/src/erasure_coding/encode.rs +++ b/crates/ecstore/src/erasure_coding/encode.rs @@ -17,11 +17,12 @@ use crate::disk::error_reduce::count_errs; use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs}; use crate::erasure_coding::BitrotWriterWrapper; use crate::erasure_coding::Erasure; +use crate::erasure_coding::erasure::{EncodeBlockBuffer, EncodedShardBlock, EncodedShardBufferPool}; use bytes::Bytes; use futures::StreamExt; use futures::stream::FuturesUnordered; +use rustfs_rio::BlockReadable; use std::sync::Arc; -use std::vec; use tokio::io::AsyncRead; use tokio::sync::mpsc; use tracing::error; @@ -32,6 +33,164 @@ pub(crate) struct MultiWriter<'a> { errs: Vec>, } +pub(crate) struct BlockAssembler { + reader: R, + block_buffer: EncodeBlockBuffer, + total_bytes: usize, +} + +impl BlockAssembler +where + R: AsyncRead + BlockReadable + Send + Sync + Unpin + 'static, +{ + pub(crate) fn new(reader: R, block_size: usize) -> Self { + Self { + reader, + block_buffer: EncodeBlockBuffer::new(block_size), + total_bytes: 0, + } + } + + pub(crate) async fn next_block(&mut self) -> std::io::Result>> { + match self.block_buffer.read_from_block(&mut self.reader).await { + Ok(n) if n > 0 => { + self.total_bytes += n; + Ok(Some(self.block_buffer.filled(n).to_vec())) + } + Ok(_) => Ok(None), + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { + if let Some(inner) = e.get_ref() + && rustfs_rio::is_checksum_mismatch(inner) + { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())); + } + Ok(None) + } + Err(e) => Err(e), + } + } + + pub(crate) fn total_bytes(&self) -> usize { + self.total_bytes + } + + pub(crate) fn into_inner(self) -> R { + self.reader + } +} + +#[derive(Clone)] +pub(crate) struct ErasureChunkEncoder { + erasure: Arc, + buffer_pool: EncodedShardBufferPool, +} + +impl ErasureChunkEncoder { + pub(crate) async fn new(erasure: Arc) -> Self { + let reusable_capacity = erasure.shard_size() * erasure.total_shard_count(); + Self { + erasure, + buffer_pool: EncodedShardBufferPool::with_prefill(reusable_capacity, 2).await, + } + } + + pub(crate) async fn encode_block(&self, block: &[u8]) -> std::io::Result { + let reusable_buffer = self.buffer_pool.acquire().await; + self.erasure.encode_data_block_with_buffer(block, reusable_buffer) + } + + pub(crate) async fn release(&self, block: EncodedShardBlock) { + self.buffer_pool.release(block).await; + } +} + +pub(crate) struct ErasureWritePipeline { + erasure: Arc, + write_quorum: usize, +} + +impl ErasureWritePipeline { + pub(crate) fn new(erasure: Arc, write_quorum: usize) -> Self { + Self { erasure, write_quorum } + } + + pub(crate) async fn run(&self, reader: R, writers: &mut [Option]) -> std::io::Result<(R, usize)> + where + R: AsyncRead + BlockReadable + Send + Sync + Unpin + 'static, + { + let (tx, mut rx) = mpsc::channel::(8); + let producer = ErasureChunkEncoder::new(self.erasure.clone()).await; + let writer_pool = producer.clone(); + let block_size = self.erasure.block_size; + + let task = tokio::spawn(async move { + let mut assembler = BlockAssembler::new(reader, block_size); + while let Some(block) = assembler.next_block().await? { + let res = producer.encode_block(&block).await?; + if let Err(err) = tx.send(res).await { + return Err(std::io::Error::other(format!("Failed to send encoded data : {err}"))); + } + } + + let total = assembler.total_bytes(); + Ok((assembler.into_inner(), total)) + }); + + let mut writers = MultiWriter::new(writers, self.write_quorum); + let mut write_err = None; + + while let Some(block) = rx.recv().await { + if block.is_empty() { + break; + } + let write_result = writers.write(&block).await; + writer_pool.release(block).await; + if let Err(err) = write_result { + write_err = Some(err); + break; + } + } + + if let Some(err) = write_err { + task.abort(); + let _ = task.await; + if let Err(shutdown_err) = writers.shutdown().await { + error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err); + } + return Err(err); + } + + let (reader, total) = task.await??; + writers.shutdown().await?; + Ok((reader, total)) + } +} + +pub(crate) trait ShardSource { + fn shard_count(&self) -> usize; + fn shard(&self, idx: usize) -> Bytes; +} + +impl ShardSource for EncodedShardBlock { + fn shard_count(&self) -> usize { + self.shard_count() + } + + fn shard(&self, idx: usize) -> Bytes { + self.shard(idx) + } +} + +impl ShardSource for Vec { + fn shard_count(&self) -> usize { + self.len() + } + + fn shard(&self, idx: usize) -> Bytes { + self[idx].clone() + } +} + impl<'a> MultiWriter<'a> { pub fn new(writers: &'a mut [Option], write_quorum: usize) -> Self { let length = writers.len(); @@ -42,10 +201,10 @@ impl<'a> MultiWriter<'a> { } } - async fn write_shard(writer_opt: &mut Option, err: &mut Option, shard: &Bytes) { + async fn write_shard(writer_opt: &mut Option, err: &mut Option, shard: Bytes) { match writer_opt { Some(writer) => { - match writer.write(shard).await { + match writer.write(&shard).await { Ok(n) => { if n < shard.len() { *err = Some(Error::ShortWrite); @@ -65,16 +224,40 @@ impl<'a> MultiWriter<'a> { } } - pub async fn write(&mut self, data: Vec) -> std::io::Result<()> { - assert_eq!(data.len(), self.writers.len()); + fn write_shard_inline(writer_opt: &mut Option, err: &mut Option, shard: Bytes) { + match writer_opt { + Some(writer) => match writer.write_inline_sync(&shard) { + Ok(n) => { + if n < shard.len() { + *err = Some(Error::ShortWrite); + *writer_opt = None; + } else { + *err = None; + } + } + Err(e) => { + *err = Some(Error::from(e)); + } + }, + None => { + *err = Some(Error::DiskNotFound); + } + } + } + + pub async fn write(&mut self, data: &T) -> std::io::Result<()> + where + T: ShardSource, + { + assert_eq!(data.shard_count(), self.writers.len()); { let mut futures = FuturesUnordered::new(); - for ((writer_opt, err), shard) in self.writers.iter_mut().zip(self.errs.iter_mut()).zip(data.iter()) { + for (idx, (writer_opt, err)) in self.writers.iter_mut().zip(self.errs.iter_mut()).enumerate() { if err.is_some() { continue; // Skip if we already have an error for this writer } - futures.push(Self::write_shard(writer_opt, err, shard)); + futures.push(Self::write_shard(writer_opt, err, data.shard(idx))); } while let Some(()) = futures.next().await {} } @@ -112,6 +295,45 @@ impl<'a> MultiWriter<'a> { ))) } + pub fn write_inline(&mut self, data: &T) -> std::io::Result<()> + where + T: ShardSource, + { + assert_eq!(data.shard_count(), self.writers.len()); + + for (idx, (writer_opt, err)) in self.writers.iter_mut().zip(self.errs.iter_mut()).enumerate() { + if err.is_some() { + continue; + } + Self::write_shard_inline(writer_opt, err, data.shard(idx)); + } + + let nil_count = self.errs.iter().filter(|&e| e.is_none()).count(); + if nil_count >= self.write_quorum { + return Ok(()); + } + + if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) { + return Err(std::io::Error::other(format!( + "Failed to write inline data: {} (offline-disks={}/{})", + write_err, + count_errs(&self.errs, &Error::DiskNotFound), + self.writers.len() + ))); + } + + Err(std::io::Error::other(format!( + "Failed to write inline data: (offline-disks={}/{}): {}", + count_errs(&self.errs, &Error::DiskNotFound), + self.writers.len(), + self.errs + .iter() + .map(|e| e.as_ref().map_or("".to_string(), |e| e.to_string())) + .collect::>() + .join(", ") + ))) + } + async fn shutdown_writer(writer_opt: &mut Option, err: &mut Option) { match writer_opt { Some(writer) => match writer.shutdown().await { @@ -129,6 +351,23 @@ impl<'a> MultiWriter<'a> { } } + fn shutdown_writer_inline(writer_opt: &mut Option, err: &mut Option) { + match writer_opt { + Some(writer) => match writer.shutdown_inline_sync() { + Ok(()) => { + *err = None; + } + Err(e) => { + *err = Some(Error::from(e)); + *writer_opt = None; + } + }, + None => { + *err = Some(Error::DiskNotFound); + } + } + } + pub async fn shutdown(&mut self) -> std::io::Result<()> { { let mut futures = FuturesUnordered::new(); @@ -173,66 +412,53 @@ impl<'a> MultiWriter<'a> { .join(", ") ))) } + + pub fn shutdown_inline(&mut self) -> std::io::Result<()> { + for (writer_opt, err) in self.writers.iter_mut().zip(self.errs.iter_mut()) { + if err.is_some() { + continue; + } + Self::shutdown_writer_inline(writer_opt, err); + } + + let nil_count = self.errs.iter().filter(|&e| e.is_none()).count(); + if nil_count >= self.write_quorum { + return Ok(()); + } + + if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) { + return Err(std::io::Error::other(format!( + "Failed to shutdown inline writers: {} (offline-disks={}/{})", + write_err, + count_errs(&self.errs, &Error::DiskNotFound), + self.writers.len() + ))); + } + + Err(std::io::Error::other(format!( + "Failed to shutdown inline writers: (offline-disks={}/{}): {}", + count_errs(&self.errs, &Error::DiskNotFound), + self.writers.len(), + self.errs + .iter() + .map(|e| e.as_ref().map_or("".to_string(), |e| e.to_string())) + .collect::>() + .join(", ") + ))) + } } impl Erasure { pub async fn encode( self: Arc, - mut reader: R, + reader: R, writers: &mut [Option], quorum: usize, ) -> std::io::Result<(R, usize)> where - R: AsyncRead + Send + Sync + Unpin + 'static, + R: AsyncRead + BlockReadable + Send + Sync + Unpin + 'static, { - let (tx, mut rx) = mpsc::channel::>(8); - - let task = tokio::spawn(async move { - let block_size = self.block_size; - let mut total = 0; - let mut buf = vec![0u8; block_size]; - loop { - match rustfs_utils::read_full(&mut reader, &mut buf).await { - Ok(n) if n > 0 => { - total += n; - let res = self.encode_data(&buf[..n])?; - if let Err(err) = tx.send(res).await { - return Err(std::io::Error::other(format!("Failed to send encoded data : {err}"))); - } - } - Ok(_) => { - break; - } - Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { - // Check if the inner error is a checksum mismatch - if so, propagate it - if let Some(inner) = e.get_ref() - && rustfs_rio::is_checksum_mismatch(inner) - { - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())); - } - break; - } - Err(e) => { - return Err(e); - } - } - } - - Ok((reader, total)) - }); - - let mut writers = MultiWriter::new(writers, quorum); - - while let Some(block) = rx.recv().await { - if block.is_empty() { - break; - } - writers.write(block).await?; - } - - let (reader, total) = task.await??; - writers.shutdown().await?; - Ok((reader, total)) + ErasureWritePipeline::new(self, quorum).run(reader, writers).await } } @@ -241,6 +467,7 @@ mod tests { use super::*; use crate::erasure_coding::{BitrotWriterWrapper, CustomWriter}; use rustfs_utils::HashAlgorithm; + use std::io::Cursor; use std::pin::Pin; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; @@ -296,4 +523,28 @@ mod tests { assert_eq!(written, b"small payload".len()); assert!(!committed.lock().unwrap().is_empty()); } + + #[tokio::test] + async fn block_assembler_splits_input_into_erasure_blocks() { + let reader = tokio::io::BufReader::new(Cursor::new(b"abcdefghijkl".to_vec())); + let mut assembler = BlockAssembler::new(reader, 4); + + assert_eq!(assembler.next_block().await.unwrap(), Some(b"abcd".to_vec())); + assert_eq!(assembler.next_block().await.unwrap(), Some(b"efgh".to_vec())); + assert_eq!(assembler.next_block().await.unwrap(), Some(b"ijkl".to_vec())); + assert_eq!(assembler.next_block().await.unwrap(), None); + assert_eq!(assembler.total_bytes(), 12); + } + + #[tokio::test] + async fn erasure_chunk_encoder_produces_full_shard_block() { + let erasure = Arc::new(Erasure::new(2, 1, 4)); + let encoder = ErasureChunkEncoder::new(erasure.clone()).await; + let block = encoder.encode_block(b"abcd").await.unwrap(); + + assert_eq!(block.shard_count(), 3); + assert_eq!(block.shard(0).len(), erasure.shard_size()); + + encoder.release(block).await; + } } diff --git a/crates/ecstore/src/erasure_coding/erasure.rs b/crates/ecstore/src/erasure_coding/erasure.rs index 8942ab4e7..63a1350d2 100644 --- a/crates/ecstore/src/erasure_coding/erasure.rs +++ b/crates/ecstore/src/erasure_coding/erasure.rs @@ -19,12 +19,131 @@ use bytes::{Bytes, BytesMut}; use reed_solomon_erasure::galois_8::ReedSolomon; use reed_solomon_simd; +use rustfs_rio::BlockReadable; use smallvec::SmallVec; use std::io; +use std::sync::Arc; use tokio::io::AsyncRead; +use tokio::sync::Mutex; use tracing::warn; use uuid::Uuid; +pub(crate) struct EncodeBlockBuffer { + buf: Vec, +} + +impl EncodeBlockBuffer { + pub(crate) fn new(block_size: usize) -> Self { + Self { + buf: vec![0u8; block_size], + } + } + + pub(crate) async fn read_from(&mut self, reader: &mut R) -> io::Result + where + R: AsyncRead + Send + Sync + Unpin, + { + rustfs_utils::read_full(&mut *reader, &mut self.buf).await + } + + pub(crate) async fn read_from_block(&mut self, reader: &mut R) -> io::Result + where + R: BlockReadable + Send + Sync + Unpin, + { + reader.read_block(&mut self.buf).await + } + + pub(crate) fn filled(&self, len: usize) -> &[u8] { + &self.buf[..len] + } +} + +pub struct EncodedShardBlock { + data: Bytes, + shard_size: usize, + shard_count: usize, +} + +impl EncodedShardBlock { + pub(crate) fn new(data: Bytes, shard_size: usize, shard_count: usize) -> Self { + Self { + data, + shard_size, + shard_count, + } + } + + pub fn shard_count(&self) -> usize { + self.shard_count + } + + pub fn len(&self) -> usize { + self.shard_count + } + + pub fn is_empty(&self) -> bool { + self.shard_count == 0 + } + + pub fn shard(&self, idx: usize) -> Bytes { + let start = idx * self.shard_size; + let end = start + self.shard_size; + self.data.slice(start..end) + } + + pub fn iter(&self) -> impl Iterator + '_ { + (0..self.shard_count).map(|idx| self.shard(idx)) + } + + pub fn into_vec(self) -> Vec { + (0..self.shard_count).map(|idx| self.shard(idx)).collect() + } + + pub fn into_reusable_buffer(self) -> BytesMut { + match self.data.try_into_mut() { + Ok(mut buf) => { + buf.clear(); + buf + } + Err(data) => BytesMut::with_capacity(data.len()), + } + } +} + +#[derive(Clone)] +pub(crate) struct EncodedShardBufferPool { + capacity: usize, + free: Arc>>, +} + +impl EncodedShardBufferPool { + pub(crate) async fn with_prefill(capacity: usize, initial: usize) -> Self { + let mut free = Vec::with_capacity(initial); + for _ in 0..initial { + free.push(BytesMut::with_capacity(capacity)); + } + + Self { + capacity, + free: Arc::new(Mutex::new(free)), + } + } + + pub(crate) async fn acquire(&self) -> BytesMut { + let mut free = self.free.lock().await; + free.pop().unwrap_or_else(|| BytesMut::with_capacity(self.capacity)) + } + + pub(crate) async fn release(&self, block: EncodedShardBlock) { + let mut free = self.free.lock().await; + let mut buf = block.into_reusable_buffer(); + if buf.capacity() < self.capacity { + buf.reserve(self.capacity - buf.capacity()); + } + free.push(buf); + } +} + /// Legacy calc_shard_size formula: (block_size.div_ceil(data_shards) + 1) & !1 /// Matches main branch and filemeta::ErasureInfo for old-version files. pub fn calc_shard_size_legacy(block_size: usize, data_shards: usize) -> usize { @@ -351,6 +470,25 @@ impl Erasure { /// A vector of encoded shards as `Bytes`. #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))] pub fn encode_data(&self, data: &[u8]) -> io::Result> { + Ok(self.encode_data_block(data)?.into_vec()) + } + + /// Encode one logical block into an `EncodedShardBlock` using a caller-provided backing buffer. + /// + /// This is the explicit reuse-oriented variant for non-hot paths that want to + /// thread a reusable `BytesMut` across multiple encode calls. + #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))] + pub fn encode_data_with_buffer(&self, data: &[u8], data_buffer: BytesMut) -> io::Result { + self.encode_data_block_with_buffer(data, data_buffer) + } + + #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))] + pub(crate) fn encode_data_block(&self, data: &[u8]) -> io::Result { + self.encode_data_block_with_buffer(data, BytesMut::with_capacity(self.shard_size() * self.total_shard_count())) + } + + #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))] + pub(crate) fn encode_data_block_with_buffer(&self, data: &[u8], mut data_buffer: BytesMut) -> io::Result { let shard_size_fn = if self.uses_legacy { calc_shard_size_legacy } else { @@ -359,7 +497,10 @@ impl Erasure { let per_shard_size = shard_size_fn(data.len(), self.data_shards); let need_total_size = per_shard_size * self.total_shard_count(); - let mut data_buffer = BytesMut::with_capacity(need_total_size); + data_buffer.clear(); + if data_buffer.capacity() < need_total_size { + data_buffer.reserve(need_total_size - data_buffer.capacity()); + } data_buffer.extend_from_slice(data); data_buffer.resize(need_total_size, 0u8); @@ -382,14 +523,7 @@ impl Erasure { } // Zero-copy split, all shards reference data_buffer - let mut data_buffer = data_buffer.freeze(); - let mut shards = Vec::with_capacity(self.total_shard_count()); - for _ in 0..self.total_shard_count() { - let shard = data_buffer.split_to(per_shard_size); - shards.push(shard); - } - - Ok(shards) + Ok(EncodedShardBlock::new(data_buffer.freeze(), per_shard_size, self.total_shard_count())) } /// Decode and reconstruct missing shards in-place. @@ -478,8 +612,8 @@ impl Erasure { /// /// # Arguments /// * `reader` - An async reader implementing AsyncRead + Send + Sync + Unpin - /// * `mut on_block` - Async callback that receives encoded blocks and returns a Result - /// * `F` - Callback type: FnMut(Result, std::io::Error>) -> Future> + Send + /// * `mut on_block` - Async callback that receives encoded blocks and returns the block for reuse + /// * `F` - Callback type: FnMut(Result) -> Future, E>> + Send /// * `Fut` - Future type returned by the callback /// * `E` - Error type returned by the callback /// * `R` - Reader type implementing AsyncRead + Send + Sync + Unpin @@ -489,26 +623,31 @@ impl Erasure { /// /// # Errors /// Returns error if reading from reader fails or if callback returns error - pub async fn encode_stream_callback_async( + pub(crate) async fn encode_stream_callback_async( self: std::sync::Arc, reader: &mut R, mut on_block: F, ) -> Result where R: AsyncRead + Send + Sync + Unpin, - F: FnMut(std::io::Result>) -> Fut + Send, - Fut: std::future::Future> + Send, + F: FnMut(std::io::Result) -> Fut + Send, + Fut: std::future::Future, E>> + Send, { let block_size = self.block_size; let mut total = 0; + let mut block_buffer = EncodeBlockBuffer::new(block_size); + let reusable_capacity = self.shard_size() * self.total_shard_count(); + let buffer_pool = EncodedShardBufferPool::with_prefill(reusable_capacity, 1).await; loop { - let mut buf = vec![0u8; block_size]; - match rustfs_utils::read_full(&mut *reader, &mut buf).await { + match block_buffer.read_from(&mut *reader).await { Ok(n) if n > 0 => { warn!("encode_stream_callback_async read n={}", n); total += n; - let res = self.encode_data(&buf[..n]); - on_block(res).await? + let reusable_buffer = buffer_pool.acquire().await; + let res = self.encode_data_block_with_buffer(block_buffer.filled(n), reusable_buffer); + if let Some(block) = on_block(res).await? { + buffer_pool.release(block).await; + } } Ok(_) => { warn!("encode_stream_callback_async read unexpected ok"); @@ -520,11 +659,10 @@ impl Erasure { } Err(e) => { warn!("encode_stream_callback_async read error={:?}", e); - on_block(Err(e)).await?; + let _ = on_block(Err(e)).await?; break; } } - buf.clear(); } Ok(total) } @@ -747,8 +885,8 @@ mod tests { let tx = tx.clone(); async move { let shards = res.unwrap(); - tx.send(shards).await.unwrap(); - Ok(()) + tx.send(shards.iter().collect()).await.unwrap(); + Ok(Some(shards)) } }) .await @@ -760,6 +898,36 @@ mod tests { assert_eq!(collected_shards.len(), data_shards + parity_shards); } + #[test] + fn test_encode_data_with_buffer_supports_explicit_reuse() { + let erasure = Erasure::new(4, 2, 1024); + let reusable_capacity = erasure.shard_size() * erasure.total_shard_count(); + + let first_data = b"explicit reusable buffer path".repeat(32); + let first_block = erasure + .encode_data_with_buffer(&first_data, BytesMut::with_capacity(reusable_capacity)) + .expect("first encode should succeed"); + let reusable_buffer = first_block.into_reusable_buffer(); + assert!(reusable_buffer.capacity() >= reusable_capacity); + + let second_data = b"second encode through same reusable buffer".repeat(24); + let second_block = erasure + .encode_data_with_buffer(&second_data, reusable_buffer) + .expect("second encode should succeed"); + + let mut shards_opt: Vec>> = second_block.iter().map(|shard| Some(shard.to_vec())).collect(); + shards_opt[1] = None; + shards_opt[5] = None; + erasure.decode_data(&mut shards_opt).expect("decode should succeed"); + + let mut recovered = Vec::new(); + for shard in shards_opt.iter().take(erasure.data_shards) { + recovered.extend_from_slice(shard.as_ref().expect("data shard should exist after decode")); + } + recovered.truncate(second_data.len()); + assert_eq!(&recovered, &second_data); + } + #[tokio::test] async fn test_encode_stream_callback_async_channel_decode() { use std::io::Cursor; @@ -786,8 +954,8 @@ mod tests { let tx = tx.clone(); async move { let shards = res.unwrap(); - tx.send(shards).await.unwrap(); - Ok(()) + tx.send(shards.iter().collect()).await.unwrap(); + Ok(Some(shards)) } }) .await @@ -800,8 +968,8 @@ mod tests { // Test decode using the old API that operates in-place let mut decode_input: Vec>> = vec![None; data_shards + parity_shards]; - for i in 0..data_shards { - decode_input[i] = Some(shards[i].to_vec()); + for (i, shard) in shards.iter().enumerate().take(data_shards) { + decode_input[i] = Some(shard.to_vec()); } erasure.decode_data(&mut decode_input).unwrap(); @@ -1198,8 +1366,8 @@ mod tests { let tx = tx.clone(); async move { let shards = res.unwrap(); - tx.send(shards).await.unwrap(); - Ok(()) + tx.send(shards.iter().collect()).await.unwrap(); + Ok(Some(shards)) } }) .await @@ -1233,5 +1401,63 @@ mod tests { recovered.truncate(data_clone.len()); assert_eq!(&recovered, &data_clone); } + + #[tokio::test] + #[ignore] + async fn stress_simd_stream_callback_reuses_backing_buffers_across_many_blocks() { + use std::io::Cursor; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::sync::Mutex; + + let data_shards = 4; + let parity_shards = 2; + let block_size = 1024; + let erasure = Arc::new(Erasure::new(data_shards, parity_shards, block_size)); + + let sample = + b"SIMD stress callback test payload that intentionally spans many blocks to exercise reusable backing buffers."; + let data = sample.repeat((4 * 1024 * 1024 / sample.len()).max(1)); + let data_clone = data.clone(); + let mut reader = Cursor::new(data); + + let recovered = Arc::new(Mutex::new(Vec::with_capacity(data_clone.len()))); + let block_count = Arc::new(AtomicUsize::new(0)); + let erasure_for_callback = erasure.clone(); + let recovered_for_callback = recovered.clone(); + let block_count_for_callback = block_count.clone(); + + erasure + .clone() + .encode_stream_callback_async::<_, _, (), _>(&mut reader, move |res| { + let erasure_for_callback = erasure_for_callback.clone(); + let recovered_for_callback = recovered_for_callback.clone(); + let block_count_for_callback = block_count_for_callback.clone(); + async move { + let shards = res.unwrap(); + block_count_for_callback.fetch_add(1, Ordering::Relaxed); + + let mut shards_opt: Vec>> = shards.iter().map(|b| Some(b.to_vec())).collect(); + shards_opt[1] = None; + shards_opt[5] = None; + erasure_for_callback.decode_data(&mut shards_opt).unwrap(); + + let mut recovered = recovered_for_callback.lock().await; + for shard in shards_opt.iter().take(data_shards) { + recovered.extend_from_slice(shard.as_ref().unwrap()); + } + + Ok(Some(shards)) + } + }) + .await + .unwrap(); + + assert!(block_count.load(Ordering::Relaxed) > 1024); + + let mut recovered = recovered.lock().await; + recovered.truncate(data_clone.len()); + assert_eq!(&*recovered, &data_clone); + } } } diff --git a/crates/ecstore/src/erasure_coding/heal.rs b/crates/ecstore/src/erasure_coding/heal.rs index 422ae7dac..66b6d0802 100644 --- a/crates/ecstore/src/erasure_coding/heal.rs +++ b/crates/ecstore/src/erasure_coding/heal.rs @@ -77,7 +77,7 @@ impl super::Erasure { let available_writers = writers.iter().filter(|w| w.is_some()).count(); let write_quorum = available_writers.max(1); // At least 1 writer must succeed let mut writers = MultiWriter::new(writers, write_quorum); - writers.write(shards).await?; + writers.write(&shards).await?; } Ok(()) diff --git a/crates/ecstore/src/rpc/remote_disk.rs b/crates/ecstore/src/rpc/remote_disk.rs index dd7945f60..781091ac2 100644 --- a/crates/ecstore/src/rpc/remote_disk.rs +++ b/crates/ecstore/src/rpc/remote_disk.rs @@ -30,8 +30,10 @@ use crate::{ }; use bytes::Bytes; use futures::lock::Mutex; +use futures_util::StreamExt; use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE}; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; +use rustfs_io_core::{BoxChunkStream, IoChunk}; use rustfs_protos::proto_gen::node_service::RenamePartRequest; use rustfs_protos::proto_gen::node_service::{ CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, @@ -40,7 +42,7 @@ use rustfs_protos::proto_gen::node_service::{ RenameFileRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient, }; -use rustfs_rio::{HttpReader, HttpWriter}; +use rustfs_rio::{HttpReader, HttpWriter, open_http_byte_stream}; use serde::{Serialize, de::DeserializeOwned}; use std::{ io::Cursor, @@ -1071,6 +1073,33 @@ impl DiskAPI for RemoteDisk { Ok(Bytes::from(buffer)) } + #[tracing::instrument(level = "debug", skip(self))] + async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result { + if self.health.is_faulty() { + return Err(DiskError::FaultyDisk); + } + let disk = self.disk_ref().await; + + let url = format!( + "{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}", + self.endpoint.grid_host(), + urlencoding::encode(&disk), + urlencoding::encode(volume), + urlencoding::encode(path), + offset, + length + ); + + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + build_auth_headers(&url, &Method::GET, &mut headers); + + let stream = open_http_byte_stream(url, Method::GET, headers, None) + .await? + .map(|result| result.map(IoChunk::Shared)); + Ok(Box::pin(stream)) + } + #[tracing::instrument(level = "debug", skip(self))] async fn append_file(&self, volume: &str, path: &str) -> Result { info!("append_file {}/{}", volume, path); diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 7a150245b..b474d0c59 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -52,9 +52,9 @@ use crate::{ event_notification::{EventArgs, send_event}, global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure}, store_api::{ - BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, - HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectsV2Info, ListOperations, MakeBucketOptions, MultipartInfo, - MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations, PartInfo, PutObjReader, StorageAPI, + BucketInfo, BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, DeleteBucketOptions, DeletedObject, + GetObjectReader, HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectsV2Info, ListOperations, MakeBucketOptions, + MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations, PartInfo, StorageAPI, }, store_init::load_format_erasure, }; @@ -118,6 +118,66 @@ use tokio::{ time::{interval, timeout}, }; use tokio_util::sync::CancellationToken; + +const ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES: &str = "RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES"; +const ENV_RUSTFS_PUT_FORCE_DISABLE_INLINE: &str = "RUSTFS_PUT_FORCE_DISABLE_INLINE"; +const SLOW_PUT_STORAGE_PHASE_DEBUG_THRESHOLD_MS: u64 = 100; +const SLOW_PUT_STORAGE_PHASE_WARN_THRESHOLD_MS: u64 = 1_000; +const SLOW_PUT_STORAGE_PHASE_ERROR_THRESHOLD_MS: u64 = 5_000; + +fn env_flag_enabled(name: &str) -> bool { + rustfs_utils::get_env_bool(name, false) +} + +fn env_non_negative_usize(name: &str) -> Option { + rustfs_utils::get_env_opt_usize(name) +} + +fn resolved_put_inline_buffer_enabled(object_size: i64, inline_by_topology: bool) -> bool { + if !inline_by_topology || object_size < 0 { + return false; + } + + if env_flag_enabled(ENV_RUSTFS_PUT_FORCE_DISABLE_INLINE) { + return false; + } + + env_non_negative_usize(ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES) + .map(|value| usize::try_from(object_size).is_ok_and(|size| size <= value)) + .unwrap_or(inline_by_topology) +} + +fn log_put_storage_phase( + bucket: &str, + object: &str, + phase: &str, + elapsed: Duration, + object_size: i64, + inline_selected: bool, + write_quorum: usize, +) { + let duration_ms = elapsed.as_millis() as u64; + if duration_ms < SLOW_PUT_STORAGE_PHASE_DEBUG_THRESHOLD_MS { + return; + } + + if duration_ms >= SLOW_PUT_STORAGE_PHASE_ERROR_THRESHOLD_MS { + error!( + phase, + duration_ms, object_size, inline_selected, write_quorum, bucket, object, "PUT storage phase is critically slow" + ); + } else if duration_ms >= SLOW_PUT_STORAGE_PHASE_WARN_THRESHOLD_MS { + warn!( + phase, + duration_ms, object_size, inline_selected, write_quorum, bucket, object, "PUT storage phase is slow" + ); + } else { + debug!( + phase, + duration_ms, object_size, inline_selected, write_quorum, bucket, object, "PUT storage phase exceeded debug threshold" + ); + } +} use tracing::error; use tracing::{debug, info, warn}; use uuid::Uuid; @@ -151,6 +211,9 @@ mod read; mod replication; mod write; +#[doc(hidden)] +pub use read::collect_direct_data_shard_chunks_for_benchmark; + /// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds) /// Defaults to 30 seconds if not set or invalid pub fn get_lock_acquire_timeout() -> Duration { @@ -692,7 +755,13 @@ impl ObjectIO for SetDisks { } #[tracing::instrument(skip(self, data,))] - async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result { + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut ChunkNativePutData, + opts: &ObjectOptions, + ) -> Result { let disks = self.get_disks_internal().await; let mut object_lock_guard = None; @@ -774,13 +843,18 @@ impl ObjectIO for SetDisks { let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); let is_inline_buffer = { - if let Some(sc) = GLOBAL_STORAGE_CLASS.get() { + let inline_by_topology = if let Some(sc) = GLOBAL_STORAGE_CLASS.get() { sc.should_inline(erasure.shard_file_size(data.size()), opts.versioned) } else { false - } + }; + resolved_put_inline_buffer_enabled(data.size(), inline_by_topology) }; + if is_inline_buffer { + rustfs_io_metrics::record_put_inline_selected(data.size(), opts.versioned); + } + let writer_setup_start = Instant::now(); let mut writers = Vec::with_capacity(shuffle_disks.len()); let mut errors = Vec::with_capacity(shuffle_disks.len()); for disk_op in shuffle_disks.iter() { @@ -814,6 +888,15 @@ impl ObjectIO for SetDisks { writers.push(None); } } + log_put_storage_phase( + bucket, + object, + "writer_setup", + writer_setup_start.elapsed(), + data.size(), + is_inline_buffer, + write_quorum, + ); let nil_count = errors.iter().filter(|&e| e.is_none()).count(); if nil_count < write_quorum { @@ -825,23 +908,33 @@ impl ObjectIO for SetDisks { return Err(Error::other(format!("not enough disks to write: {errors:?}"))); } - let stream = mem::replace( - &mut data.stream, - HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?, - ); - - let (reader, w_size) = match Arc::new(erasure).encode(stream, &mut writers, write_quorum).await { - Ok((r, w)) => (r, w), + let object_size = data.size(); + let encode_write_start = Instant::now(); + let w_size = match Self::write_chunk_native_put_data(data, Arc::new(erasure), &mut writers, write_quorum).await { + Ok(written) => written, Err(e) => { + log_put_storage_phase( + bucket, + object, + "encode_write", + encode_write_start.elapsed(), + object_size, + is_inline_buffer, + write_quorum, + ); error!("encode err {:?}", e); return Err(e.into()); } }; // TODO: delete temporary directory on error - - let _ = mem::replace(&mut data.stream, reader); - // if let Err(err) = close_bitrot_writers(&mut writers).await { - // error!("close_bitrot_writers err {:?}", err); - // } + log_put_storage_phase( + bucket, + object, + "encode_write", + encode_write_start.elapsed(), + data.size(), + is_inline_buffer, + write_quorum, + ); if (w_size as i64) < data.size() { warn!("put_object write size < data.size(), w_size={}, data.size={}", w_size, data.size()); @@ -856,11 +949,11 @@ impl ObjectIO for SetDisks { insert_str(&mut user_defined, SUFFIX_COMPRESSION_SIZE, w_size.to_string()); } - let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec()); + let index_op = data.index_bytes(); //TODO: userDefined - let etag = data.stream.try_resolve_etag().unwrap_or_default(); + let etag = data.resolve_etag().unwrap_or_default(); user_defined.insert("etag".to_owned(), etag.clone()); @@ -877,9 +970,9 @@ impl ObjectIO for SetDisks { } if fi.checksum.is_none() - && let Some(content_hash) = data.as_hash_reader().content_hash() + && let Some(content_hash) = data.content_hash_bytes()? { - fi.checksum = Some(content_hash.to_bytes(&[])); + fi.checksum = Some(content_hash); } if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS) @@ -917,6 +1010,7 @@ impl ObjectIO for SetDisks { drop(writers); // drop writers to close all files, this is to prevent FileAccessDenied errors when renaming data + let post_write_lock_start = Instant::now(); if !opts.no_lock && object_lock_guard.is_none() { let ns_lock = self.new_ns_lock(bucket, object).await?; object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| { @@ -926,7 +1020,17 @@ impl ObjectIO for SetDisks { )) })?); } + log_put_storage_phase( + bucket, + object, + "post_write_lock", + post_write_lock_start.elapsed(), + data.size(), + is_inline_buffer, + write_quorum, + ); + let finalize_start = Instant::now(); let (online_disks, _, op_old_dir) = Self::rename_data( &shuffle_disks, RUSTFS_META_TMP_BUCKET, @@ -936,7 +1040,18 @@ impl ObjectIO for SetDisks { object, write_quorum, ) - .await?; + .await + .inspect_err(|_| { + log_put_storage_phase( + bucket, + object, + "finalize", + finalize_start.elapsed(), + data.size(), + is_inline_buffer, + write_quorum, + ); + })?; if let Some(old_dir) = op_old_dir { self.commit_rename_data_dir(&online_disks, bucket, object, &old_dir.to_string(), write_quorum) @@ -946,6 +1061,15 @@ impl ObjectIO for SetDisks { drop(object_lock_guard); // drop object lock guard to release the lock self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?; + log_put_storage_phase( + bucket, + object, + "finalize", + finalize_start.elapsed(), + data.size(), + is_inline_buffer, + write_quorum, + ); for (i, op_disk) in online_disks.iter().enumerate() { if let Some(disk) = op_disk @@ -1864,6 +1988,9 @@ impl ObjectOperations for SetDisks { if let Some(ref version_id) = opts.version_id { fi.version_id = Uuid::parse_str(version_id).ok(); } + if let Some(checksum) = &opts.resolved_checksum { + fi.checksum = Some(checksum.clone()); + } self.update_object_meta(bucket, object, fi.clone(), &online_disks) .await @@ -2090,7 +2217,7 @@ impl ObjectOperations for SetDisks { let gr = gr.unwrap(); let reader = BufReader::new(gr.stream); let hash_reader = HashReader::from_stream(reader, gr.object_info.size, gr.object_info.size, None, None, false)?; - let mut p_reader = PutObjReader::new(hash_reader); + let mut p_reader = ChunkNativePutData::new(hash_reader); return match self_.clone().put_object(bucket, object, &mut p_reader, &ropts).await { Ok(restored_info) => { send_event(EventArgs { @@ -2158,7 +2285,7 @@ impl ObjectOperations for SetDisks { }; let reader = BufReader::new(gr.stream); let hash_reader = HashReader::from_stream(reader, part_info.actual_size, part_info.actual_size, None, None, false)?; - let mut p_reader = PutObjReader::new(hash_reader); + let mut p_reader = ChunkNativePutData::new(hash_reader); let p_info = self_ .clone() .put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ObjectOptions::default()) @@ -2382,7 +2509,7 @@ impl MultipartOperations for SetDisks { object: &str, upload_id: &str, part_id: usize, - data: &mut PutObjReader, + data: &mut ChunkNativePutData, opts: &ObjectOptions, ) -> Result { let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); @@ -2394,9 +2521,8 @@ impl MultipartOperations for SetDisks { if let Some(checksum) = fi.metadata.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM) && !checksum.is_empty() && data - .as_hash_reader() .content_crc_type() - .is_none_or(|v| v.to_string() != *checksum) + .is_none_or(|v: rustfs_rio::ChecksumType| v.to_string() != *checksum) { return Err(Error::other(format!("checksum mismatch: {checksum}"))); } @@ -2461,14 +2587,7 @@ impl MultipartOperations for SetDisks { return Err(Error::other(format!("not enough disks to write: {errors:?}"))); } - let stream = mem::replace( - &mut data.stream, - HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?, - ); - - let (reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: delete temporary directory on error - - let _ = mem::replace(&mut data.stream, reader); + let w_size = Self::write_chunk_native_put_data(data, Arc::new(erasure), &mut writers, write_quorum).await?; // TODO: delete temporary directory on error if (w_size as i64) < data.size() { warn!("put_object_part write size < data.size(), w_size={}, data.size={}", w_size, data.size()); @@ -2479,9 +2598,9 @@ impl MultipartOperations for SetDisks { ))); } - let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec()); + let index_op = data.index_bytes(); - let mut etag = data.stream.try_resolve_etag().unwrap_or_default(); + let mut etag = data.resolve_etag().unwrap_or_default(); if let Some(ref tag) = opts.preserve_etag { etag = tag.clone(); @@ -2495,7 +2614,7 @@ impl MultipartOperations for SetDisks { } } - let checksums = data.as_hash_reader().content_crc(); + let checksums = data.content_crc(); let part_info = ObjectPartInfo { etag: etag.clone(), @@ -4212,6 +4331,31 @@ mod tests { } } + #[test] + #[serial] + fn resolved_put_inline_buffer_enabled_honors_disable_env() { + temp_env::with_var(ENV_RUSTFS_PUT_FORCE_DISABLE_INLINE, Some("true"), || { + assert!(!resolved_put_inline_buffer_enabled(4096, true)); + }); + } + + #[test] + #[serial] + fn resolved_put_inline_buffer_enabled_honors_max_bytes_override() { + temp_env::with_var(ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES, Some("4096"), || { + assert!(resolved_put_inline_buffer_enabled(4096, true)); + assert!(!resolved_put_inline_buffer_enabled(4097, true)); + }); + } + + #[test] + #[serial] + fn resolved_put_inline_buffer_enabled_ignores_invalid_override() { + temp_env::with_var(ENV_RUSTFS_PUT_INLINE_OBJECT_MAX_BYTES, Some("invalid"), || { + assert!(resolved_put_inline_buffer_enabled(4096, true)); + }); + } + async fn current_setup_type() -> SetupType { if is_dist_erasure().await { SetupType::DistErasure diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 08f6ec838..c527a7769 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -13,9 +13,879 @@ // limitations under the License. use super::*; +use crate::bitrot::create_bitrot_chunk_stream; +use crate::erasure_coding::decode::ErasureChunkDecoder; +use crate::erasure_coding::{calc_shard_size, calc_shard_size_legacy}; +use crate::store_api::{GetObjectChunkCopyMode, GetObjectChunkPath, GetObjectChunkResult}; +use bytes::BytesMut; +use futures_util::{Stream, StreamExt, stream}; use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE}; +use rustfs_io_core::{BoxChunkStream, IoChunk}; +use std::io; +use std::pin::Pin; +use std::sync::Mutex; +use std::task::{Context, Poll}; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; +use tokio_util::io::ReaderStream; + +struct ChannelChunkStream { + receiver: Mutex>>, +} + +impl ChannelChunkStream { + fn new(receiver: UnboundedReceiver>) -> Self { + Self { + receiver: Mutex::new(receiver), + } + } +} + +struct DirectShardCursor { + stream: BoxChunkStream, + current_chunk: Option, + current_offset: usize, +} + +impl DirectShardCursor { + fn new(stream: BoxChunkStream) -> Self { + Self { + stream, + current_chunk: None, + current_offset: 0, + } + } + + fn current_remaining(&self) -> usize { + self.current_chunk + .as_ref() + .map(|chunk| chunk.len().saturating_sub(self.current_offset)) + .unwrap_or(0) + } + + fn consume_current(&mut self, len: usize) { + self.current_offset += len; + if self.current_remaining() == 0 { + self.current_chunk = None; + self.current_offset = 0; + } + } + + async fn ensure_chunk(&mut self, shard_index: usize) -> io::Result { + if self.current_chunk.is_some() && self.current_remaining() > 0 { + return Ok(true); + } + + match self.stream.next().await { + Some(Ok(chunk)) => { + self.current_chunk = Some(chunk); + self.current_offset = 0; + Ok(true) + } + Some(Err(err)) => Err(err), + None => { + debug!(shard_index, "direct shard cursor reached EOF"); + Ok(false) + } + } + } +} + +impl Stream for ChannelChunkStream { + type Item = io::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut receiver = self.receiver.lock().unwrap(); + receiver.poll_recv(cx) + } +} + +struct ChannelChunkWriter { + sender: UnboundedSender>, + buffer: BytesMut, + chunk_size: usize, +} + +impl ChannelChunkWriter { + fn new(sender: UnboundedSender>, chunk_size: usize) -> Self { + Self { + sender, + buffer: BytesMut::with_capacity(chunk_size.max(1)), + chunk_size: chunk_size.max(1), + } + } + + fn push_buffer(&mut self) -> io::Result<()> { + if self.buffer.is_empty() { + return Ok(()); + } + + let bytes = self.buffer.split().freeze(); + self.sender + .send(Ok(IoChunk::Shared(bytes))) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "chunk stream receiver dropped")) + } + + fn finish(&mut self) -> io::Result<()> { + self.push_buffer() + } + + fn send_error(&self, err: io::Error) { + let _ = self.sender.send(Err(err)); + } +} + +impl AsyncWrite for ChannelChunkWriter { + fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + self.buffer.extend_from_slice(buf); + let chunk_size = self.chunk_size; + while self.buffer.len() >= chunk_size { + let chunk = self.buffer.split_to(chunk_size).freeze(); + if self.sender.send(Ok(IoChunk::Shared(chunk))).is_err() { + return Poll::Ready(Err(io::Error::new(io::ErrorKind::BrokenPipe, "chunk stream receiver dropped"))); + } + } + + Poll::Ready(Ok(buf.len())) + } + + fn poll_flush(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.push_buffer()) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.finish()) + } +} + +fn merge_chunk_copy_mode(current: GetObjectChunkCopyMode, next: GetObjectChunkCopyMode) -> GetObjectChunkCopyMode { + use GetObjectChunkCopyMode::{Reconstructed, SharedBytes, SingleCopy, TrueZeroCopy}; + + match (current, next) { + (Reconstructed, _) | (_, Reconstructed) => Reconstructed, + (SingleCopy, _) | (_, SingleCopy) => SingleCopy, + (SharedBytes, _) | (_, SharedBytes) => SharedBytes, + (TrueZeroCopy, TrueZeroCopy) => TrueZeroCopy, + } +} + +fn multipart_logical_part_size(fi: &FileInfo, part_index: usize) -> usize { + let part = &fi.parts[part_index]; + if part.actual_size > 0 { + part.actual_size as usize + } else { + part.size + } +} + +fn multipart_logical_total_size(fi: &FileInfo) -> usize { + if fi.parts.is_empty() { + return fi.size.max(0) as usize; + } + + fi.parts + .iter() + .map(|part| { + if part.actual_size > 0 { + part.actual_size as usize + } else { + part.size + } + }) + .sum() +} + +fn multipart_stored_part_size(fi: &FileInfo, part_index: usize) -> usize { + fi.parts[part_index].size +} + +fn multipart_stored_total_size(fi: &FileInfo) -> usize { + if fi.parts.is_empty() { + return fi.size.max(0) as usize; + } + + fi.parts.iter().map(|part| part.size).sum() +} + +fn multipart_to_logical_part_offset(fi: &FileInfo, offset: usize) -> Result<(usize, usize)> { + if offset == 0 { + return Ok((0, 0)); + } + + let mut part_offset = offset; + for (i, _) in fi.parts.iter().enumerate() { + let logical_part_size = multipart_logical_part_size(fi, i); + if part_offset < logical_part_size { + return Ok((i, part_offset)); + } + + part_offset -= logical_part_size; + } + + Err(Error::other("part not found")) +} + +fn multipart_to_stored_part_offset(fi: &FileInfo, offset: usize) -> Result<(usize, usize)> { + if offset == 0 { + return Ok((0, 0)); + } + + let mut part_offset = offset; + for (i, part) in fi.parts.iter().enumerate() { + if part_offset < part.size { + return Ok((i, part_offset)); + } + + part_offset -= part.size; + } + + Err(Error::other("part not found")) +} + +fn block_window( + offset: usize, + length: usize, + block_size: usize, + block_index: usize, + start_block: usize, + end_block: usize, +) -> (usize, usize) { + let end_remainder = offset.saturating_add(length) % block_size; + if start_block == end_block { + (offset % block_size, length) + } else if block_index == start_block { + (offset % block_size, block_size - (offset % block_size)) + } else if block_index == end_block { + (0, if end_remainder == 0 { block_size } else { end_remainder }) + } else { + (0, block_size) + } +} + +fn direct_block_shard_size( + total_size: usize, + block_size: usize, + data_shards: usize, + block_index: usize, + uses_legacy: bool, +) -> usize { + let block_start = block_index.saturating_mul(block_size); + let logical_block_size = total_size.saturating_sub(block_start).min(block_size); + if uses_legacy { + calc_shard_size_legacy(logical_block_size, data_shards) + } else { + calc_shard_size(logical_block_size, data_shards) + } +} + +#[allow(clippy::too_many_arguments)] +async fn send_direct_data_shard_chunks( + sender: UnboundedSender>, + shard_streams: Vec, + data_shards: usize, + block_size: usize, + total_size: usize, + uses_legacy: bool, + offset: usize, + length: usize, +) { + if length == 0 { + return; + } + + let start_block = offset / block_size; + let end_block = offset.saturating_add(length.saturating_sub(1)) / block_size; + let mut shard_cursors = shard_streams.into_iter().map(DirectShardCursor::new).collect::>(); + + for block_index in start_block..=end_block { + let (block_offset, block_length) = block_window(offset, length, block_size, block_index, start_block, end_block); + if block_length == 0 { + break; + } + + let shard_block_size = direct_block_shard_size(total_size, block_size, data_shards, block_index, uses_legacy); + let mut write_left = block_length; + let mut skip = block_offset; + + for (shard_index, shard_cursor) in shard_cursors.iter_mut().enumerate().take(data_shards) { + let mut shard_block_left = shard_block_size; + + while shard_block_left > 0 { + let has_chunk = match shard_cursor.ensure_chunk(shard_index).await { + Ok(has_chunk) => has_chunk, + Err(err) => { + let _ = sender.send(Err(err)); + return; + } + }; + + if !has_chunk { + let _ = sender.send(Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + format!("missing chunk for data shard {shard_index}"), + ))); + return; + } + + let chunk = shard_cursor.current_chunk.as_ref().expect("chunk should exist after ensure"); + let chunk_remaining = chunk.len().saturating_sub(shard_cursor.current_offset); + let take_from_chunk = chunk_remaining.min(shard_block_left); + if skip >= take_from_chunk { + skip -= take_from_chunk; + shard_cursor.consume_current(take_from_chunk); + shard_block_left -= take_from_chunk; + continue; + } + + let start = shard_cursor.current_offset + skip; + let available = take_from_chunk.saturating_sub(skip); + let take = available.min(write_left); + let out_chunk = if start == shard_cursor.current_offset && take == take_from_chunk { + chunk.slice(start, take).expect("full remaining slice should succeed") + } else { + match chunk.slice(start, take) { + Ok(chunk) => chunk, + Err(err) => { + let _ = sender.send(Err(err)); + return; + } + } + }; + + let consumed = skip + take; + skip = 0; + shard_cursor.consume_current(consumed); + shard_block_left -= consumed; + if sender.send(Ok(out_chunk)).is_err() { + return; + } + write_left -= take; + + if write_left == 0 { + break; + } + } + + if write_left == 0 { + break; + } + } + + if write_left != 0 { + let _ = sender.send(Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "not enough decoded shard data for requested block", + ))); + return; + } + } +} + +#[doc(hidden)] +pub async fn collect_direct_data_shard_chunks_for_benchmark( + shard_streams: Vec, + data_shards: usize, + block_size: usize, + total_size: usize, + uses_legacy: bool, + offset: usize, + length: usize, +) -> io::Result> { + let (tx, rx) = unbounded_channel(); + send_direct_data_shard_chunks(tx, shard_streams, data_shards, block_size, total_size, uses_legacy, offset, length).await; + + let mut stream = ChannelChunkStream::new(rx); + let mut chunks = Vec::new(); + while let Some(chunk) = stream.next().await { + chunks.push(chunk?); + } + + Ok(chunks) +} + +#[allow(clippy::too_many_arguments)] +async fn build_reconstructed_part_stream( + bucket: &str, + object: &str, + part_number: usize, + part_offset: usize, + part_length: usize, + part_size: usize, + read_offset: usize, + till_offset: usize, + files: &[FileInfo], + disks: &[Option], + erasure: &erasure_coding::Erasure, + checksum_algo: rustfs_utils::HashAlgorithm, + skip_verify_bitrot: bool, + use_zero_copy: bool, +) -> Result> { + let mut readers = Vec::with_capacity(disks.len()); + let mut errors = Vec::with_capacity(disks.len()); + for (idx, disk_op) in disks.iter().enumerate() { + match create_bitrot_reader( + files[idx].data.as_deref(), + disk_op.as_ref(), + bucket, + &format!("{}/{}/part.{}", object, files[idx].data_dir.unwrap_or_default(), part_number), + read_offset, + till_offset, + erasure.shard_size(), + checksum_algo.clone(), + skip_verify_bitrot, + use_zero_copy, + ) + .await + { + Ok(Some(reader)) => { + readers.push(Some(reader)); + errors.push(None); + } + Ok(None) => { + readers.push(None); + errors.push(Some(DiskError::DiskNotFound)); + } + Err(err) => { + readers.push(None); + errors.push(Some(err)); + } + } + } + + let available_shards = errors.iter().filter(|error| error.is_none()).count(); + if available_shards < erasure.data_shards { + return Ok(None); + } + + let missing_shards = readers.len().saturating_sub(available_shards); + if missing_shards > 0 { + debug!( + bucket, + object, + part_number, + missing_shards, + available_shards, + data_shards = erasure.data_shards, + parity_shards = erasure.parity_shards, + "using reconstructed part stream for missing shards" + ); + } + + let (tx, rx) = unbounded_channel(); + let bucket = bucket.to_string(); + let object = object.to_string(); + let erasure = erasure.clone(); + tokio::spawn(async move { + let mut decoder = match ErasureChunkDecoder::new(erasure, readers, part_offset, part_length, part_size) { + Ok(decoder) => decoder, + Err(err) => { + let _ = tx.send(Err(err)); + return; + } + }; + + loop { + match decoder.next_chunks().await { + Ok(Some(chunks)) => { + for chunk in chunks { + if tx.send(Ok(chunk)).is_err() { + return; + } + } + } + Ok(None) => break, + Err(err) => { + let _ = tx.send(Err(err)); + return; + } + } + } + + if let Some(err) = decoder.finish_error() { + let _ = tx.send(Err(err)); + return; + } + + if let Some(disk_err) = decoder.take_healable_error() { + let allow_heal_only = + decoder.written() == part_length && matches!(disk_err, DiskError::FileNotFound | DiskError::FileCorrupt); + if !allow_heal_only { + let _ = tx.send(Err(io::Error::other(disk_err.to_string()))); + return; + } + + debug!( + bucket, + object, + part_number, + bytes_written = decoder.written(), + error = %disk_err, + "reconstructed part completed with healable shard error" + ); + } + }); + + Ok(Some(Box::pin(ChannelChunkStream::new(rx)))) +} impl SetDisks { + #[tracing::instrument(level = "debug", skip(self, h, opts))] + pub(crate) async fn get_object_chunks( + &self, + bucket: &str, + object: &str, + range: Option, + h: HeaderMap, + opts: &ObjectOptions, + ) -> Result { + let lock_optimization_enabled = is_lock_optimization_enabled(); + + let read_lock_guard = if !opts.no_lock { + let acquire_start = Instant::now(); + + if is_deadlock_detection_enabled() { + debug!( + lock_id = format!("{}:{}", bucket, object), + lock_type = "read", + resource = format!("{}/{}", bucket, object), + "Waiting for read lock" + ); + } + + let guard = self + .new_ns_lock(bucket, object) + .await? + .get_read_lock(get_lock_acquire_timeout()) + .await + .map_err(|e| { + Error::other(format!( + "Failed to acquire read lock: {}", + self.format_lock_error_from_error(bucket, object, "read", &e) + )) + })?; + + let _lock_id = record_lock_acquire(bucket, object, "read"); + metrics::counter!("rustfs.lock.acquire.total", "type" => "read").increment(1); + metrics::histogram!("rustfs.lock.acquire.duration.seconds").record(acquire_start.elapsed().as_secs_f64()); + + Some(guard) + } else { + None + }; + + let (fi, files, disks) = self + .get_object_fileinfo(bucket, object, opts, true) + .await + .map_err(|err| to_object_err(err, vec![bucket, object]))?; + let object_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended); + + if object_info.delete_marker { + if opts.version_id.is_none() { + return Err(to_object_err(Error::FileNotFound, vec![bucket, object])); + } + return Err(to_object_err(Error::MethodNotAllowed, vec![bucket, object])); + } + + if object_info.size == 0 { + return Ok(GetObjectChunkResult { + stream: Box::pin(stream::iter(Vec::>::new())), + path: GetObjectChunkPath::Direct, + copy_mode: GetObjectChunkCopyMode::SharedBytes, + }); + } + + let (bridge_offset, bridge_length) = if fi.parts.is_empty() { + (0, fi.size) + } else { + let total_size = multipart_logical_total_size(&fi); + if let Some(range) = &range { + let (offset, length) = range + .get_offset_length(total_size as i64) + .map_err(|err| to_object_err(err, vec![bucket, object]))?; + (offset, length) + } else { + (0, total_size as i64) + } + }; + + if object_info.is_remote() { + let mut opts = opts.clone(); + if object_info.parts.len() == 1 { + opts.part_number = Some(1); + } + let gr = get_transitioned_object_reader(bucket, object, &range, &h, &object_info, &opts).await?; + let stream = ReaderStream::new(gr.stream).map(|result| result.map(IoChunk::Shared)); + return Ok(GetObjectChunkResult { + stream: Box::pin(stream), + path: GetObjectChunkPath::Bridge, + copy_mode: GetObjectChunkCopyMode::SingleCopy, + }); + } + + if fi.erasure.data_blocks > 0 { + let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(&disks, &files, &fi); + let total_size = multipart_logical_total_size(&fi); + let requested_length = if let Some(range) = &range { + let (offset, length) = range + .get_offset_length(total_size as i64) + .map_err(|err| to_object_err(err, vec![bucket, object]))?; + (offset, length as usize) + } else { + (0, total_size) + }; + + let (part_index, mut part_offset) = multipart_to_logical_part_offset(&fi, requested_length.0)?; + let mut end_offset = requested_length.0; + if requested_length.1 > 0 { + end_offset += requested_length.1 - 1; + } + let (last_part_index, _) = multipart_to_logical_part_offset(&fi, end_offset)?; + + let use_zero_copy = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE); + let single_shard_file = if fi.erasure.data_blocks == 1 { + Some( + files + .first() + .ok_or_else(|| Error::other("single-shard multipart metadata missing"))?, + ) + } else { + None + }; + let single_shard_disk = if fi.erasure.data_blocks == 1 { + Some( + disks + .first() + .ok_or_else(|| Error::other("single-shard multipart disk slot missing"))?, + ) + } else { + None + }; + let mut part_streams = Vec::new(); + let mut part_total_read = 0usize; + let mut merged_copy_mode = GetObjectChunkCopyMode::TrueZeroCopy; + + for current_part in part_index..=last_part_index { + let part_number = fi.parts[current_part].number; + let part_size = multipart_logical_part_size(&fi, current_part); + let mut part_length = part_size - part_offset; + if part_length > (requested_length.1 - part_total_read) { + part_length = requested_length.1 - part_total_read; + } + let checksum_info = fi.erasure.get_checksum_info(part_number); + let checksum_algo = + if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S { + rustfs_utils::HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm.clone() + }; + + if fi.erasure.data_blocks == 1 { + let single_shard_file = single_shard_file.expect("single-shard multipart metadata must exist"); + let single_shard_disk = single_shard_disk.expect("single-shard multipart disk slot must exist"); + let data_dir = single_shard_file + .data_dir + .as_ref() + .map(uuid::Uuid::to_string) + .unwrap_or_default(); + let data_path = format!("{}/{}/part.{}", object, data_dir, part_number); + let chunk_result = create_bitrot_chunk_stream( + single_shard_file.data.as_deref(), + single_shard_disk.as_ref(), + bucket, + &data_path, + part_offset, + part_length, + part_size, + fi.erasure.shard_size(), + checksum_algo, + opts.skip_verify_bitrot, + use_zero_copy, + ) + .await?; + + let Some(chunk_result) = chunk_result else { + part_streams.clear(); + break; + }; + merged_copy_mode = merge_chunk_copy_mode(merged_copy_mode, chunk_result.copy_mode); + part_streams.push(chunk_result.stream); + } else { + let erasure = erasure_coding::Erasure::new_with_options( + fi.erasure.data_blocks, + fi.erasure.parity_blocks, + fi.erasure.block_size, + fi.uses_legacy_checksum, + ); + let read_offset = (part_offset / erasure.block_size) * erasure.shard_size(); + let till_offset = erasure.shard_file_offset(part_offset, part_length, part_size); + let shard_length = till_offset.saturating_sub(read_offset); + let shard_total_size = erasure.shard_file_size(part_size as i64) as usize; + let mut shard_streams = Vec::with_capacity(erasure.data_shards); + let mut part_copy_mode = GetObjectChunkCopyMode::TrueZeroCopy; + let mut needs_reconstruct = false; + + for shard_index in 0..erasure.data_shards { + let data_path = + format!("{}/{}/part.{}", object, files[shard_index].data_dir.unwrap_or_default(), part_number); + let chunk_result = match create_bitrot_chunk_stream( + files[shard_index].data.as_deref(), + disks[shard_index].as_ref(), + bucket, + &data_path, + read_offset, + shard_length, + shard_total_size, + erasure.shard_size(), + checksum_algo.clone(), + opts.skip_verify_bitrot, + use_zero_copy, + ) + .await + { + Ok(Some(chunk_result)) => chunk_result, + Ok(None) => { + needs_reconstruct = true; + shard_streams.clear(); + break; + } + Err(err) => { + debug!( + bucket, + object, + part_number, + shard_index, + error = %err, + "multi-shard direct chunk path unavailable, falling back to decoded read path" + ); + needs_reconstruct = true; + shard_streams.clear(); + break; + } + }; + part_copy_mode = merge_chunk_copy_mode(part_copy_mode, chunk_result.copy_mode); + shard_streams.push(chunk_result.stream); + } + + if needs_reconstruct { + let reconstructed_stream = match build_reconstructed_part_stream( + bucket, + object, + part_number, + part_offset, + part_length, + part_size, + read_offset, + till_offset, + &files, + &disks, + &erasure, + checksum_algo, + opts.skip_verify_bitrot, + use_zero_copy, + ) + .await? + { + Some(stream) => stream, + None => { + part_streams.clear(); + break; + } + }; + + merged_copy_mode = merge_chunk_copy_mode(merged_copy_mode, GetObjectChunkCopyMode::Reconstructed); + part_streams.push(reconstructed_stream); + part_total_read += part_length; + part_offset = 0; + continue; + } + + if shard_streams.len() != erasure.data_shards { + part_streams.clear(); + break; + } + + let (tx, rx) = unbounded_channel(); + tokio::spawn(send_direct_data_shard_chunks( + tx, + shard_streams, + erasure.data_shards, + erasure.block_size, + part_size, + fi.uses_legacy_checksum, + part_offset, + part_length, + )); + + merged_copy_mode = merge_chunk_copy_mode(merged_copy_mode, part_copy_mode); + part_streams.push(Box::pin(ChannelChunkStream::new(rx))); + } + part_total_read += part_length; + part_offset = 0; + } + + if !part_streams.is_empty() { + return Ok(GetObjectChunkResult { + stream: Box::pin(stream::iter(part_streams).flatten()), + path: GetObjectChunkPath::Direct, + copy_mode: merged_copy_mode, + }); + } + } + + let read_lock_guard = if lock_optimization_enabled { + if read_lock_guard.is_some() { + let lock_id = format!("{}:{}", bucket, object); + record_lock_release(bucket, object, &lock_id, "read"); + metrics::counter!("rustfs.lock.release.early.total", "type" => "read").increment(1); + } + drop(read_lock_guard); + debug!(bucket, object, "Lock optimization: released read lock after metadata read"); + None + } else { + read_lock_guard + }; + + let chunk_size = get_duplex_buffer_size(); + let bucket = bucket.to_owned(); + let object = object.to_owned(); + let set_index = self.set_index; + let pool_index = self.pool_index; + let skip_verify = opts.skip_verify_bitrot; + let (tx, rx) = unbounded_channel(); + + tokio::spawn(async move { + let _guard = read_lock_guard; + let mut writer = ChannelChunkWriter::new(tx, chunk_size); + if let Err(err) = Self::get_object_with_fileinfo( + &bucket, + &object, + bridge_offset, + bridge_length, + &mut writer, + fi, + files, + &disks, + set_index, + pool_index, + skip_verify, + ) + .await + { + error!("get_object_with_fileinfo {bucket}/{object} err {:?}", err); + writer.send_error(io::Error::other(err.to_string())); + } + + if let Err(err) = writer.finish() { + debug!(bucket, object, error = %err, "failed to flush chunk writer"); + } + }); + + Ok(GetObjectChunkResult { + stream: Box::pin(ChannelChunkStream::new(rx)), + path: GetObjectChunkPath::Bridge, + copy_mode: GetObjectChunkCopyMode::SingleCopy, + }) + } + pub(super) async fn read_parts( disks: &[Option], bucket: &str, @@ -583,27 +1453,42 @@ impl SetDisks { debug!(bucket, object, requested_length = length, offset, "get_object_with_fileinfo start"); let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, &files, &fi); - let total_size = fi.size as usize; - - let length = if length < 0 { - fi.size as usize - offset + let logical_total_size = multipart_logical_total_size(&fi); + let use_stored_part_sizes = length > logical_total_size as i64; + let total_size = if use_stored_part_sizes { + multipart_stored_total_size(&fi) } else { - length as usize + logical_total_size }; - if offset > total_size || offset + length > total_size { + let length = if length < 0 { total_size - offset } else { length as usize }; + + let Some(end_offset_exclusive) = offset.checked_add(length) else { + error!("get_object_with_fileinfo offset overflow: {}, length: {}", offset, length); + return Err(Error::other("offset out of range")); + }; + + if offset > total_size || end_offset_exclusive > total_size { error!("get_object_with_fileinfo offset out of range: {}, total_size: {}", offset, total_size); return Err(Error::other("offset out of range")); } - let (part_index, mut part_offset) = fi.to_part_offset(offset)?; + let (part_index, mut part_offset) = if use_stored_part_sizes { + multipart_to_stored_part_offset(&fi, offset)? + } else { + multipart_to_logical_part_offset(&fi, offset)? + }; let mut end_offset = offset; if length > 0 { - end_offset += length - 1 + end_offset = end_offset_exclusive - 1; } - let (last_part_index, last_part_relative_offset) = fi.to_part_offset(end_offset)?; + let (last_part_index, last_part_relative_offset) = if use_stored_part_sizes { + multipart_to_stored_part_offset(&fi, end_offset)? + } else { + multipart_to_logical_part_offset(&fi, end_offset)? + }; debug!( bucket, @@ -635,7 +1520,11 @@ impl SetDisks { } let part_number = fi.parts[current_part].number; - let part_size = fi.parts[current_part].size; + let part_size = if use_stored_part_sizes { + multipart_stored_part_size(&fi, current_part) + } else { + multipart_logical_part_size(&fi, current_part) + }; let mut part_length = part_size - part_offset; if part_length > (length - total_read) { part_length = length - total_read @@ -833,3 +1722,103 @@ impl SetDisks { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use futures_util::StreamExt; + + #[tokio::test] + async fn send_direct_data_shard_chunks_reassembles_multi_block_range() { + let data_shards = 4; + let block_size = 16; + let shard_streams: Vec = vec![ + Box::pin(stream::iter(vec![ + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[0, 1, 2, 3]))), + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[16, 17, 18, 19]))), + ])), + Box::pin(stream::iter(vec![ + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[4, 5, 6, 7]))), + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[20, 21, 22, 23]))), + ])), + Box::pin(stream::iter(vec![ + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[8, 9, 10, 11]))), + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[24, 25, 26, 27]))), + ])), + Box::pin(stream::iter(vec![ + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[12, 13, 14, 15]))), + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[28, 29, 30, 31]))), + ])), + ]; + let (tx, rx) = unbounded_channel(); + + send_direct_data_shard_chunks(tx, shard_streams, data_shards, block_size, 32, false, 3, 18).await; + + let mut stream = ChannelChunkStream::new(rx); + let mut collected = Vec::new(); + while let Some(chunk) = stream.next().await { + collected.extend_from_slice(&chunk.unwrap().as_bytes()); + } + + assert_eq!(collected, (3u8..21).collect::>()); + } + + #[tokio::test] + async fn send_direct_data_shard_chunks_keeps_block_boundaries_with_cross_block_chunks() { + let data_shards = 4; + let block_size = 16; + let shard_streams: Vec = vec![ + Box::pin(stream::iter(vec![Ok(IoChunk::Shared(Bytes::copy_from_slice(&[ + 0, 1, 2, 3, 16, 17, 18, 19, + ])))])), + Box::pin(stream::iter(vec![Ok(IoChunk::Shared(Bytes::copy_from_slice(&[ + 4, 5, 6, 7, 20, 21, 22, 23, + ])))])), + Box::pin(stream::iter(vec![Ok(IoChunk::Shared(Bytes::copy_from_slice(&[ + 8, 9, 10, 11, 24, 25, 26, 27, + ])))])), + Box::pin(stream::iter(vec![Ok(IoChunk::Shared(Bytes::copy_from_slice(&[ + 12, 13, 14, 15, 28, 29, 30, 31, + ])))])), + ]; + let (tx, rx) = unbounded_channel(); + + send_direct_data_shard_chunks(tx, shard_streams, data_shards, block_size, 32, false, 3, 18).await; + + let mut stream = ChannelChunkStream::new(rx); + let mut collected = Vec::new(); + while let Some(chunk) = stream.next().await { + collected.extend_from_slice(&chunk.unwrap().as_bytes()); + } + + assert_eq!(collected, (3u8..21).collect::>()); + } + + #[tokio::test] + async fn send_direct_data_shard_chunks_keeps_final_full_block_when_length_is_block_aligned() { + let data_shards = 2; + let block_size = 16; + let shard_streams: Vec = vec![ + Box::pin(stream::iter(vec![ + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[0, 1, 2, 3, 4, 5, 6, 7]))), + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[16, 17, 18, 19, 20, 21, 22, 23]))), + ])), + Box::pin(stream::iter(vec![ + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[8, 9, 10, 11, 12, 13, 14, 15]))), + Ok(IoChunk::Shared(Bytes::copy_from_slice(&[24, 25, 26, 27, 28, 29, 30, 31]))), + ])), + ]; + let (tx, rx) = unbounded_channel(); + + send_direct_data_shard_chunks(tx, shard_streams, data_shards, block_size, 32, false, 0, 32).await; + + let mut stream = ChannelChunkStream::new(rx); + let mut collected = Vec::new(); + while let Some(chunk) = stream.next().await { + collected.extend_from_slice(&chunk.unwrap().as_bytes()); + } + + assert_eq!(collected, (0u8..32).collect::>()); + } +} diff --git a/crates/ecstore/src/set_disk/write.rs b/crates/ecstore/src/set_disk/write.rs index d937ede3e..7c0122ee4 100644 --- a/crates/ecstore/src/set_disk/write.rs +++ b/crates/ecstore/src/set_disk/write.rs @@ -13,12 +13,87 @@ // limitations under the License. use super::*; +use crate::store_api::ChunkNativePutData; impl SetDisks { + fn all_inline_bitrot_writers(writers: &[Option]) -> bool { + writers.iter().all(|writer| { + writer + .as_ref() + .is_some_and(crate::erasure_coding::BitrotWriterWrapper::is_inline_buffer) + }) + } + + async fn write_chunk_native_put_data_inline( + data: &mut ChunkNativePutData, + erasure: Arc, + writers: &mut [Option], + write_quorum: usize, + ) -> std::io::Result { + let stream = data.take_stream()?; + let mut assembler = erasure_coding::encode::BlockAssembler::new(stream, erasure.block_size); + let encoder = erasure_coding::encode::ErasureChunkEncoder::new(erasure).await; + let mut writer_group = erasure_coding::encode::MultiWriter::new(writers, write_quorum); + + loop { + let block = match assembler.next_block().await { + Ok(Some(block)) => block, + Ok(None) => break, + Err(err) => { + data.restore_stream(assembler.into_inner()); + return Err(err); + } + }; + + let encoded = match encoder.encode_block(&block).await { + Ok(encoded) => encoded, + Err(err) => { + data.restore_stream(assembler.into_inner()); + return Err(err); + } + }; + + if let Err(err) = writer_group.write_inline(&encoded) { + encoder.release(encoded).await; + data.restore_stream(assembler.into_inner()); + return Err(err); + } + + encoder.release(encoded).await; + } + + let total_bytes = assembler.total_bytes(); + let stream = assembler.into_inner(); + if let Err(err) = writer_group.shutdown_inline() { + data.restore_stream(stream); + return Err(err); + } + data.restore_stream(stream); + Ok(total_bytes) + } + pub(super) fn default_read_quorum(&self) -> usize { self.set_drive_count - self.default_parity_count } + pub(super) async fn write_chunk_native_put_data( + data: &mut ChunkNativePutData, + erasure: Arc, + writers: &mut [Option], + write_quorum: usize, + ) -> std::io::Result { + if Self::all_inline_bitrot_writers(writers) { + return Self::write_chunk_native_put_data_inline(data, erasure, writers, write_quorum).await; + } + + let stream = data.take_stream()?; + let (stream, written) = erasure_coding::encode::ErasureWritePipeline::new(erasure, write_quorum) + .run(stream, writers) + .await?; + data.restore_stream(stream); + Ok(written) + } + pub(super) fn default_write_quorum(&self) -> usize { let mut data_count = self.set_drive_count - self.default_parity_count; if data_count == self.default_parity_count { @@ -627,3 +702,60 @@ impl SetDisks { None } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::erasure_coding::{BitrotWriterWrapper, CustomWriter}; + use crate::store_api::ChunkNativePutData; + + #[tokio::test] + async fn write_chunk_native_put_data_restores_reader_state_after_encoding() { + let payload = b"chunk-native-put-payload".repeat(8); + let erasure = Arc::new(erasure_coding::Erasure::new(2, 1, 8)); + let mut reader = ChunkNativePutData::from_vec(payload.clone()); + let mut writers: Vec> = (0..erasure.total_shard_count()) + .map(|_| { + Some(BitrotWriterWrapper::new( + CustomWriter::new_inline_buffer(), + erasure.shard_size(), + HashAlgorithm::HighwayHash256S, + )) + }) + .collect(); + + let written = SetDisks::write_chunk_native_put_data(&mut reader, erasure.clone(), &mut writers, 2) + .await + .unwrap(); + + assert_eq!(written, payload.len()); + assert_eq!(reader.size(), payload.len() as i64); + assert_eq!(reader.actual_size(), payload.len() as i64); + assert!( + reader.resolve_etag().is_some(), + "restored reader should preserve computed etag state after chunk-native encode" + ); + + let inline_lengths: Vec = writers + .into_iter() + .map(|writer| writer.expect("writer").into_inline_data().expect("inline data").len()) + .collect(); + assert!(inline_lengths.iter().all(|len| *len > 0)); + } + + #[tokio::test] + async fn write_chunk_native_put_data_detects_inline_writer_set() { + let erasure = Arc::new(erasure_coding::Erasure::new(2, 1, 8)); + let writers: Vec> = (0..erasure.total_shard_count()) + .map(|_| { + Some(BitrotWriterWrapper::new( + CustomWriter::new_inline_buffer(), + erasure.shard_size(), + HashAlgorithm::HighwayHash256S, + )) + }) + .collect(); + + assert!(SetDisks::all_inline_bitrot_writers(&writers)); + } +} diff --git a/crates/ecstore/src/sets.rs b/crates/ecstore/src/sets.rs index 48502a4b6..3d32847a8 100644 --- a/crates/ecstore/src/sets.rs +++ b/crates/ecstore/src/sets.rs @@ -15,7 +15,7 @@ use crate::disk::error_reduce::count_errs; use crate::error::{Error, Result}; -use crate::store_api::{ListPartsInfo, ObjectInfoOrErr, WalkOptions}; +use crate::store_api::{GetObjectChunkResult, ListPartsInfo, ObjectInfoOrErr, WalkOptions}; use crate::{ disk::{ DiskAPI, DiskInfo, DiskOption, DiskStore, @@ -28,10 +28,10 @@ use crate::{ global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure}, set_disk::SetDisks, store_api::{ - BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, - HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, - MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations, - ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, + BucketInfo, BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, DeleteBucketOptions, DeletedObject, + GetObjectReader, HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, + ListOperations, MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, + ObjectOperations, ObjectOptions, ObjectToDelete, PartInfo, StorageAPI, }, store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, }; @@ -287,6 +287,19 @@ impl Sets { self.get_disks(self.get_hashed_set_index(key)) } + pub async fn get_object_chunks( + &self, + bucket: &str, + object: &str, + range: Option, + h: HeaderMap, + opts: &ObjectOptions, + ) -> Result { + self.get_disks_by_key(object) + .get_object_chunks(bucket, object, range, h, opts) + .await + } + fn get_hashed_set_index(&self, input: &str) -> usize { match self.distribution_algo { DistributionAlgoVersion::V1 => crc_hash(input, self.disk_set.len()), @@ -375,7 +388,13 @@ impl ObjectIO for Sets { .await } #[tracing::instrument(level = "debug", skip(self, data))] - async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result { + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut ChunkNativePutData, + opts: &ObjectOptions, + ) -> Result { self.get_disks_by_key(object).put_object(bucket, object, data, opts).await } } @@ -688,7 +707,7 @@ impl MultipartOperations for Sets { object: &str, upload_id: &str, part_id: usize, - data: &mut PutObjReader, + data: &mut ChunkNativePutData, opts: &ObjectOptions, ) -> Result { self.get_disks_by_key(object) diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index 07e16baa4..a90e19aef 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -59,9 +59,10 @@ use crate::{ rpc::S3PeerSys, sets::Sets, store_api::{ - BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, - HTTPRangeSpec, HealOperations, ListObjectsV2Info, ListOperations, MakeBucketOptions, MultipartOperations, - MultipartUploadResult, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, + BucketInfo, BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, DeleteBucketOptions, DeletedObject, + GetObjectChunkResult, GetObjectReader, HTTPRangeSpec, HealOperations, ListObjectsV2Info, ListOperations, + MakeBucketOptions, MultipartOperations, MultipartUploadResult, ObjectInfo, ObjectOperations, ObjectOptions, + ObjectToDelete, PartInfo, StorageAPI, }, store_init, }; @@ -259,11 +260,31 @@ impl ObjectIO for ECStore { self.handle_get_object_reader(bucket, object, range, h, opts).await } #[instrument(level = "debug", skip(self, data))] - async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result { + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut ChunkNativePutData, + opts: &ObjectOptions, + ) -> Result { enqueue_transition_after_write(self.handle_put_object(bucket, object, data, opts).await, LcEventSrc::S3PutObject).await } } +impl ECStore { + #[instrument(level = "debug", skip(self))] + pub async fn get_object_chunks( + &self, + bucket: &str, + object: &str, + range: Option, + h: HeaderMap, + opts: &ObjectOptions, + ) -> Result { + self.handle_get_object_chunks(bucket, object, range, h, opts).await + } +} + lazy_static! { static ref enableObjcetLockConfig: ObjectLockConfiguration = ObjectLockConfiguration { object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), @@ -495,7 +516,7 @@ impl MultipartOperations for ECStore { object: &str, upload_id: &str, part_id: usize, - data: &mut PutObjReader, + data: &mut ChunkNativePutData, opts: &ObjectOptions, ) -> Result { self.handle_put_object_part(bucket, object, upload_id, part_id, data, opts) diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index dfeb1c143..12ef5712f 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -176,7 +176,7 @@ impl ECStore { object: &str, upload_id: &str, part_id: usize, - data: &mut PutObjReader, + data: &mut ChunkNativePutData, opts: &ObjectOptions, ) -> Result { check_put_object_part_args(bucket, object, upload_id)?; diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index e648e0e02..e7f044b50 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::*; +use crate::store_api::GetObjectChunkResult; fn select_data_movement_target_pool( existing_pool_idx: Result, @@ -213,12 +214,40 @@ impl ECStore { .await } + #[instrument(level = "debug", skip(self))] + pub(super) async fn handle_get_object_chunks( + &self, + bucket: &str, + object: &str, + range: Option, + h: HeaderMap, + opts: &ObjectOptions, + ) -> Result { + check_get_obj_args(bucket, object)?; + + let object = encode_dir_object(object); + + if self.single_pool() { + return self.pools[0].get_object_chunks(bucket, object.as_str(), range, h, opts).await; + } + + let mut opts = opts.clone(); + opts.no_lock = true; + + let (_, idx) = self + .get_latest_accessible_object_info_with_idx(bucket, &object, &opts) + .await?; + self.pools[idx] + .get_object_chunks(bucket, object.as_str(), range, h, &opts) + .await + } + #[instrument(level = "debug", skip(self, data))] pub(super) async fn handle_put_object( &self, bucket: &str, object: &str, - data: &mut PutObjReader, + data: &mut ChunkNativePutData, opts: &ObjectOptions, ) -> Result { check_put_object_args(bucket, object)?; diff --git a/crates/ecstore/src/store_api.rs b/crates/ecstore/src/store_api.rs index cdfde143b..cdaf86a1e 100644 --- a/crates/ecstore/src/store_api.rs +++ b/crates/ecstore/src/store_api.rs @@ -31,6 +31,7 @@ use rustfs_filemeta::{ RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map, version_purge_statuses_map, }; +use rustfs_io_core::BoxChunkStream; use rustfs_lock::NamespaceLockWrapper; use rustfs_madmin::heal_commands::HealResultItem; use rustfs_rio::Checksum; diff --git a/crates/ecstore/src/store_api/readers.rs b/crates/ecstore/src/store_api/readers.rs index 461e8ff7e..867d5d9ae 100644 --- a/crates/ecstore/src/store_api/readers.rs +++ b/crates/ecstore/src/store_api/readers.rs @@ -1,7 +1,101 @@ use super::*; +use rustfs_rio::TryGetIndex; + +pub struct ChunkNativePutData { + stream: Option, + size: i64, + actual_size: i64, +} + +impl Debug for ChunkNativePutData { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ChunkNativePutData").finish() + } +} + +impl ChunkNativePutData { + pub fn new(stream: HashReader) -> Self { + let size = stream.size(); + let actual_size = stream.actual_size(); + Self { + stream: Some(stream), + size, + actual_size, + } + } + + pub fn from_vec(data: Vec) -> Self { + use sha2::{Digest, Sha256}; + + let content_length = data.len() as i64; + let sha256hex = if content_length > 0 { + Some(hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower)) + } else { + None + }; + Self::new(HashReader::from_stream(Cursor::new(data), content_length, content_length, None, sha256hex, false).unwrap()) + } + + pub fn take_stream(&mut self) -> std::io::Result { + self.stream + .take() + .ok_or_else(|| std::io::Error::other("ChunkNativePutData stream already taken")) + } + + pub fn restore_stream(&mut self, stream: HashReader) { + self.size = stream.size(); + self.actual_size = stream.actual_size(); + self.stream = Some(stream); + } + + pub fn as_hash_reader(&self) -> Option<&HashReader> { + self.stream.as_ref() + } + + pub fn as_hash_reader_mut(&mut self) -> Option<&mut HashReader> { + self.stream.as_mut() + } + + pub fn index_bytes(&self) -> Option { + self.as_hash_reader() + .and_then(|reader| reader.try_get_index().map(|index| index.clone().into_vec())) + } + + pub fn resolve_etag(&mut self) -> Option { + self.as_hash_reader_mut() + .and_then(rustfs_rio::EtagResolvable::try_resolve_etag) + } + + pub fn content_hash_bytes(&mut self) -> std::io::Result> { + let Some(reader) = self.as_hash_reader_mut() else { + return Ok(None); + }; + + Ok(reader + .finalize_content_hash()? + .as_ref() + .map(|checksum| checksum.to_bytes(&[]))) + } + + pub fn content_crc_type(&self) -> Option { + self.as_hash_reader().and_then(HashReader::content_crc_type) + } + + pub fn content_crc(&self) -> HashMap { + self.as_hash_reader().map_or_else(HashMap::new, HashReader::content_crc) + } + + pub fn size(&self) -> i64 { + self.size + } + + pub fn actual_size(&self) -> i64 { + self.actual_size + } +} pub struct PutObjReader { - pub stream: HashReader, + data: ChunkNativePutData, } impl Debug for PutObjReader { @@ -12,32 +106,81 @@ impl Debug for PutObjReader { impl PutObjReader { pub fn new(stream: HashReader) -> Self { - PutObjReader { stream } + Self { + data: ChunkNativePutData::new(stream), + } } - pub fn as_hash_reader(&self) -> &HashReader { - &self.stream + pub fn chunk_native_data(&self) -> &ChunkNativePutData { + &self.data + } + + pub fn chunk_native_data_mut(&mut self) -> &mut ChunkNativePutData { + &mut self.data + } + + pub fn take_stream(&mut self) -> std::io::Result { + self.data.take_stream() + } + + pub fn restore_stream(&mut self, stream: HashReader) { + self.data.restore_stream(stream); + } + + pub fn as_hash_reader(&self) -> Option<&HashReader> { + self.data.as_hash_reader() + } + + pub fn as_hash_reader_mut(&mut self) -> Option<&mut HashReader> { + self.data.as_hash_reader_mut() + } + + pub fn index_bytes(&self) -> Option { + self.data.index_bytes() + } + + pub fn resolve_etag(&mut self) -> Option { + self.data.resolve_etag() + } + + pub fn content_hash_bytes(&mut self) -> std::io::Result> { + self.data.content_hash_bytes() + } + + pub fn content_crc_type(&self) -> Option { + self.data.content_crc_type() + } + + pub fn content_crc(&self) -> HashMap { + self.data.content_crc() } pub fn from_vec(data: Vec) -> Self { - use sha2::{Digest, Sha256}; - let content_length = data.len() as i64; - let sha256hex = if content_length > 0 { - Some(hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower)) - } else { - None - }; - PutObjReader { - stream: HashReader::from_stream(Cursor::new(data), content_length, content_length, None, sha256hex, false).unwrap(), + Self { + data: ChunkNativePutData::from_vec(data), } } pub fn size(&self) -> i64 { - self.stream.size() + self.data.size() } pub fn actual_size(&self) -> i64 { - self.stream.actual_size() + self.data.actual_size() + } +} + +impl std::ops::Deref for PutObjReader { + type Target = ChunkNativePutData; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + +impl std::ops::DerefMut for PutObjReader { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.data } } @@ -46,6 +189,26 @@ pub struct GetObjectReader { pub object_info: ObjectInfo, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GetObjectChunkPath { + Direct, + Bridge, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GetObjectChunkCopyMode { + TrueZeroCopy, + SharedBytes, + SingleCopy, + Reconstructed, +} + +pub struct GetObjectChunkResult { + pub stream: BoxChunkStream, + pub path: GetObjectChunkPath, + pub copy_mode: GetObjectChunkCopyMode, +} + impl GetObjectReader { #[tracing::instrument(level = "debug", skip(reader, rs, opts, _h))] pub fn new( @@ -63,31 +226,34 @@ impl GetObjectReader { rs = HTTPRangeSpec::from_object_info(oi, part_number); } - // TODO:Encrypted + let logical_size = oi.get_actual_size()?; + let encrypted_object = oi.user_defined.contains_key("x-rustfs-encryption-key") + || oi + .user_defined + .contains_key("x-amz-server-side-encryption-customer-algorithm"); let (algo, is_compressed) = oi.is_compressed_ok()?; // TODO: check TRANSITION if is_compressed { - let actual_size = oi.get_actual_size()?; let (off, length, dec_off, dec_length) = if let Some(rs) = rs { // Support range requests for compressed objects - let (dec_off, dec_length) = rs.get_offset_length(actual_size)?; + let (dec_off, dec_length) = rs.get_offset_length(logical_size)?; (0, oi.size, dec_off, dec_length) } else { - (0, oi.size, 0, actual_size) + (0, oi.size, 0, logical_size) }; let dec_reader = DecompressReader::new(reader, algo); - let actual_size_usize = if actual_size > 0 { - actual_size as usize + let actual_size_usize = if logical_size > 0 { + logical_size as usize } else { - return Err(Error::other(format!("invalid decompressed size {actual_size}"))); + return Err(Error::other(format!("invalid decompressed size {logical_size}"))); }; - let final_reader: Box = if dec_off > 0 || dec_length != actual_size { + let final_reader: Box = if dec_off > 0 || dec_length != logical_size { // Use RangedDecompressReader for streaming range processing // The new implementation supports any offset size by streaming and skipping data match RangedDecompressReader::new(dec_reader, dec_off, dec_length, actual_size_usize) { @@ -122,8 +288,19 @@ impl GetObjectReader { )); } + if encrypted_object && rs.is_none() { + return Ok(( + GetObjectReader { + stream: reader, + object_info: oi.clone(), + }, + 0, + oi.size, + )); + } + if let Some(rs) = rs { - let (off, length) = rs.get_offset_length(oi.size)?; + let (off, length) = rs.get_offset_length(logical_size)?; Ok(( GetObjectReader { @@ -140,7 +317,7 @@ impl GetObjectReader { object_info: oi.clone(), }, 0, - oi.size, + logical_size, )) } } diff --git a/crates/ecstore/src/store_api/traits.rs b/crates/ecstore/src/store_api/traits.rs index facabeae3..a351976e8 100644 --- a/crates/ecstore/src/store_api/traits.rs +++ b/crates/ecstore/src/store_api/traits.rs @@ -11,7 +11,13 @@ pub trait ObjectIO: Send + Sync + Debug + 'static { opts: &ObjectOptions, ) -> Result; - async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result; + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut ChunkNativePutData, + opts: &ObjectOptions, + ) -> Result; } /// Bucket-level storage operations. @@ -126,7 +132,7 @@ pub trait MultipartOperations: Send + Sync + Debug { object: &str, upload_id: &str, part_id: usize, - data: &mut PutObjReader, + data: &mut ChunkNativePutData, opts: &ObjectOptions, ) -> Result; async fn get_multipart_info( diff --git a/crates/ecstore/src/store_api/types.rs b/crates/ecstore/src/store_api/types.rs index 829f81ce5..efbcf396d 100644 --- a/crates/ecstore/src/store_api/types.rs +++ b/crates/ecstore/src/store_api/types.rs @@ -70,6 +70,7 @@ pub struct ObjectOptions { pub eval_metadata: Option>, + pub resolved_checksum: Option, pub want_checksum: Option, pub skip_verify_bitrot: bool, } @@ -283,7 +284,7 @@ pub struct ObjectInfo { pub expires: Option, pub num_versions: usize, pub successor_mod_time: Option, - pub put_object_reader: Option, + pub put_object_reader: Option, pub etag: Option, pub inlined: bool, pub metadata_only: bool, @@ -509,6 +510,18 @@ impl ObjectInfo { }) .collect(); + let actual_size = fi + .parts + .iter() + .map(|part| { + if part.actual_size > 0 { + part.actual_size + } else { + i64::try_from(part.size).unwrap_or_default() + } + }) + .sum(); + // TODO: part checksums ObjectInfo { @@ -521,6 +534,7 @@ impl ObjectInfo { delete_marker: fi.deleted, mod_time: fi.mod_time, size: fi.size, + actual_size, parts, is_latest: fi.is_latest, user_tags, diff --git a/crates/ecstore/src/tier/tier.rs b/crates/ecstore/src/tier/tier.rs index 8cf6e6062..97b546f38 100644 --- a/crates/ecstore/src/tier/tier.rs +++ b/crates/ecstore/src/tier/tier.rs @@ -51,7 +51,7 @@ use crate::{ disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}, global::is_first_cluster_node_local, store::ECStore, - store_api::{ObjectIO as _, ObjectOptions, PutObjReader}, + store_api::{ChunkNativePutData, ObjectIO as _, ObjectOptions}, }; use rustfs_rio::HashReader; use rustfs_utils::path::{SLASH_SEPARATOR, path_join}; @@ -1046,9 +1046,8 @@ impl TierConfigMgr { opts: &ObjectOptions, ) -> std::result::Result<(), std::io::Error> { debug!("save tier config:{}", file); - let _ = api - .put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts) - .await?; + let mut put_data = ChunkNativePutData::from_vec(data.to_vec()); + let _ = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await?; Ok(()) } @@ -1104,11 +1103,12 @@ async fn load_tier_config(api: Arc) -> std::result::Result { let cfg = TierConfigMgr::unmarshal(&data)?; let normalized = encode_external_tiering_config_blob(&cfg)?; + let mut put_data = ChunkNativePutData::from_vec(normalized.to_vec()); let _ = api .put_object( RUSTFS_META_BUCKET, &config_file, - &mut PutObjReader::from_vec(normalized.to_vec()), + &mut put_data, &ObjectOptions { max_parity: true, ..Default::default() @@ -1158,10 +1158,11 @@ async fn read_tier_config_from_bucket( } async fn write_tier_config_to_rustfs(api: Arc, path: &str, data: Bytes) -> io::Result<()> { + let mut put_data = ChunkNativePutData::from_vec(data.to_vec()); api.put_object( RUSTFS_META_BUCKET, path, - &mut PutObjReader::from_vec(data.to_vec()), + &mut put_data, &ObjectOptions { max_parity: true, ..Default::default() diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 33eee875a..80e5f4653 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -24,7 +24,7 @@ use rustfs_utils::http::headers::{ AMZ_STORAGE_CLASS, }; use rustfs_utils::http::{ - AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS, + AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_CRC, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, has_internal_suffix, insert_bytes, is_internal_key, }; @@ -230,6 +230,10 @@ impl FileMeta { } } + if let Some(checksum) = fi.checksum.as_ref() { + insert_bytes(&mut obj.meta_sys, SUFFIX_CRC, checksum.to_vec()); + } + if let Some(mod_time) = fi.mod_time { obj.mod_time = Some(mod_time); } diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index 33f6210bc..e1699c192 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -208,7 +208,7 @@ impl HealStorageAPI for ECStoreHealStorage { async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()> { debug!("Putting object data: {}/{} ({} bytes)", bucket, object, data.len()); - let mut reader = rustfs_ecstore::store_api::PutObjReader::from_vec(data.to_vec()); + let mut reader = rustfs_ecstore::store_api::ChunkNativePutData::from_vec(data.to_vec()); match (*self.ecstore) .put_object(bucket, object, &mut reader, &Default::default()) .await diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index e7ba323ef..765221408 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -18,7 +18,7 @@ use rustfs_ecstore::{ disk::endpoint::Endpoint, endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, store::ECStore, - store_api::{BucketOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, + store_api::{BucketOperations, ChunkNativePutData, ObjectIO, ObjectOperations, ObjectOptions}, }; use rustfs_heal::heal::{ manager::{HealConfig, HealManager}, @@ -162,7 +162,7 @@ async fn create_test_bucket(ecstore: &Arc, bucket_name: &str) { /// Test helper: Upload test object async fn upload_test_object(ecstore: &Arc, bucket: &str, object: &str, data: &[u8]) { - let mut reader = PutObjReader::from_vec(data.to_vec()); + let mut reader = ChunkNativePutData::from_vec(data.to_vec()); let object_info = (**ecstore) .put_object(bucket, object, &mut reader, &ObjectOptions::default()) .await diff --git a/crates/io-core/Cargo.toml b/crates/io-core/Cargo.toml index 81f932532..fd4fb14b5 100644 --- a/crates/io-core/Cargo.toml +++ b/crates/io-core/Cargo.toml @@ -29,12 +29,14 @@ workspace = true [dependencies] bytes = { workspace = true } +futures-core = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["io-util", "fs", "rt", "sync"] } memmap2 = { workspace = true } rustfs-io-metrics = { workspace = true } [dev-dependencies] +futures-util = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } [lib] diff --git a/crates/io-core/src/adapter.rs b/crates/io-core/src/adapter.rs new file mode 100644 index 000000000..b2548382c --- /dev/null +++ b/crates/io-core/src/adapter.rs @@ -0,0 +1,124 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Compatibility adapter that exposes chunk streams as `AsyncRead`. + +use crate::chunk::BoxChunkStream; +use bytes::Bytes; +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::io::{AsyncRead, ReadBuf}; + +/// `AsyncRead` adapter for boxed chunk streams. +pub struct ChunkStreamReader { + stream: BoxChunkStream, + current: Option, + offset: usize, +} + +impl ChunkStreamReader { + #[must_use] + pub fn new(stream: BoxChunkStream) -> Self { + Self { + stream, + current: None, + offset: 0, + } + } +} + +impl AsyncRead for ChunkStreamReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + loop { + if let Some(current) = &self.current { + if self.offset < current.len() { + let remaining = ¤t[self.offset..]; + let to_read = remaining.len().min(buf.remaining()); + buf.put_slice(&remaining[..to_read]); + self.offset += to_read; + return Poll::Ready(Ok(())); + } + + self.current = None; + self.offset = 0; + continue; + } + + match self.stream.as_mut().poll_next(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Some(Ok(chunk))) => { + let next = chunk.as_bytes(); + if next.is_empty() { + continue; + } + self.current = Some(next); + self.offset = 0; + } + Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)), + Poll::Ready(None) => return Poll::Ready(Ok(())), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chunk::{IoChunk, MappedChunk, PooledChunk}; + use bytes::Bytes; + use futures_util::stream; + use tokio::io::AsyncReadExt; + + #[tokio::test] + async fn test_chunk_stream_reader_reads_single_chunk() { + let stream: BoxChunkStream = Box::pin(stream::iter(vec![Ok(IoChunk::Shared(Bytes::from_static(b"hello")))])); + let mut reader = ChunkStreamReader::new(stream); + let mut out = Vec::new(); + reader.read_to_end(&mut out).await.unwrap(); + assert_eq!(out, b"hello"); + } + + #[tokio::test] + async fn test_chunk_stream_reader_reads_multiple_chunks() { + let stream: BoxChunkStream = Box::pin(stream::iter(vec![ + Ok(IoChunk::Shared(Bytes::from_static(b"he"))), + Ok(IoChunk::Mapped(MappedChunk::new(Bytes::from_static(b"llo!"), 0, 4).unwrap())), + Ok(IoChunk::Pooled(PooledChunk::from_bytes(Bytes::from_static(b" world")).unwrap())), + ])); + let mut reader = ChunkStreamReader::new(stream); + let mut out = Vec::new(); + reader.read_to_end(&mut out).await.unwrap(); + assert_eq!(out, b"hello! world"); + } + + #[tokio::test] + async fn test_chunk_stream_reader_handles_empty_stream() { + let stream: BoxChunkStream = Box::pin(stream::iter(Vec::>::new())); + let mut reader = ChunkStreamReader::new(stream); + let mut out = Vec::new(); + reader.read_to_end(&mut out).await.unwrap(); + assert!(out.is_empty()); + } + + #[tokio::test] + async fn test_chunk_stream_reader_propagates_stream_error() { + let stream: BoxChunkStream = Box::pin(stream::iter(vec![Err(io::Error::other("chunk stream failure"))])); + let mut reader = ChunkStreamReader::new(stream); + let mut out = Vec::new(); + let err = reader.read_to_end(&mut out).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::Other); + assert!(err.to_string().contains("chunk stream failure")); + } +} diff --git a/crates/io-core/src/chunk.rs b/crates/io-core/src/chunk.rs new file mode 100644 index 000000000..4a3441cc5 --- /dev/null +++ b/crates/io-core/src/chunk.rs @@ -0,0 +1,276 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Core chunk ownership abstractions for the zero-copy data plane. + +use crate::pool::PooledBuffer; +use bytes::Bytes; +use futures_core::Stream; +use std::io; +use std::pin::Pin; + +/// Boxed asynchronous stream of I/O chunks. +pub type BoxChunkStream = Pin> + Send + Sync + 'static>>; + +/// Source of chunked data. +pub trait ChunkSource { + fn into_chunk_stream(self) -> BoxChunkStream + where + Self: Sized; +} + +/// Owned chunk variants used by the zero-copy data plane. +#[derive(Debug)] +pub enum IoChunk { + Shared(Bytes), + Mapped(MappedChunk), + Pooled(PooledChunk), +} + +impl IoChunk { + /// Returns the visible length of this chunk. + #[must_use] + pub fn len(&self) -> usize { + match self { + Self::Shared(bytes) => bytes.len(), + Self::Mapped(chunk) => chunk.len(), + Self::Pooled(chunk) => chunk.len(), + } + } + + /// Returns true when the chunk has no visible data. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns a shared read-only view of the visible bytes. + #[must_use] + pub fn as_bytes(&self) -> Bytes { + match self { + Self::Shared(bytes) => bytes.clone(), + Self::Mapped(chunk) => chunk.as_bytes(), + Self::Pooled(chunk) => chunk.as_bytes(), + } + } + + /// Returns a sliced view relative to the currently visible bytes. + pub fn slice(&self, offset: usize, len: usize) -> io::Result { + match self { + Self::Shared(bytes) => { + validate_slice_bounds(bytes.len(), offset, len)?; + Ok(Self::Shared(bytes.slice(offset..offset + len))) + } + Self::Mapped(chunk) => chunk.slice(offset, len).map(Self::Mapped), + Self::Pooled(chunk) => chunk.slice(offset, len).map(Self::Pooled), + } + } +} + +/// Logical view into mapped file bytes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MappedChunk { + bytes: Bytes, + logical_offset: usize, + logical_len: usize, +} + +impl MappedChunk { + pub fn new(bytes: Bytes, logical_offset: usize, logical_len: usize) -> io::Result { + validate_slice_bounds(bytes.len(), logical_offset, logical_len)?; + Ok(Self { + bytes, + logical_offset, + logical_len, + }) + } + + /// Returns the visible length of this mapped chunk. + #[must_use] + pub const fn len(&self) -> usize { + self.logical_len + } + + /// Returns true when the chunk has no visible data. + #[must_use] + pub const fn is_empty(&self) -> bool { + self.logical_len == 0 + } + + /// Returns the visible bytes for this logical view. + #[must_use] + pub fn as_bytes(&self) -> Bytes { + self.bytes + .slice(self.logical_offset..self.logical_offset.saturating_add(self.logical_len)) + } + + /// Returns a sliced logical view relative to the current logical view. + pub fn slice(&self, offset: usize, len: usize) -> io::Result { + validate_slice_bounds(self.logical_len, offset, len)?; + Self::new(self.bytes.clone(), self.logical_offset + offset, len) + } +} + +/// Placeholder pooled chunk variant for commit 4. +/// +/// This is backed by a `PooledBuffer` and exposes a visible read-only window. +#[derive(Debug)] +pub struct PooledChunk { + bytes: Bytes, +} + +#[derive(Debug)] +struct PooledChunkOwner { + buffer: PooledBuffer, + visible_len: usize, +} + +impl AsRef<[u8]> for PooledChunkOwner { + fn as_ref(&self) -> &[u8] { + &self.buffer[..self.visible_len] + } +} + +#[derive(Debug)] +struct DetachedVecChunkOwner { + bytes: Vec, +} + +impl AsRef<[u8]> for DetachedVecChunkOwner { + fn as_ref(&self) -> &[u8] { + &self.bytes + } +} + +impl PooledChunk { + pub fn new(buffer: PooledBuffer, len: usize) -> io::Result { + validate_slice_bounds(buffer.len(), 0, len)?; + Ok(Self { + bytes: Bytes::from_owner(PooledChunkOwner { + buffer, + visible_len: len, + }), + }) + } + + /// Convenience constructor for detached test and compatibility values. + pub fn from_bytes(bytes: Bytes) -> io::Result { + let len = bytes.len(); + Self::new(PooledBuffer::from_bytes(bytes), len) + } + + /// Detached constructor that takes ownership of an existing `Vec` + /// without introducing an additional copy. + pub fn from_vec(bytes: Vec) -> Self { + Self { + bytes: Bytes::from_owner(DetachedVecChunkOwner { bytes }), + } + } + + /// Returns the visible length of this pooled chunk. + #[must_use] + pub fn len(&self) -> usize { + self.bytes.len() + } + + /// Returns true when the chunk has no visible data. + #[must_use] + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + /// Returns the visible bytes for this pooled chunk. + #[must_use] + pub fn as_bytes(&self) -> Bytes { + self.bytes.clone() + } + + /// Returns a sliced pooled chunk relative to the current visible view. + pub fn slice(&self, offset: usize, len: usize) -> io::Result { + validate_slice_bounds(self.bytes.len(), offset, len)?; + Ok(Self { + bytes: self.bytes.slice(offset..offset + len), + }) + } +} + +fn validate_slice_bounds(visible_len: usize, offset: usize, len: usize) -> io::Result<()> { + let end = offset + .checked_add(len) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "chunk slice overflows"))?; + if end > visible_len { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "chunk slice exceeds visible length")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pool::BytesPool; + + #[test] + fn test_shared_chunk_len_and_slice() { + let chunk = IoChunk::Shared(Bytes::from_static(b"abcdef")); + assert_eq!(chunk.len(), 6); + assert!(!chunk.is_empty()); + assert_eq!(chunk.as_bytes(), Bytes::from_static(b"abcdef")); + assert_eq!(chunk.slice(1, 3).unwrap().as_bytes(), Bytes::from_static(b"bcd")); + } + + #[test] + fn test_mapped_chunk_len_and_slice() { + let chunk = MappedChunk::new(Bytes::from_static(b"abcdefgh"), 2, 4).unwrap(); + assert_eq!(chunk.len(), 4); + assert_eq!(chunk.as_bytes(), Bytes::from_static(b"cdef")); + assert_eq!(chunk.slice(1, 2).unwrap().as_bytes(), Bytes::from_static(b"de")); + } + + #[test] + fn test_pooled_chunk_len_and_as_bytes() { + let chunk = PooledChunk::from_bytes(Bytes::from_static(b"hello")).unwrap(); + assert_eq!(chunk.len(), 5); + assert_eq!(chunk.as_bytes(), Bytes::from_static(b"hello")); + assert_eq!(chunk.slice(1, 3).unwrap().as_bytes(), Bytes::from_static(b"ell")); + } + + #[test] + fn test_io_chunk_as_bytes_for_all_variants() { + let shared = IoChunk::Shared(Bytes::from_static(b"s")); + let mapped = IoChunk::Mapped(MappedChunk::new(Bytes::from_static(b"mapped"), 0, 6).unwrap()); + let pooled = IoChunk::Pooled(PooledChunk::from_bytes(Bytes::from_static(b"p")).unwrap()); + + assert_eq!(shared.as_bytes(), Bytes::from_static(b"s")); + assert_eq!(mapped.as_bytes(), Bytes::from_static(b"mapped")); + assert_eq!(pooled.as_bytes(), Bytes::from_static(b"p")); + } + + #[tokio::test] + async fn test_pooled_chunk_keeps_owner_alive_until_last_view_drops() { + let pool = BytesPool::new_tiered(); + let mut buffer = pool.acquire_buffer(16).await; + buffer.extend_from_slice(b"pooled-bytes"); + + let chunk = PooledChunk::new(buffer, "pooled-bytes".len()).unwrap(); + let bytes = chunk.as_bytes(); + + assert_eq!(pool.available_buffers(), 0); + drop(chunk); + assert_eq!(pool.available_buffers(), 0); + assert_eq!(bytes, Bytes::from_static(b"pooled-bytes")); + + drop(bytes); + assert_eq!(pool.available_buffers(), 1); + } +} diff --git a/crates/io-core/src/lib.rs b/crates/io-core/src/lib.rs index fe0ee031f..f8565d50e 100644 --- a/crates/io-core/src/lib.rs +++ b/crates/io-core/src/lib.rs @@ -46,8 +46,10 @@ //! let mut buffer = pool.acquire_buffer(8192).await; //! ``` +pub mod adapter; pub mod backpressure; pub mod bufreader_optimizer; +pub mod chunk; pub mod config; pub mod deadlock_detector; pub mod direct_io; @@ -68,7 +70,9 @@ pub use reader::{ZeroCopyObjectReader, ZeroCopyReadError}; pub use writer::{ZeroCopyObjectWriter, ZeroCopyWriteError}; // BufReader optimizer exports +pub use adapter::ChunkStreamReader; pub use bufreader_optimizer::{BufReaderConfig, BufReaderOptimizer, BufReaderStats, BufferedSource}; +pub use chunk::{BoxChunkStream, ChunkSource, IoChunk, MappedChunk, PooledChunk}; // Shared memory exports pub use shared_memory::{ArcData, ArcMetadata, SharedMemoryConfig, SharedMemoryPool, SharedMemoryStats}; diff --git a/crates/io-core/src/pool.rs b/crates/io-core/src/pool.rs index aa08fcfde..fc5e642f3 100644 --- a/crates/io-core/src/pool.rs +++ b/crates/io-core/src/pool.rs @@ -17,7 +17,7 @@ //! Migrated from rustfs-ecstore to provide unified buffer pooling //! across rustfs and rustfs-ecstore without cyclic dependencies. -use bytes::BytesMut; +use bytes::{Bytes, BytesMut}; use std::mem::ManuallyDrop; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -108,6 +108,7 @@ pub struct BytesPoolMetrics { /// A buffer managed by the BytesPool. /// /// When dropped, the buffer is automatically returned to the pool for reuse. +#[derive(Debug)] pub struct PooledBuffer { /// The underlying buffer (ManuallyDrop to allow taking on drop) pub buffer: ManuallyDrop, @@ -117,6 +118,45 @@ pub struct PooledBuffer { _permit: Option, } +impl PooledBuffer { + /// Create a detached pooled buffer from bytes. + /// + /// This is primarily used for tests and transitional adapters where the + /// chunk abstraction needs a pool-shaped owner before a real pool-backed + /// producer exists. + #[must_use] + pub fn from_bytes(bytes: Bytes) -> Self { + Self { + buffer: ManuallyDrop::new(BytesMut::from(bytes.as_ref())), + tier: None, + _permit: None, + } + } + + /// Current visible length of the underlying buffer. + #[must_use] + pub fn len(&self) -> usize { + self.buffer.len() + } + + /// Total buffer capacity. + #[must_use] + pub fn capacity(&self) -> usize { + self.buffer.capacity() + } + + /// Clear the visible contents while preserving capacity. + pub fn clear(&mut self) { + self.buffer.clear(); + } + + /// Returns true when the visible buffer is empty. + #[must_use] + pub fn is_empty(&self) -> bool { + self.buffer.is_empty() + } +} + /// BytesPool configuration. /// /// Allows customization of buffer sizes and limits for each tier. diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index 3a0f1ce46..345a9d98a 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -118,8 +118,129 @@ pub use config::{ // Re-exports for convenience pub use collector::MetricsCollector; +pub use metric_names::data_plane; pub use performance::PerformanceMetrics; +/// High-level request path selected for an I/O operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IoPath { + Fast, + Legacy, +} + +impl IoPath { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Fast => "fast", + Self::Legacy => "legacy", + } + } +} + +/// Effective copy mode observed for an I/O operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CopyMode { + TrueZeroCopy, + SharedBytes, + SingleCopy, + Reconstructed, + Transformed, +} + +impl CopyMode { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::TrueZeroCopy => "true_zero_copy", + Self::SharedBytes => "shared_bytes", + Self::SingleCopy => "single_copy", + Self::Reconstructed => "reconstructed", + Self::Transformed => "transformed", + } + } +} + +/// Stage where a data plane decision or fallback happened. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IoStage { + Unknown, + ReadSetup, + HttpBridge, + CacheWriteback, + LocalDiskChunk, + RangeGuard, + PutTransform, +} + +impl IoStage { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Unknown => "unknown", + Self::ReadSetup => "read_setup", + Self::HttpBridge => "http_bridge", + Self::CacheWriteback => "cache_writeback", + Self::LocalDiskChunk => "local_disk_chunk", + Self::RangeGuard => "range_guard", + Self::PutTransform => "put_transform", + } + } +} + +/// Reason why the data plane fell back from a preferred path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FallbackReason { + Unknown, + MmapDisabled, + MmapUnavailable, + SmallObject, + WindowLimitExceeded, + UnalignedWindow, + RangeNotSupported, + EncryptionEnabled, + CompressionEnabled, + TransformEncryptionLegacy, + TransformCompressionLegacy, + TransformCompressionEncryptionLegacy, + ChunkBridgeUnavailable, + NonLocalBackend, +} + +impl FallbackReason { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Unknown => "unknown", + Self::MmapDisabled => "mmap_disabled", + Self::MmapUnavailable => "mmap_unavailable", + Self::SmallObject => "small_object", + Self::WindowLimitExceeded => "window_limit_exceeded", + Self::UnalignedWindow => "unaligned_window", + Self::RangeNotSupported => "range_not_supported", + Self::EncryptionEnabled => "encryption_enabled", + Self::CompressionEnabled => "compression_enabled", + Self::TransformEncryptionLegacy => "transform_encryption_legacy", + Self::TransformCompressionLegacy => "transform_compression_legacy", + Self::TransformCompressionEncryptionLegacy => "transform_compression_encryption_legacy", + Self::ChunkBridgeUnavailable => "chunk_bridge_unavailable", + Self::NonLocalBackend => "non_local_backend", + } + } +} + +#[inline(always)] +fn put_size_bucket_label(size_bytes: i64) -> &'static str { + match size_bytes { + ..=0 => "unknown", + 1..=16_384 => "le_16kib", + 16_385..=65_536 => "le_64kib", + 65_537..=262_144 => "le_256kib", + 262_145..=1_048_576 => "le_1mib", + _ => "gt_1mib", + } +} + /// Record GetObject request start. #[inline(always)] pub fn record_get_object_request_start(concurrent_requests: usize) { @@ -206,40 +327,138 @@ pub fn record_object_cache_writeback() { counter!("rustfs_io_object_cache_writeback_total").increment(1); } -/// Record a zero-copy read operation. -/// -/// # Arguments -/// -/// * `size_bytes` - Size of the data read in bytes -/// * `duration_ms` - Time taken for the read operation in milliseconds +/// Record which request path was selected for an operation. #[inline(always)] -pub fn record_zero_copy_read(size_bytes: usize, duration_ms: f64) { - counter!("rustfs.zero_copy.reads.total").increment(1); - histogram!("rustfs.zero_copy.read.size.bytes").record(size_bytes as f64); - histogram!("rustfs.zero_copy.read.duration.ms").record(duration_ms); +pub fn record_io_path_selected(operation: &'static str, io_path: IoPath) { + counter!( + metric_names::data_plane::PATH_SELECTED_TOTAL, + "path" => operation, + "mode" => io_path.as_str() + ) + .increment(1); } -/// Record memory copies avoided by using zero-copy. -/// -/// # Arguments -/// -/// * `bytes_saved` - Number of bytes that would have been copied without zero-copy +/// Record the effective copy mode for an operation. #[inline(always)] -pub fn record_memory_copy_saved(bytes_saved: usize) { - counter!("rustfs.zero_copy.memory.saved.bytes").increment(bytes_saved as u64); +pub fn record_io_copy_mode(operation: &'static str, copy_mode: CopyMode, size_bytes: usize) { + counter!( + metric_names::data_plane::COPY_MODE_BYTES_TOTAL, + "path" => operation, + "mode" => copy_mode.as_str() + ) + .increment(size_bytes as u64); } -/// Record a fallback from zero-copy to regular read. -/// -/// This happens when zero-copy read fails (e.g., mmap not available, -/// file too large, etc.) and the system falls back to regular I/O. -/// -/// # Arguments -/// -/// * `reason` - Reason for the fallback (e.g., "mmap_unavailable", "file_too_large") +/// Record a data plane fallback decision. #[inline(always)] -pub fn record_zero_copy_fallback(reason: &str) { - counter!("rustfs.zero_copy.fallback.total", "reason" => reason.to_string()).increment(1); +pub fn record_io_fallback(stage: IoStage, reason: FallbackReason) { + counter!( + metric_names::data_plane::FALLBACK_TOTAL, + "stage" => stage.as_str(), + "reason" => reason.as_str() + ) + .increment(1); +} + +/// Record the currently active mmap bytes held by LocalDisk chunk streams. +#[inline(always)] +pub fn record_local_disk_active_mmap_bytes(active_bytes: usize) { + gauge!(metric_names::data_plane::LOCAL_DISK_ACTIVE_MMAP_BYTES).set(active_bytes as f64); +} + +/// Record pooled chunk usage in LocalDisk compatibility paths. +#[inline(always)] +pub fn record_local_disk_pooled_chunk(source: &'static str, size_bytes: usize) { + counter!( + metric_names::data_plane::LOCAL_DISK_POOLED_CHUNKS_TOTAL, + "source" => source + ) + .increment(1); + counter!( + metric_names::data_plane::LOCAL_DISK_POOLED_BYTES_TOTAL, + "source" => source + ) + .increment(size_bytes as u64); +} + +/// Record a compatibility chunk-stream aggregation performed by `read_file_zero_copy()`. +#[inline(always)] +pub fn record_local_disk_compat_collect(chunk_count: usize, total_bytes: usize) { + counter!(metric_names::data_plane::LOCAL_DISK_COMPAT_COLLECT_TOTAL).increment(1); + histogram!(metric_names::data_plane::LOCAL_DISK_COMPAT_COLLECT_CHUNKS).record(chunk_count as f64); + histogram!(metric_names::data_plane::LOCAL_DISK_COMPAT_COLLECT_BYTES).record(total_bytes as f64); +} + +/// Record an attempted PUT fast path. +#[inline(always)] +pub fn record_put_object_attempted_fast_path(size_bytes: i64) { + counter!(metric_names::data_plane::PUT_FAST_PATH_ATTEMPTS_TOTAL).increment(1); + + if size_bytes > 0 { + histogram!(metric_names::data_plane::PUT_FAST_PATH_ATTEMPT_SIZE_BYTES).record(size_bytes as f64); + } +} + +/// Record which transformed PUT pipeline was selected. +#[inline(always)] +pub fn record_put_transform_selected(kind: &'static str, io_path: IoPath, size_bytes: usize) { + counter!( + metric_names::data_plane::PUT_TRANSFORM_SELECTED_TOTAL, + "kind" => kind, + "mode" => io_path.as_str() + ) + .increment(1); + + histogram!( + metric_names::data_plane::PUT_TRANSFORM_SIZE_BYTES, + "kind" => kind, + "mode" => io_path.as_str() + ) + .record(size_bytes as f64); +} + +/// Record PUT path selection with size-bucket context. +#[inline(always)] +pub fn record_put_path_selected(size_bytes: i64, io_path: IoPath) { + counter!( + "rustfs.s3.put_object.path.selected.total", + "mode" => io_path.as_str(), + "size_bucket" => put_size_bucket_label(size_bytes) + ) + .increment(1); +} + +/// Record PUT copy mode with size-bucket context. +#[inline(always)] +pub fn record_put_copy_mode(size_bytes: i64, copy_mode: CopyMode) { + counter!( + "rustfs.s3.put_object.copy_mode.total", + "mode" => copy_mode.as_str(), + "size_bucket" => put_size_bucket_label(size_bytes) + ) + .increment(1); +} + +/// Record PUT fallback with size-bucket context. +#[inline(always)] +pub fn record_put_fallback(size_bytes: i64, reason: FallbackReason) { + counter!( + "rustfs.s3.put_object.fallback.total", + "reason" => reason.as_str(), + "size_bucket" => put_size_bucket_label(size_bytes) + ) + .increment(1); +} + +/// Record inline-object selection for PUT with size-bucket context. +#[inline(always)] +pub fn record_put_inline_selected(size_bytes: i64, versioned: bool) { + counter!( + "rustfs.s3.put_object.inline.selected.total", + "versioned" => if versioned { "true" } else { "false" }, + "size_bucket" => put_size_bucket_label(size_bytes) + ) + .increment(1); } // ============================================================================ @@ -297,41 +516,6 @@ pub fn record_bytes_pool_hit_rate(tier: &str, hit_rate: f64) { gauge!("rustfs.bytes.pool.hit.rate", "tier" => tier.to_string()).set(hit_rate * 100.0); } -/// Record zero-copy write operation. -/// -/// # Arguments -/// -/// * `size_bytes` - Size of the data written in bytes -/// * `duration_ms` - Time taken for the write operation in milliseconds -#[inline(always)] -pub fn record_zero_copy_write(size_bytes: usize, duration_ms: f64) { - counter!("rustfs.zero_copy.write.total").increment(1); - histogram!("rustfs.zero_copy.write.size.bytes").record(size_bytes as f64); - histogram!("rustfs.zero_copy.write.duration.ms").record(duration_ms); -} - -/// Record zero-copy write fallback. -/// -/// This happens when zero-copy write fails and the system falls back to regular I/O. -/// -/// # Arguments -/// -/// * `reason` - Reason for the fallback -#[inline(always)] -pub fn record_zero_copy_write_fallback(reason: &str) { - counter!("rustfs.zero_copy.write.fallback.total", "reason" => reason.to_string()).increment(1); -} - -/// Record bytes saved from zero-copy. -/// -/// # Arguments -/// -/// * `size_bytes` - Number of bytes saved from zero-copy -#[inline(always)] -pub fn record_bytes_saved(size_bytes: usize) { - counter!("rustfs.zero_copy.bytes.saved.total").increment(size_bytes as u64); -} - // ============================================================================ // S3 Operation Metrics (GetObject, PutObject, etc.) // ============================================================================ @@ -343,6 +527,9 @@ pub fn record_bytes_saved(size_bytes: usize) { /// * `duration_ms` - Operation duration in milliseconds /// * `size_bytes` - Object size in bytes /// * `from_cache` - Whether the object was served from cache +/// +/// Note: this function records aggregate S3 GET metrics only. It must not be +/// interpreted as the definitive source of truth for data-plane copy mode. #[inline(always)] pub fn record_get_object(duration_ms: f64, size_bytes: i64, from_cache: bool) { counter!("rustfs.s3.get_object.total").increment(1); @@ -365,14 +552,33 @@ pub fn record_get_object(duration_ms: f64, size_bytes: i64, from_cache: bool) { /// /// * `duration_ms` - Operation duration in milliseconds /// * `size_bytes` - Object size in bytes -/// * `zero_copy_enabled` - Whether zero-copy was enabled for this operation +/// * `zero_copy_enabled` - Legacy aggregate flag preserved for compatibility +/// +/// Note: this function records aggregate S3 PUT metrics only. The definitive +/// outcome of request-level fast-path attempts must be tracked separately via +/// ADR 0001 data-plane helpers. #[inline(always)] pub fn record_put_object(duration_ms: f64, size_bytes: i64, zero_copy_enabled: bool) { counter!("rustfs.s3.put_object.total").increment(1); histogram!("rustfs.s3.put_object.duration.ms").record(duration_ms); + counter!( + "rustfs.s3.put_object.bucketed.total", + "size_bucket" => put_size_bucket_label(size_bytes) + ) + .increment(1); + histogram!( + "rustfs.s3.put_object.bucketed.duration.ms", + "size_bucket" => put_size_bucket_label(size_bytes) + ) + .record(duration_ms); if size_bytes > 0 { histogram!("rustfs.s3.put_object.size.bytes").record(size_bytes as f64); + histogram!( + "rustfs.s3.put_object.bucketed.size.bytes", + "size_bucket" => put_size_bucket_label(size_bytes) + ) + .record(size_bytes as f64); } if zero_copy_enabled { @@ -717,10 +923,110 @@ mod tests { use super::*; #[test] - fn test_record_zero_copy_read() { - record_zero_copy_read(1024, 10.5); - record_memory_copy_saved(1024); - record_zero_copy_fallback("test"); + fn test_io_path_as_str_values_stable() { + assert_eq!(IoPath::Fast.as_str(), "fast"); + assert_eq!(IoPath::Legacy.as_str(), "legacy"); + } + + #[test] + fn test_copy_mode_as_str_values_stable() { + assert_eq!(CopyMode::TrueZeroCopy.as_str(), "true_zero_copy"); + assert_eq!(CopyMode::SharedBytes.as_str(), "shared_bytes"); + assert_eq!(CopyMode::SingleCopy.as_str(), "single_copy"); + assert_eq!(CopyMode::Reconstructed.as_str(), "reconstructed"); + assert_eq!(CopyMode::Transformed.as_str(), "transformed"); + } + + #[test] + fn test_fallback_reason_as_str_values_stable() { + assert_eq!(FallbackReason::Unknown.as_str(), "unknown"); + assert_eq!(FallbackReason::MmapDisabled.as_str(), "mmap_disabled"); + assert_eq!(FallbackReason::MmapUnavailable.as_str(), "mmap_unavailable"); + assert_eq!(FallbackReason::SmallObject.as_str(), "small_object"); + assert_eq!(FallbackReason::WindowLimitExceeded.as_str(), "window_limit_exceeded"); + assert_eq!(FallbackReason::UnalignedWindow.as_str(), "unaligned_window"); + assert_eq!(FallbackReason::RangeNotSupported.as_str(), "range_not_supported"); + assert_eq!(FallbackReason::EncryptionEnabled.as_str(), "encryption_enabled"); + assert_eq!(FallbackReason::CompressionEnabled.as_str(), "compression_enabled"); + assert_eq!(FallbackReason::TransformEncryptionLegacy.as_str(), "transform_encryption_legacy"); + assert_eq!(FallbackReason::TransformCompressionLegacy.as_str(), "transform_compression_legacy"); + assert_eq!( + FallbackReason::TransformCompressionEncryptionLegacy.as_str(), + "transform_compression_encryption_legacy" + ); + assert_eq!(FallbackReason::ChunkBridgeUnavailable.as_str(), "chunk_bridge_unavailable"); + assert_eq!(FallbackReason::NonLocalBackend.as_str(), "non_local_backend"); + } + + #[test] + fn test_record_io_path_selected() { + record_io_path_selected("get", IoPath::Fast); + record_io_path_selected("put", IoPath::Legacy); + } + + #[test] + fn test_record_io_copy_mode() { + record_io_copy_mode("get", CopyMode::SharedBytes, 1024); + record_io_copy_mode("put", CopyMode::Transformed, 2048); + } + + #[test] + fn test_record_io_fallback() { + record_io_fallback(IoStage::ReadSetup, FallbackReason::MmapUnavailable); + record_io_fallback(IoStage::HttpBridge, FallbackReason::ChunkBridgeUnavailable); + } + + #[test] + fn test_record_local_disk_active_mmap_bytes() { + record_local_disk_active_mmap_bytes(4096); + record_local_disk_active_mmap_bytes(0); + } + + #[test] + fn test_record_local_disk_pooled_chunk() { + record_local_disk_pooled_chunk("fallback", 4096); + record_local_disk_pooled_chunk("compat_collect", 8192); + } + + #[test] + fn test_record_local_disk_compat_collect() { + record_local_disk_compat_collect(3, 16384); + } + + #[test] + fn test_record_put_object_attempted_fast_path() { + record_put_object_attempted_fast_path(1024 * 1024); + record_put_object_attempted_fast_path(0); + } + + #[test] + fn test_record_put_transform_selected() { + record_put_transform_selected("compression", IoPath::Fast, 2048); + record_put_transform_selected("compression_encryption", IoPath::Legacy, 4096); + } + + #[test] + fn test_record_put_path_selected() { + record_put_path_selected(8 * 1024, IoPath::Fast); + record_put_path_selected(2 * 1024 * 1024, IoPath::Legacy); + } + + #[test] + fn test_record_put_copy_mode() { + record_put_copy_mode(8 * 1024, CopyMode::SingleCopy); + record_put_copy_mode(512 * 1024, CopyMode::Transformed); + } + + #[test] + fn test_record_put_fallback() { + record_put_fallback(32 * 1024, FallbackReason::CompressionEnabled); + record_put_fallback(2 * 1024 * 1024, FallbackReason::EncryptionEnabled); + } + + #[test] + fn test_record_put_inline_selected() { + record_put_inline_selected(8 * 1024, false); + record_put_inline_selected(32 * 1024, true); } #[test] @@ -731,13 +1037,6 @@ mod tests { record_bytes_pool_hit_rate("small", 0.85); } - #[test] - fn test_record_zero_copy_write() { - record_zero_copy_write(1024, 10.5); - record_zero_copy_write_fallback("test"); - record_bytes_saved(1024); - } - // S3 Operation Metrics Tests #[test] fn test_record_get_object() { @@ -857,157 +1156,6 @@ mod tests { } } -// ============================================================================ -// Zero-Copy Optimization Metrics (Phase 1 Extension) -// ============================================================================ - pub mod bandwidth; pub mod global_metrics; pub mod metric_names; - -pub use metric_names::zero_copy; - -/// Record a zero-copy buffer operation. -/// -/// This function records metrics for zero-copy buffer operations, -/// including the operation type and size. -#[inline(always)] -pub fn record_zero_copy_buffer_operation(operation: &str, size: usize) { - counter!( - zero_copy::BUFFER_OPERATIONS_TOTAL, - "operation" => operation.to_string() - ) - .increment(1); - - counter!( - zero_copy::BUFFER_BYTES_TOTAL, - "operation" => operation.to_string() - ) - .increment(size as u64); -} - -/// Record memory copy operations. -/// -/// This function tracks the number and size of memory copies, -/// which should be minimized in zero-copy paths. -#[inline(always)] -pub fn record_memory_copy(count: u32, size: usize) { - counter!(zero_copy::MEMORY_COPY_TOTAL).increment(count as u64); - - counter!(zero_copy::MEMORY_COPY_BYTES_TOTAL).increment(size as u64); - - histogram!("rustfs_memory_copy_size_bytes").record(size as f64); -} - -/// Record a shared reference operation. -/// -/// This function tracks operations that create or use shared references -/// for zero-copy data sharing. -#[inline(always)] -pub fn record_shared_ref_operation(operation: &str) { - counter!( - zero_copy::SHARED_REF_OPERATIONS_TOTAL, - "operation" => operation.to_string() - ) - .increment(1); -} - -/// Record BufReader optimization. -/// -/// This function tracks BufReader layer elimination and buffer size -/// adjustments. -#[inline(always)] -pub fn record_bufreader_optimization(layers_eliminated: u32, buffer_size: usize) { - counter!(zero_copy::BUFREADER_LAYERS_ELIMINATED_TOTAL).increment(layers_eliminated as u64); - - histogram!(zero_copy::BUFREADER_BUFFER_SIZE_BYTES).record(buffer_size as f64); -} - -/// Record Direct I/O operation. -/// -/// This function tracks Direct I/O operations and their success/fallback -/// status. -#[inline(always)] -pub fn record_direct_io_operation(operation: &str, size: usize, success: bool) { - let status = if success { "success" } else { "fallback" }; - - counter!( - zero_copy::DIRECT_IO_OPERATIONS_TOTAL, - "operation" => operation.to_string(), - "status" => status.to_string() - ) - .increment(1); - - counter!( - zero_copy::DIRECT_IO_BYTES_TOTAL, - "operation" => operation.to_string(), - "status" => status.to_string() - ) - .increment(size as u64); -} - -/// Update zero-copy performance metrics. -/// -/// This function updates gauge metrics for overall zero-copy performance. -#[inline(always)] -pub fn update_zero_copy_performance_metrics(copy_count: u32, throughput_mbps: f64, memory_saved: u64) { - gauge!(zero_copy::AVG_COPY_COUNT).set(copy_count as f64); - - gauge!(zero_copy::THROUGHPUT_MBPS).set(throughput_mbps); - - gauge!(zero_copy::MEMORY_SAVED_BYTES).set(memory_saved as f64); -} - -// ============================================================================ -// Zero-Copy Metrics Tests -// ============================================================================ - -#[cfg(test)] -mod zero_copy_tests { - use super::*; - - #[test] - fn test_record_zero_copy_buffer_operation() { - // This test verifies the function compiles and runs - // Actual metric verification requires a metrics recorder - record_zero_copy_buffer_operation("read", 1024); - record_zero_copy_buffer_operation("write", 2048); - } - - #[test] - fn test_record_memory_copy() { - record_memory_copy(1, 1024); - record_memory_copy(2, 2048); - } - - #[test] - fn test_record_shared_ref_operation() { - record_shared_ref_operation("create"); - record_shared_ref_operation("share"); - } - - #[test] - fn test_record_bufreader_optimization() { - record_bufreader_optimization(1, 8192); - record_bufreader_optimization(2, 65536); - } - - #[test] - fn test_record_direct_io_operation() { - record_direct_io_operation("read", 4096, true); - record_direct_io_operation("write", 8192, false); - } - - #[test] - fn test_update_zero_copy_performance_metrics() { - update_zero_copy_performance_metrics(2, 150.5, 1024 * 1024); - } - - #[test] - fn test_metric_names() { - // Verify metric names are defined - assert!(!zero_copy::BUFFER_OPERATIONS_TOTAL.is_empty()); - assert!(!zero_copy::MEMORY_COPY_TOTAL.is_empty()); - assert!(!zero_copy::THROUGHPUT_MBPS.is_empty()); - } -} diff --git a/crates/io-metrics/src/metric_names.rs b/crates/io-metrics/src/metric_names.rs index e7581ff8f..6c611b6aa 100644 --- a/crates/io-metrics/src/metric_names.rs +++ b/crates/io-metrics/src/metric_names.rs @@ -14,41 +14,44 @@ //! Metric name constants for consistent naming across the codebase. -/// Zero-copy operation metric names. -pub mod zero_copy { - /// Total number of zero-copy buffer operations - pub const BUFFER_OPERATIONS_TOTAL: &str = "rustfs_zero_copy_buffer_operations_total"; +/// Request-level data plane metric names introduced by ADR 0001. +pub mod data_plane { + /// Total number of selected request paths. + pub const PATH_SELECTED_TOTAL: &str = "rustfs.io.path.selected_total"; - /// Total bytes processed by zero-copy buffer operations - pub const BUFFER_BYTES_TOTAL: &str = "rustfs_zero_copy_buffer_bytes_total"; + /// Total bytes observed for a given effective copy mode. + pub const COPY_MODE_BYTES_TOTAL: &str = "rustfs.io.copy_mode.bytes_total"; - /// Total number of memory copies - pub const MEMORY_COPY_TOTAL: &str = "rustfs_memory_copy_total"; + /// Total number of data plane fallbacks. + pub const FALLBACK_TOTAL: &str = "rustfs.io.zero_copy.fallback_total"; - /// Total bytes copied in memory - pub const MEMORY_COPY_BYTES_TOTAL: &str = "rustfs_memory_copy_bytes_total"; + /// Current active local-disk mmap bytes held by chunk fast paths. + pub const LOCAL_DISK_ACTIVE_MMAP_BYTES: &str = "rustfs.io.local_disk.active_mmap.bytes"; - /// Total number of shared reference operations - pub const SHARED_REF_OPERATIONS_TOTAL: &str = "rustfs_shared_ref_operations_total"; + /// Total pooled chunks produced or consumed by LocalDisk compatibility paths. + pub const LOCAL_DISK_POOLED_CHUNKS_TOTAL: &str = "rustfs.io.local_disk.pooled_chunks.total"; - /// Total number of BufReader layers eliminated - pub const BUFREADER_LAYERS_ELIMINATED_TOTAL: &str = "rustfs_bufreader_layers_eliminated_total"; + /// Total pooled bytes produced or consumed by LocalDisk compatibility paths. + pub const LOCAL_DISK_POOLED_BYTES_TOTAL: &str = "rustfs.io.local_disk.pooled_bytes.total"; - /// BufReader buffer size distribution - pub const BUFREADER_BUFFER_SIZE_BYTES: &str = "rustfs_bufreader_buffer_size_bytes"; + /// Total number of compatibility chunk-stream aggregations performed for LocalDisk reads. + pub const LOCAL_DISK_COMPAT_COLLECT_TOTAL: &str = "rustfs.io.local_disk.compat_collect.total"; - /// Total number of Direct I/O operations - pub const DIRECT_IO_OPERATIONS_TOTAL: &str = "rustfs_direct_io_operations_total"; + /// Chunk count distribution for LocalDisk compatibility chunk aggregation. + pub const LOCAL_DISK_COMPAT_COLLECT_CHUNKS: &str = "rustfs.io.local_disk.compat_collect.chunks"; - /// Total bytes processed by Direct I/O - pub const DIRECT_IO_BYTES_TOTAL: &str = "rustfs_direct_io_bytes_total"; + /// Byte distribution for LocalDisk compatibility chunk aggregation. + pub const LOCAL_DISK_COMPAT_COLLECT_BYTES: &str = "rustfs.io.local_disk.compat_collect.bytes"; - /// Average copy count per operation - pub const AVG_COPY_COUNT: &str = "rustfs_zero_copy_avg_copy_count"; + /// Total number of attempted PUT fast paths. + pub const PUT_FAST_PATH_ATTEMPTS_TOTAL: &str = "rustfs.io.put.fast_path.attempts_total"; - /// Throughput in MB/s - pub const THROUGHPUT_MBPS: &str = "rustfs_zero_copy_throughput_mbps"; + /// Size distribution for attempted PUT fast paths. + pub const PUT_FAST_PATH_ATTEMPT_SIZE_BYTES: &str = "rustfs.io.put.fast_path.attempt.size.bytes"; - /// Memory saved by zero-copy in bytes - pub const MEMORY_SAVED_BYTES: &str = "rustfs_zero_copy_memory_saved_bytes"; + /// Total number of transformed PUT selections grouped by transform kind and ingress path. + pub const PUT_TRANSFORM_SELECTED_TOTAL: &str = "rustfs.io.put.transform.selected_total"; + + /// Size distribution for transformed PUT selections. + pub const PUT_TRANSFORM_SIZE_BYTES: &str = "rustfs.io.put.transform.size.bytes"; } diff --git a/crates/object-io/Cargo.toml b/crates/object-io/Cargo.toml new file mode 100644 index 000000000..797c8bfcd --- /dev/null +++ b/crates/object-io/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "rustfs-object-io" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +homepage.workspace = true +description = "Object I/O policy helpers and zero-copy support primitives for RustFS." +keywords.workspace = true +categories.workspace = true + +[lints] +workspace = true + +[dependencies] +atoi = { workspace = true } +bytes = { workspace = true } +futures-util = { workspace = true } +rustfs-ecstore = { workspace = true } +rustfs-concurrency = { workspace = true } +rustfs-io-core = { workspace = true } +rustfs-io-metrics = { workspace = true } +rustfs-rio = { workspace = true } +rustfs-s3select-api = { workspace = true } +rustfs-utils = { workspace = true ,features = ["http"]} +http = { workspace = true } +s3s.workspace = true +thiserror = { workspace = true } +time = { workspace = true, features = ["parsing", "formatting"] } +tokio = { workspace = true, features = ["io-util"] } +tokio-util = { workspace = true, features = ["io"] } +astral-tokio-tar = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +serial_test = { workspace = true } diff --git a/crates/object-io/src/get.rs b/crates/object-io/src/get.rs new file mode 100644 index 000000000..90e49b992 --- /dev/null +++ b/crates/object-io/src/get.rs @@ -0,0 +1,1703 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytes::Bytes; +use futures_util::StreamExt; +use http::HeaderMap; +use http::header::{CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_LANGUAGE}; +use rustfs_concurrency::GetObjectCacheEligibility; +use rustfs_ecstore::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; +use rustfs_ecstore::client::object_api_utils::to_s3s_etag; +use rustfs_ecstore::error::StorageError; +use rustfs_ecstore::store_api::{GetObjectChunkCopyMode, GetObjectChunkPath, GetObjectChunkResult, HTTPRangeSpec, ObjectInfo}; +use rustfs_io_core::BoxChunkStream; +use rustfs_rio::Reader; +use rustfs_s3select_api::object_store::bytes_stream; +use rustfs_utils::http::{AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE}; +use s3s::dto::{ + ChecksumType, ContentType, GetObjectOutput, SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, ServerSideEncryption, + StreamingBlob, Timestamp, +}; +use std::collections::HashMap; +use std::str::FromStr; +use std::sync::Arc; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; +use tokio::io::{AsyncRead, AsyncSeek, ReadBuf}; +use tokio_util::io::ReaderStream; + +pub struct InMemoryAsyncReader { + cursor: std::io::Cursor, +} + +impl InMemoryAsyncReader { + pub fn new(data: Bytes) -> Self { + Self { + cursor: std::io::Cursor::new(data), + } + } +} + +impl AsyncRead for InMemoryAsyncReader { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> std::task::Poll> { + let unfilled = buf.initialize_unfilled(); + let bytes_read = std::io::Read::read(&mut self.cursor, unfilled)?; + buf.advance(bytes_read); + std::task::Poll::Ready(Ok(())) + } +} + +impl AsyncSeek for InMemoryAsyncReader { + fn start_seek(mut self: std::pin::Pin<&mut Self>, position: std::io::SeekFrom) -> std::io::Result<()> { + std::io::Seek::seek(&mut self.cursor, position)?; + Ok(()) + } + + fn poll_complete(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll> { + std::task::Poll::Ready(Ok(self.cursor.position())) + } +} + +pub fn build_memory_blob(buf: Bytes, response_content_length: i64, optimal_buffer_size: usize) -> Option { + let mem_reader = InMemoryAsyncReader::new(buf); + Some(StreamingBlob::wrap(bytes_stream( + ReaderStream::with_capacity(Box::new(mem_reader), optimal_buffer_size), + response_content_length as usize, + ))) +} + +#[derive(Clone)] +pub struct FrozenGetObjectBody { + body: Arc, +} + +impl FrozenGetObjectBody { + pub fn new(body: Bytes) -> Self { + Self { body: Arc::new(body) } + } + + pub fn shared_body(&self) -> &Arc { + &self.body + } + + pub fn into_shared_body(self) -> Arc { + self.body + } + + pub fn build_blob(&self, response_content_length: i64, optimal_buffer_size: usize) -> Option { + build_memory_blob((*self.body).clone(), response_content_length, optimal_buffer_size) + } +} + +pub fn build_reader_blob(reader: R, response_content_length: i64, optimal_buffer_size: usize) -> Option +where + R: AsyncRead + Send + Sync + 'static, +{ + Some(StreamingBlob::wrap(bytes_stream( + ReaderStream::with_capacity(reader, optimal_buffer_size), + response_content_length as usize, + ))) +} + +pub fn build_chunk_blob(chunk_stream: BoxChunkStream) -> Option { + Some(StreamingBlob::wrap(chunk_stream.map(|result| result.map(|chunk| chunk.as_bytes())))) +} + +/// ADR-facing alias for the chunk-stream to HTTP body bridge. +pub fn build_chunk_http_body(chunk_stream: BoxChunkStream) -> Option { + build_chunk_blob(chunk_stream) +} + +pub fn map_chunk_copy_mode(copy_mode: GetObjectChunkCopyMode) -> rustfs_io_metrics::CopyMode { + match copy_mode { + GetObjectChunkCopyMode::TrueZeroCopy => rustfs_io_metrics::CopyMode::TrueZeroCopy, + GetObjectChunkCopyMode::SharedBytes => rustfs_io_metrics::CopyMode::SharedBytes, + GetObjectChunkCopyMode::SingleCopy => rustfs_io_metrics::CopyMode::SingleCopy, + GetObjectChunkCopyMode::Reconstructed => rustfs_io_metrics::CopyMode::Reconstructed, + } +} + +pub fn chunk_body_data_plane_labels( + path: GetObjectChunkPath, + copy_mode: rustfs_io_metrics::CopyMode, +) -> (rustfs_io_metrics::IoPath, rustfs_io_metrics::CopyMode) { + ( + match path { + GetObjectChunkPath::Direct | GetObjectChunkPath::Bridge => rustfs_io_metrics::IoPath::Fast, + }, + copy_mode, + ) +} + +pub fn get_object_chunk_fast_path_guard( + has_sse_customer_key: bool, + has_sse_customer_key_md5: bool, +) -> Result<(), ChunkReadFallback> { + if has_sse_customer_key || has_sse_customer_key_md5 { + return Err(ChunkReadFallback::read_setup(rustfs_io_metrics::FallbackReason::EncryptionEnabled)); + } + + Ok(()) +} + +pub fn get_object_sequential_hint(rs: Option<&HTTPRangeSpec>) -> bool { + if rs.is_none() { + true + } else if let Some(range_spec) = rs { + range_spec.start == 0 && !range_spec.is_suffix_length + } else { + false + } +} + +pub trait CachedGetObjectSource { + fn body(&self) -> &Arc; + fn content_length(&self) -> i64; + fn content_type(&self) -> Option<&str>; + fn e_tag(&self) -> Option<&str>; + fn last_modified(&self) -> Option<&str>; + fn cache_control(&self) -> Option<&str>; + fn content_disposition(&self) -> Option<&str>; + fn content_encoding(&self) -> Option<&str>; + fn content_language(&self) -> Option<&str>; + fn storage_class(&self) -> Option<&str>; + fn version_id(&self) -> Option<&str>; + fn delete_marker(&self) -> bool; + fn tag_count(&self) -> Option; + fn user_metadata(&self) -> &HashMap; + fn checksum_crc32(&self) -> Option<&str>; + fn checksum_crc32c(&self) -> Option<&str>; + fn checksum_sha1(&self) -> Option<&str>; + fn checksum_sha256(&self) -> Option<&str>; + fn checksum_crc64nvme(&self) -> Option<&str>; + fn checksum_type(&self) -> Option<&ChecksumType>; +} + +pub struct GetObjectOutputContext { + pub output: GetObjectOutput, + pub event_info: ObjectInfo, + pub response_content_length: i64, + pub optimal_buffer_size: usize, + pub copy_mode_override: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GetObjectStrategyLayout { + pub is_sequential_hint: bool, + pub optimal_buffer_size: usize, +} + +pub enum GetObjectBodySource { + Reader(Box), + Chunk { + stream: BoxChunkStream, + path: GetObjectChunkPath, + copy_mode: rustfs_io_metrics::CopyMode, + }, +} + +pub struct GetObjectReadSetup { + pub info: ObjectInfo, + pub event_info: ObjectInfo, + pub body_source: GetObjectBodySource, + pub rs: Option, + pub content_type: Option, + pub last_modified: Option, + pub response_content_length: i64, + pub content_range: Option, + pub server_side_encryption: Option, + pub sse_customer_algorithm: Option, + pub sse_customer_key_md5: Option, + pub ssekms_key_id: Option, + pub encryption_applied: bool, +} + +pub struct LegacyReadPlan { + pub rs: Option, + pub content_type: Option, + pub last_modified: Option, + pub response_content_length: i64, + pub content_range: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GetObjectBodyPlan { + CacheWriteback, + BufferEncrypted, + BufferSeekable, + Stream, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GetObjectDataPlaneRequestSource { + CacheServed, + Disk, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GetObjectDataPlaneMetricContract { + pub request_source: GetObjectDataPlaneRequestSource, + pub io_path: rustfs_io_metrics::IoPath, + pub copy_mode: rustfs_io_metrics::CopyMode, + pub record_cache_served_metric: bool, + pub record_cache_writeback_metric: bool, +} + +impl GetObjectDataPlaneMetricContract { + pub fn cache_served() -> Self { + Self { + request_source: GetObjectDataPlaneRequestSource::CacheServed, + io_path: rustfs_io_metrics::IoPath::Fast, + copy_mode: rustfs_io_metrics::CopyMode::SharedBytes, + record_cache_served_metric: true, + record_cache_writeback_metric: false, + } + } + + pub fn disk( + io_path: rustfs_io_metrics::IoPath, + copy_mode: rustfs_io_metrics::CopyMode, + body_plan: GetObjectBodyPlan, + ) -> Self { + Self { + request_source: GetObjectDataPlaneRequestSource::Disk, + io_path, + copy_mode, + record_cache_served_metric: false, + record_cache_writeback_metric: matches!(body_plan, GetObjectBodyPlan::CacheWriteback), + } + } +} + +pub struct GetObjectCacheWriteback { + pub body: Arc, + pub content_length: i64, + pub content_type: Option, + pub content_encoding: Option, + pub cache_control: Option, + pub content_disposition: Option, + pub content_language: Option, + pub expires: Option, + pub storage_class: Option, + pub version_id: Option, + pub delete_marker: bool, + pub user_metadata: HashMap, + pub e_tag: Option, + pub last_modified: Option, + pub checksum_crc32: Option, + pub checksum_crc32c: Option, + pub checksum_sha1: Option, + pub checksum_sha256: Option, + pub checksum_crc64nvme: Option, + pub checksum_type: Option, +} + +pub struct GetObjectBodyMaterialization { + pub body: Option, + pub cache_writeback: Option, + pub plan: GetObjectBodyPlan, +} + +#[derive(Default)] +pub struct GetObjectEncryptionState { + pub server_side_encryption: Option, + pub sse_customer_algorithm: Option, + pub sse_customer_key_md5: Option, + pub ssekms_key_id: Option, + pub encryption_applied: bool, + pub response_content_length_override: Option, +} + +pub struct ChunkReadSetupResult { + pub read_setup: GetObjectReadSetup, + pub io_path: rustfs_io_metrics::IoPath, +} + +#[derive(Debug, thiserror::Error)] +pub enum MaterializeGetObjectBodyError { + #[error("failed to read object for caching: {0}")] + CacheRead(std::io::Error), + #[error("failed to read decrypted object: {0}")] + EncryptedRead(std::io::Error), +} + +pub enum GetObjectResponseMode { + Plain, + CorsWrapped, +} + +pub struct GetObjectFlowResult { + pub output: GetObjectOutput, + pub event_info: ObjectInfo, + pub version_id_for_event: String, + pub response_mode: GetObjectResponseMode, +} + +pub fn build_get_object_flow_result( + output: GetObjectOutput, + event_info: ObjectInfo, + version_id_for_event: String, + response_mode: GetObjectResponseMode, +) -> GetObjectFlowResult { + GetObjectFlowResult { + output, + event_info, + version_id_for_event, + response_mode, + } +} + +pub fn build_cached_get_object_flow_result_from_source( + bucket: &str, + key: &str, + cached: &T, + version_id_for_event: String, +) -> GetObjectFlowResult +where + T: CachedGetObjectSource, +{ + build_get_object_flow_result( + build_cached_get_object_output_from_source(cached), + build_cached_get_object_event_info_from_source(bucket, key, cached), + version_id_for_event, + GetObjectResponseMode::Plain, + ) +} + +pub fn build_cors_wrapped_get_object_flow_result( + output_context: GetObjectOutputContext, + version_id_for_event: String, +) -> GetObjectFlowResult { + let GetObjectOutputContext { + output, + event_info, + response_content_length: _, + optimal_buffer_size: _, + copy_mode_override: _, + } = output_context; + build_get_object_flow_result(output, event_info, version_id_for_event, GetObjectResponseMode::CorsWrapped) +} + +pub fn build_cached_get_object_output_from_source(cached: &T) -> GetObjectOutput +where + T: CachedGetObjectSource, +{ + let body_data = Arc::clone(cached.body()); + let body = Some(StreamingBlob::wrap::<_, std::convert::Infallible>(futures_util::stream::once( + async move { Ok((*body_data).clone()) }, + ))); + + let last_modified = cached + .last_modified() + .and_then(|s| OffsetDateTime::parse(s, &Rfc3339).ok()) + .map(Timestamp::from); + + let content_type = cached.content_type().and_then(|ct| ContentType::from_str(ct).ok()); + + let metadata = (!cached.user_metadata().is_empty()).then(|| cached.user_metadata().clone()); + + GetObjectOutput { + body, + content_length: Some(cached.content_length()), + accept_ranges: Some("bytes".to_string()), + e_tag: cached.e_tag().map(to_s3s_etag), + last_modified, + content_type, + cache_control: cached.cache_control().map(str::to_string), + content_disposition: cached.content_disposition().map(str::to_string), + content_encoding: cached.content_encoding().map(str::to_string), + content_language: cached.content_language().map(str::to_string), + version_id: cached.version_id().map(str::to_string), + delete_marker: Some(cached.delete_marker()), + tag_count: cached.tag_count(), + metadata, + checksum_crc32: cached.checksum_crc32().map(str::to_string), + checksum_crc32c: cached.checksum_crc32c().map(str::to_string), + checksum_sha1: cached.checksum_sha1().map(str::to_string), + checksum_sha256: cached.checksum_sha256().map(str::to_string), + checksum_crc64nvme: cached.checksum_crc64nvme().map(str::to_string), + checksum_type: cached.checksum_type().cloned(), + ..Default::default() + } +} +pub fn build_cached_get_object_event_info_from_source(bucket: &str, key: &str, cached: &T) -> ObjectInfo +where + T: CachedGetObjectSource, +{ + let last_modified = cached.last_modified().and_then(|s| OffsetDateTime::parse(s, &Rfc3339).ok()); + let version_id = cached.version_id().and_then(|v| uuid::Uuid::parse_str(v).ok()); + + ObjectInfo { + bucket: bucket.to_string(), + name: key.to_string(), + storage_class: cached.storage_class().map(str::to_string), + mod_time: last_modified, + size: cached.content_length(), + actual_size: cached.content_length(), + is_dir: false, + user_defined: cached.user_metadata().clone(), + version_id, + delete_marker: cached.delete_marker(), + content_type: cached.content_type().map(str::to_string), + content_encoding: cached.content_encoding().map(str::to_string), + etag: cached.e_tag().map(str::to_string), + ..Default::default() + } +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct GetObjectChecksums { + pub crc32: Option, + pub crc32c: Option, + pub sha1: Option, + pub sha256: Option, + pub crc64nvme: Option, + pub checksum_type: Option, +} + +fn decode_get_object_checksums(decrypted_checksums: HashMap) -> GetObjectChecksums { + let mut checksums = GetObjectChecksums::default(); + + for (key, checksum) in decrypted_checksums { + if key == AMZ_CHECKSUM_TYPE { + checksums.checksum_type = Some(ChecksumType::from(checksum)); + continue; + } + + match rustfs_rio::ChecksumType::from_string(key.as_str()) { + rustfs_rio::ChecksumType::CRC32 => checksums.crc32 = Some(checksum), + rustfs_rio::ChecksumType::CRC32C => checksums.crc32c = Some(checksum), + rustfs_rio::ChecksumType::SHA1 => checksums.sha1 = Some(checksum), + rustfs_rio::ChecksumType::SHA256 => checksums.sha256 = Some(checksum), + rustfs_rio::ChecksumType::CRC64_NVME => checksums.crc64nvme = Some(checksum), + _ => (), + } + } + + checksums +} + +fn read_object_checksums( + info: &ObjectInfo, + headers: &HeaderMap, + part_number: Option, +) -> std::io::Result { + let (decrypted_checksums, _is_multipart) = info + .decrypt_checksums(part_number.unwrap_or(0), headers) + .map_err(|e| std::io::Error::other(e.to_string()))?; + + Ok(decode_get_object_checksums(decrypted_checksums)) +} + +pub fn build_get_object_checksums( + info: &ObjectInfo, + headers: &HeaderMap, + part_number: Option, + rs: Option<&HTTPRangeSpec>, +) -> std::io::Result { + if let Some(checksum_mode) = headers.get(AMZ_CHECKSUM_MODE) + && checksum_mode.to_str().unwrap_or_default() == "ENABLED" + && rs.is_none() + { + return read_object_checksums(info, headers, part_number); + } + + Ok(GetObjectChecksums::default()) +} + +pub fn build_output_version_id(versioned: bool, version_id: Option<&uuid::Uuid>) -> Option { + if !versioned { + return None; + } + + version_id.map(|vid| { + if *vid == uuid::Uuid::nil() { + "null".to_string() + } else { + vid.to_string() + } + }) +} + +pub fn plan_get_object_strategy_layout( + rs: Option<&HTTPRangeSpec>, + response_content_length: i64, + suggested_buffer_size: usize, + fallback_buffer_size: usize, +) -> GetObjectStrategyLayout { + let is_sequential_hint = get_object_sequential_hint(rs); + let optimal_buffer_size = if suggested_buffer_size > 0 { + suggested_buffer_size.min(fallback_buffer_size) + } else { + rustfs_io_core::get_concurrency_aware_buffer_size(response_content_length, fallback_buffer_size) + }; + + GetObjectStrategyLayout { + is_sequential_hint, + optimal_buffer_size, + } +} + +pub fn plan_get_object_body( + cache_eligibility: GetObjectCacheEligibility, + seekable_object_size_threshold: usize, +) -> GetObjectBodyPlan { + if cache_eligibility.should_cache() { + return GetObjectBodyPlan::CacheWriteback; + } + + let should_buffer_for_seek = cache_eligibility.response_size > 0 + && cache_eligibility.response_size <= seekable_object_size_threshold as i64 + && !cache_eligibility.is_part_request + && !cache_eligibility.is_range_request; + + if cache_eligibility.encryption_applied && should_buffer_for_seek { + GetObjectBodyPlan::BufferEncrypted + } else if should_buffer_for_seek { + GetObjectBodyPlan::BufferSeekable + } else { + GetObjectBodyPlan::Stream + } +} + +pub fn build_get_object_cache_writeback(info: &ObjectInfo, body: Bytes, content_length: i64) -> GetObjectCacheWriteback { + let checksums = read_object_checksums(info, &HeaderMap::new(), None).unwrap_or_default(); + let body = FrozenGetObjectBody::new(body); + GetObjectCacheWriteback { + body: body.into_shared_body(), + content_length, + content_type: info.content_type.clone(), + content_encoding: info.content_encoding.clone(), + cache_control: None, + content_disposition: None, + content_language: None, + expires: None, + storage_class: info.storage_class.clone(), + version_id: info.version_id.map(|vid| { + if vid == uuid::Uuid::nil() { + "null".to_string() + } else { + vid.to_string() + } + }), + delete_marker: info.delete_marker, + user_metadata: HashMap::new(), + e_tag: info.etag.clone(), + last_modified: info.mod_time.and_then(|t| t.format(&Rfc3339).ok()), + checksum_crc32: checksums.crc32, + checksum_crc32c: checksums.crc32c, + checksum_sha1: checksums.sha1, + checksum_sha256: checksums.sha256, + checksum_crc64nvme: checksums.crc64nvme, + checksum_type: checksums.checksum_type, + } +} + +pub fn finalize_get_object_cache_writeback( + info: &ObjectInfo, + writeback: GetObjectCacheWriteback, + user_metadata: HashMap, +) -> GetObjectCacheWriteback { + GetObjectCacheWriteback { + cache_control: info.user_defined.get(CACHE_CONTROL.as_str()).cloned(), + content_disposition: info.user_defined.get(CONTENT_DISPOSITION.as_str()).cloned(), + content_language: info.user_defined.get(CONTENT_LANGUAGE.as_str()).cloned(), + expires: info.expires.and_then(|t| t.format(&Rfc3339).ok()), + user_metadata, + ..writeback + } +} + +pub async fn materialize_get_object_body( + mut final_stream: R, + info: &ObjectInfo, + plan: GetObjectBodyPlan, + response_content_length: i64, + optimal_buffer_size: usize, +) -> Result +where + R: AsyncRead + Send + Sync + Unpin + 'static, +{ + match plan { + GetObjectBodyPlan::CacheWriteback => { + let mut buf = Vec::with_capacity(response_content_length as usize); + tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf) + .await + .map_err(MaterializeGetObjectBodyError::CacheRead)?; + let body = FrozenGetObjectBody::new(Bytes::from(buf)); + + Ok(GetObjectBodyMaterialization { + body: body.build_blob(response_content_length, optimal_buffer_size), + cache_writeback: Some(build_get_object_cache_writeback( + info, + body.shared_body().as_ref().clone(), + response_content_length, + )), + plan, + }) + } + GetObjectBodyPlan::BufferEncrypted => { + let mut buf = Vec::with_capacity(response_content_length as usize); + tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf) + .await + .map_err(MaterializeGetObjectBodyError::EncryptedRead)?; + let body = FrozenGetObjectBody::new(Bytes::from(buf)); + + Ok(GetObjectBodyMaterialization { + body: body.build_blob(response_content_length, optimal_buffer_size), + cache_writeback: None, + plan, + }) + } + GetObjectBodyPlan::BufferSeekable => { + let mut buf = Vec::with_capacity(response_content_length as usize); + let body = match tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { + Ok(_) => FrozenGetObjectBody::new(Bytes::from(buf)).build_blob(response_content_length, optimal_buffer_size), + Err(_) => build_reader_blob(final_stream, response_content_length, optimal_buffer_size), + }; + + Ok(GetObjectBodyMaterialization { + body, + cache_writeback: None, + plan, + }) + } + GetObjectBodyPlan::Stream => Ok(GetObjectBodyMaterialization { + body: build_reader_blob(final_stream, response_content_length, optimal_buffer_size), + cache_writeback: None, + plan, + }), + } +} + +fn resolve_requested_range( + info: &ObjectInfo, + mut rs: Option, + part_number: Option, +) -> Option { + if let Some(part_number) = part_number + && rs.is_none() + { + rs = HTTPRangeSpec::from_object_info(info, part_number); + } + + rs +} + +fn resolve_response_range( + total_size: i64, + rs: Option, +) -> std::io::Result<(Option, i64, Option)> { + let Some(range_spec) = rs else { + return Ok((None, total_size, None)); + }; + + let (start, length) = range_spec.get_offset_length(total_size)?; + let content_range = Some(format!("bytes {}-{}/{}", start, start as i64 + length - 1, total_size)); + + Ok((Some(range_spec), length, content_range)) +} + +pub fn plan_legacy_read( + info: &ObjectInfo, + rs: Option, + part_number: Option, +) -> std::io::Result { + let content_type = info + .content_type + .as_ref() + .and_then(|content_type| ContentType::from_str(content_type).ok()); + let last_modified = info.mod_time.map(Timestamp::from); + let rs = resolve_requested_range(info, rs, part_number); + let total_size = info.get_actual_size()?; + let (rs, response_content_length, content_range) = resolve_response_range(total_size, rs)?; + + Ok(LegacyReadPlan { + rs, + content_type, + last_modified, + response_content_length, + content_range, + }) +} + +pub fn build_reader_read_setup( + info: ObjectInfo, + event_info: ObjectInfo, + final_stream: Box, + plan: LegacyReadPlan, + encryption_state: GetObjectEncryptionState, +) -> GetObjectReadSetup { + let LegacyReadPlan { + rs, + content_type, + last_modified, + response_content_length, + content_range, + } = plan; + + let GetObjectEncryptionState { + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + encryption_applied, + response_content_length_override, + } = encryption_state; + + GetObjectReadSetup { + info, + event_info, + body_source: GetObjectBodySource::Reader(final_stream), + rs, + content_type, + last_modified, + response_content_length: response_content_length_override.unwrap_or(response_content_length), + content_range, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + encryption_applied, + } +} + +#[allow(clippy::too_many_arguments)] +pub fn build_get_object_output_context( + body: Option, + info: ObjectInfo, + event_info: ObjectInfo, + content_type: Option, + last_modified: Option, + response_content_length: i64, + content_range: Option, + server_side_encryption: Option, + sse_customer_algorithm: Option, + sse_customer_key_md5: Option, + ssekms_key_id: Option, + checksums: &GetObjectChecksums, + filtered_metadata: Option>, + versioned: bool, + optimal_buffer_size: usize, + copy_mode_override: Option, +) -> GetObjectOutputContext { + let output_version_id = build_output_version_id(versioned, info.version_id.as_ref()); + let output = build_get_object_output( + body, + &info, + content_type, + last_modified, + response_content_length, + content_range, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + checksums, + output_version_id, + filtered_metadata, + ); + + GetObjectOutputContext { + output, + event_info, + response_content_length, + optimal_buffer_size, + copy_mode_override, + } +} + +#[allow(clippy::too_many_arguments)] +fn build_chunk_read_setup( + info: ObjectInfo, + event_info: ObjectInfo, + path: GetObjectChunkPath, + copy_mode: rustfs_io_metrics::CopyMode, + stream: BoxChunkStream, + plan: ChunkReadPlan, +) -> GetObjectReadSetup { + let ChunkReadPlan { + rs, + content_type, + last_modified, + response_content_length, + content_range, + } = plan; + + GetObjectReadSetup { + info, + event_info, + body_source: GetObjectBodySource::Chunk { stream, path, copy_mode }, + rs, + content_type, + last_modified, + response_content_length, + content_range, + server_side_encryption: None, + sse_customer_algorithm: None, + sse_customer_key_md5: None, + ssekms_key_id: None, + encryption_applied: false, + } +} + +pub fn finalize_chunk_read_setup( + info: ObjectInfo, + event_info: ObjectInfo, + chunk_result: GetObjectChunkResult, + plan: ChunkReadPlan, +) -> ChunkReadSetupResult { + let copy_mode = map_chunk_copy_mode(chunk_result.copy_mode); + let (io_path, _) = chunk_body_data_plane_labels(chunk_result.path, copy_mode); + + ChunkReadSetupResult { + io_path, + read_setup: build_chunk_read_setup(info, event_info, chunk_result.path, copy_mode, chunk_result.stream, plan), + } +} + +#[allow(clippy::too_many_arguments)] +pub fn build_get_object_output( + body: Option, + info: &ObjectInfo, + content_type: Option, + last_modified: Option, + response_content_length: i64, + content_range: Option, + server_side_encryption: Option, + sse_customer_algorithm: Option, + sse_customer_key_md5: Option, + ssekms_key_id: Option, + checksums: &GetObjectChecksums, + output_version_id: Option, + filtered_metadata: Option>, +) -> GetObjectOutput { + GetObjectOutput { + body, + content_length: Some(response_content_length), + last_modified, + content_type, + content_encoding: info.content_encoding.clone(), + accept_ranges: Some("bytes".to_string()), + content_range, + e_tag: info.etag.as_ref().map(|etag| to_s3s_etag(etag)), + metadata: filtered_metadata, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + checksum_crc32: checksums.crc32.clone(), + checksum_crc32c: checksums.crc32c.clone(), + checksum_sha1: checksums.sha1.clone(), + checksum_sha256: checksums.sha256.clone(), + checksum_crc64nvme: checksums.crc64nvme.clone(), + checksum_type: checksums.checksum_type.clone(), + version_id: output_version_id, + ..Default::default() + } +} + +#[derive(Debug)] +pub struct ChunkReadPlan { + pub rs: Option, + pub content_type: Option, + pub last_modified: Option, + pub response_content_length: i64, + pub content_range: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ChunkReadFallback { + pub stage: rustfs_io_metrics::IoStage, + pub reason: rustfs_io_metrics::FallbackReason, +} + +impl ChunkReadFallback { + pub const fn new(stage: rustfs_io_metrics::IoStage, reason: rustfs_io_metrics::FallbackReason) -> Self { + Self { stage, reason } + } + + pub const fn read_setup(reason: rustfs_io_metrics::FallbackReason) -> Self { + Self::new(rustfs_io_metrics::IoStage::ReadSetup, reason) + } + + pub const fn range_guard(reason: rustfs_io_metrics::FallbackReason) -> Self { + Self::new(rustfs_io_metrics::IoStage::RangeGuard, reason) + } +} + +#[derive(Debug)] +pub enum ChunkReadDecision { + Eligible(ChunkReadPlan), + Fallback(ChunkReadFallback), +} + +#[derive(Debug)] +pub enum ChunkReadPlanError { + NoSuchKey, + MethodNotAllowed, + Io(std::io::Error), +} + +impl From for ChunkReadPlanError { + fn from(value: std::io::Error) -> Self { + Self::Io(value) + } +} + +impl From for ChunkReadPlanError { + fn from(value: StorageError) -> Self { + Self::Io(std::io::Error::other(value.to_string())) + } +} + +pub fn get_object_chunk_range_guard(rs: Option<&HTTPRangeSpec>) -> Result<(), ChunkReadFallback> { + let Some(range_spec) = rs else { + return Ok(()); + }; + + let unsupported = if range_spec.is_suffix_length { + range_spec.end != -1 + } else { + range_spec.start < 0 || range_spec.end < -1 || (range_spec.end != -1 && range_spec.end < range_spec.start) + }; + + if unsupported { + return Err(ChunkReadFallback::range_guard(rustfs_io_metrics::FallbackReason::RangeNotSupported)); + } + + Ok(()) +} + +pub fn plan_chunk_read( + info: &ObjectInfo, + version_id_missing: bool, + rs: Option, + part_number: Option, +) -> Result { + if info.delete_marker { + if version_id_missing { + return Err(ChunkReadPlanError::NoSuchKey); + } + return Err(ChunkReadPlanError::MethodNotAllowed); + } + + let (_, is_compressed) = info.is_compressed_ok()?; + if is_compressed { + return Ok(ChunkReadDecision::Fallback(ChunkReadFallback::read_setup( + rustfs_io_metrics::FallbackReason::CompressionEnabled, + ))); + } + + if info.transitioned_object.status == TRANSITION_COMPLETE { + return Ok(ChunkReadDecision::Fallback(ChunkReadFallback::read_setup( + rustfs_io_metrics::FallbackReason::NonLocalBackend, + ))); + } + + let has_encryption_metadata = info.user_defined.contains_key("x-rustfs-encryption-key") + || info.user_defined.contains_key("x-amz-server-side-encryption") + || info + .user_defined + .contains_key("x-amz-server-side-encryption-customer-algorithm"); + if has_encryption_metadata { + return Ok(ChunkReadDecision::Fallback(ChunkReadFallback::read_setup( + rustfs_io_metrics::FallbackReason::EncryptionEnabled, + ))); + } + + let rs = resolve_requested_range(info, rs, part_number); + if let Err(fallback) = get_object_chunk_range_guard(rs.as_ref()) { + return Ok(ChunkReadDecision::Fallback(fallback)); + } + + let content_type = info + .content_type + .as_ref() + .and_then(|content_type| ContentType::from_str(content_type).ok()); + let last_modified = info.mod_time.map(Timestamp::from); + let total_size = info.get_actual_size()?; + let (rs, response_content_length, content_range) = resolve_response_range(total_size, rs)?; + + Ok(ChunkReadDecision::Eligible(ChunkReadPlan { + rs, + content_type, + last_modified, + response_content_length, + content_range, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct MockCachedSource { + body: Arc, + content_length: i64, + content_type: Option, + e_tag: Option, + last_modified: Option, + cache_control: Option, + content_disposition: Option, + content_encoding: Option, + content_language: Option, + storage_class: Option, + version_id: Option, + delete_marker: bool, + tag_count: Option, + user_metadata: HashMap, + checksum_crc32: Option, + checksum_crc32c: Option, + checksum_sha1: Option, + checksum_sha256: Option, + checksum_crc64nvme: Option, + checksum_type: Option, + } + + impl CachedGetObjectSource for MockCachedSource { + fn body(&self) -> &Arc { + &self.body + } + + fn content_length(&self) -> i64 { + self.content_length + } + + fn content_type(&self) -> Option<&str> { + self.content_type.as_deref() + } + + fn e_tag(&self) -> Option<&str> { + self.e_tag.as_deref() + } + + fn last_modified(&self) -> Option<&str> { + self.last_modified.as_deref() + } + + fn cache_control(&self) -> Option<&str> { + self.cache_control.as_deref() + } + + fn content_disposition(&self) -> Option<&str> { + self.content_disposition.as_deref() + } + + fn content_encoding(&self) -> Option<&str> { + self.content_encoding.as_deref() + } + + fn content_language(&self) -> Option<&str> { + self.content_language.as_deref() + } + + fn storage_class(&self) -> Option<&str> { + self.storage_class.as_deref() + } + + fn version_id(&self) -> Option<&str> { + self.version_id.as_deref() + } + + fn delete_marker(&self) -> bool { + self.delete_marker + } + + fn tag_count(&self) -> Option { + self.tag_count + } + + fn user_metadata(&self) -> &HashMap { + &self.user_metadata + } + + fn checksum_crc32(&self) -> Option<&str> { + self.checksum_crc32.as_deref() + } + + fn checksum_crc32c(&self) -> Option<&str> { + self.checksum_crc32c.as_deref() + } + + fn checksum_sha1(&self) -> Option<&str> { + self.checksum_sha1.as_deref() + } + + fn checksum_sha256(&self) -> Option<&str> { + self.checksum_sha256.as_deref() + } + + fn checksum_crc64nvme(&self) -> Option<&str> { + self.checksum_crc64nvme.as_deref() + } + + fn checksum_type(&self) -> Option<&ChecksumType> { + self.checksum_type.as_ref() + } + } + + #[test] + fn map_chunk_copy_mode_uses_expected_metric_modes() { + assert_eq!( + map_chunk_copy_mode(GetObjectChunkCopyMode::TrueZeroCopy), + rustfs_io_metrics::CopyMode::TrueZeroCopy + ); + assert_eq!( + map_chunk_copy_mode(GetObjectChunkCopyMode::SharedBytes), + rustfs_io_metrics::CopyMode::SharedBytes + ); + assert_eq!( + map_chunk_copy_mode(GetObjectChunkCopyMode::SingleCopy), + rustfs_io_metrics::CopyMode::SingleCopy + ); + assert_eq!( + map_chunk_copy_mode(GetObjectChunkCopyMode::Reconstructed), + rustfs_io_metrics::CopyMode::Reconstructed + ); + } + + #[test] + fn chunk_body_labels_keep_fast_path_and_copy_mode() { + let (path, copy_mode) = chunk_body_data_plane_labels(GetObjectChunkPath::Bridge, rustfs_io_metrics::CopyMode::SingleCopy); + assert_eq!(path, rustfs_io_metrics::IoPath::Fast); + assert_eq!(copy_mode, rustfs_io_metrics::CopyMode::SingleCopy); + } + + #[test] + fn get_object_chunk_fast_path_guard_rejects_ssec_requests() { + let err = get_object_chunk_fast_path_guard(true, false).unwrap_err(); + assert_eq!( + err, + ChunkReadFallback { + stage: rustfs_io_metrics::IoStage::ReadSetup, + reason: rustfs_io_metrics::FallbackReason::EncryptionEnabled, + } + ); + } + + #[test] + fn get_object_chunk_fast_path_guard_allows_plain_request() { + assert!(get_object_chunk_fast_path_guard(false, false).is_ok()); + } + + #[test] + fn get_object_chunk_range_guard_rejects_invalid_suffix_range() { + let err = get_object_chunk_range_guard(Some(&HTTPRangeSpec { + is_suffix_length: true, + start: 4, + end: 0, + })) + .unwrap_err(); + + assert_eq!( + err, + ChunkReadFallback { + stage: rustfs_io_metrics::IoStage::RangeGuard, + reason: rustfs_io_metrics::FallbackReason::RangeNotSupported, + } + ); + } + + #[test] + fn build_get_object_checksums_returns_default_when_mode_absent() { + let checksums = build_get_object_checksums(&ObjectInfo::default(), &HeaderMap::new(), None, None).unwrap(); + assert_eq!(checksums, GetObjectChecksums::default()); + } + + #[test] + fn plan_chunk_read_returns_fallback_for_compressed_object() { + let mut info = ObjectInfo::default(); + rustfs_utils::http::insert_str( + &mut info.user_defined, + rustfs_utils::http::SUFFIX_COMPRESSION, + rustfs_utils::CompressionAlgorithm::Zstd.to_string(), + ); + + let decision = plan_chunk_read(&info, true, None, None).unwrap(); + assert!( + matches!( + decision, + ChunkReadDecision::Fallback(ChunkReadFallback { + stage: rustfs_io_metrics::IoStage::ReadSetup, + reason: rustfs_io_metrics::FallbackReason::CompressionEnabled, + }) + ), + "unexpected decision: {:?}", + decision + ); + } + + #[test] + fn plan_chunk_read_returns_fallback_for_transitioned_object() { + let mut info = ObjectInfo::default(); + info.transitioned_object.status = TRANSITION_COMPLETE.to_string(); + + let decision = plan_chunk_read(&info, true, None, None).unwrap(); + assert!(matches!( + decision, + ChunkReadDecision::Fallback(ChunkReadFallback { + stage: rustfs_io_metrics::IoStage::ReadSetup, + reason: rustfs_io_metrics::FallbackReason::NonLocalBackend, + }) + )); + } + + #[test] + fn plan_chunk_read_returns_range_guard_fallback_for_invalid_range() { + let info = ObjectInfo { + size: 16, + actual_size: 16, + ..Default::default() + }; + + let decision = plan_chunk_read( + &info, + true, + Some(HTTPRangeSpec { + is_suffix_length: false, + start: -1, + end: 4, + }), + None, + ) + .unwrap(); + + assert!(matches!( + decision, + ChunkReadDecision::Fallback(ChunkReadFallback { + stage: rustfs_io_metrics::IoStage::RangeGuard, + reason: rustfs_io_metrics::FallbackReason::RangeNotSupported, + }) + )); + } + + #[test] + fn plan_chunk_read_allows_suffix_range() { + let info = ObjectInfo { + size: 16, + actual_size: 16, + ..Default::default() + }; + + let decision = plan_chunk_read( + &info, + true, + Some(HTTPRangeSpec { + is_suffix_length: true, + start: 4, + end: -1, + }), + None, + ) + .unwrap(); + + let ChunkReadDecision::Eligible(plan) = decision else { + panic!("expected eligible plan"); + }; + let rs = plan.rs.expect("suffix range should be preserved"); + assert!(rs.is_suffix_length); + assert_eq!(rs.start, 4); + assert_eq!(plan.response_content_length, 4); + assert_eq!(plan.content_range.as_deref(), Some("bytes 12-15/16")); + } + + #[test] + fn plan_chunk_read_uses_part_number_range_when_available() { + let mut info = ObjectInfo { + size: 12, + actual_size: 12, + content_type: Some("application/octet-stream".to_string()), + ..Default::default() + }; + info.parts = vec![Default::default(), Default::default()]; + info.parts[0].number = 1; + info.parts[0].size = 5; + info.parts[0].actual_size = 5; + info.parts[1].number = 2; + info.parts[1].size = 7; + info.parts[1].actual_size = 7; + + let decision = plan_chunk_read(&info, true, None, Some(2)).unwrap(); + let ChunkReadDecision::Eligible(plan) = decision else { + panic!("expected eligible plan"); + }; + let rs = plan.rs.expect("range from part number"); + assert_eq!(rs.start, 5); + assert_eq!(rs.end, 11); + assert_eq!(plan.response_content_length, 7); + assert_eq!(plan.content_range.as_deref(), Some("bytes 5-11/12")); + assert!(plan.content_type.is_some()); + } + + #[test] + fn plan_chunk_read_returns_delete_marker_errors() { + let info = ObjectInfo { + delete_marker: true, + ..Default::default() + }; + + let err = plan_chunk_read(&info, true, None, None).unwrap_err(); + assert!(matches!(err, ChunkReadPlanError::NoSuchKey)); + + let err = plan_chunk_read(&info, false, None, None).unwrap_err(); + assert!(matches!(err, ChunkReadPlanError::MethodNotAllowed)); + } + + #[test] + fn plan_legacy_read_uses_part_number_range_when_available() { + let mut info = ObjectInfo { + size: 12, + actual_size: 12, + content_type: Some("application/octet-stream".to_string()), + ..Default::default() + }; + info.parts = vec![Default::default(), Default::default()]; + info.parts[0].number = 1; + info.parts[0].size = 5; + info.parts[0].actual_size = 5; + info.parts[1].number = 2; + info.parts[1].size = 7; + info.parts[1].actual_size = 7; + + let plan = plan_legacy_read(&info, None, Some(2)).unwrap(); + + let rs = plan.rs.expect("range from part number"); + assert_eq!(rs.start, 5); + assert_eq!(rs.end, 11); + assert_eq!(plan.response_content_length, 7); + assert_eq!(plan.content_range.as_deref(), Some("bytes 5-11/12")); + assert!(plan.content_type.is_some()); + } + + #[test] + fn build_reader_read_setup_uses_encryption_length_override() { + let plan = LegacyReadPlan { + rs: None, + content_type: None, + last_modified: None, + response_content_length: 16, + content_range: None, + }; + let encryption_state = GetObjectEncryptionState { + encryption_applied: true, + response_content_length_override: Some(12), + ..Default::default() + }; + let reader = Box::new(rustfs_rio::WarpReader::new(tokio::io::empty())) as Box; + + let setup = build_reader_read_setup(ObjectInfo::default(), ObjectInfo::default(), reader, plan, encryption_state); + + assert!(setup.encryption_applied); + assert_eq!(setup.response_content_length, 12); + match setup.body_source { + GetObjectBodySource::Reader(_) => {} + GetObjectBodySource::Chunk { .. } => panic!("expected reader body source"), + } + } + + #[test] + fn plan_get_object_body_prefers_cache_writeback_when_cacheable() { + let plan = plan_get_object_body( + GetObjectCacheEligibility { + cache_enabled: true, + cache_writeback_enabled: true, + is_part_request: false, + is_range_request: false, + encryption_applied: false, + response_size: 1024, + max_cacheable_size: 2048, + }, + 4096, + ); + + assert_eq!(plan, GetObjectBodyPlan::CacheWriteback); + } + + #[test] + fn cache_served_metric_contract_is_mutually_exclusive_with_cache_writeback() { + let contract = GetObjectDataPlaneMetricContract::cache_served(); + + assert_eq!(contract.request_source, GetObjectDataPlaneRequestSource::CacheServed); + assert_eq!(contract.io_path, rustfs_io_metrics::IoPath::Fast); + assert_eq!(contract.copy_mode, rustfs_io_metrics::CopyMode::SharedBytes); + assert!(contract.record_cache_served_metric); + assert!(!contract.record_cache_writeback_metric); + } + + #[test] + fn disk_metric_contract_can_mark_cache_writeback_without_reclassifying_request_source() { + let contract = GetObjectDataPlaneMetricContract::disk( + rustfs_io_metrics::IoPath::Legacy, + rustfs_io_metrics::CopyMode::SingleCopy, + GetObjectBodyPlan::CacheWriteback, + ); + + assert_eq!(contract.request_source, GetObjectDataPlaneRequestSource::Disk); + assert_eq!(contract.io_path, rustfs_io_metrics::IoPath::Legacy); + assert_eq!(contract.copy_mode, rustfs_io_metrics::CopyMode::SingleCopy); + assert!(!contract.record_cache_served_metric); + assert!(contract.record_cache_writeback_metric); + } + + #[test] + fn plan_get_object_body_uses_encrypted_buffer_for_small_plain_request() { + let plan = plan_get_object_body( + GetObjectCacheEligibility { + cache_enabled: false, + cache_writeback_enabled: false, + is_part_request: false, + is_range_request: false, + encryption_applied: true, + response_size: 1024, + max_cacheable_size: 0, + }, + 4096, + ); + + assert_eq!(plan, GetObjectBodyPlan::BufferEncrypted); + } + + #[test] + fn get_object_sequential_hint_distinguishes_prefix_and_suffix_ranges() { + assert!(get_object_sequential_hint(None)); + assert!(get_object_sequential_hint(Some(&HTTPRangeSpec { + is_suffix_length: false, + start: 0, + end: -1, + }))); + assert!(!get_object_sequential_hint(Some(&HTTPRangeSpec { + is_suffix_length: false, + start: 4, + end: 8, + }))); + assert!(!get_object_sequential_hint(Some(&HTTPRangeSpec { + is_suffix_length: true, + start: 4, + end: -1, + }))); + } + + #[test] + fn plan_get_object_strategy_layout_caps_buffer_to_fallback() { + let layout = plan_get_object_strategy_layout(None, 1024, 8192, 4096); + + assert!(layout.is_sequential_hint); + assert_eq!(layout.optimal_buffer_size, 4096); + } + + #[test] + fn build_get_object_cache_writeback_formats_metadata() { + let info = ObjectInfo { + content_type: Some("application/octet-stream".to_string()), + etag: Some("abc123".to_string()), + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + checksum: rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::CRC32, b"abc") + .map(|checksum| checksum.to_bytes(&[])), + ..Default::default() + }; + + let writeback = build_get_object_cache_writeback(&info, Bytes::from_static(b"abc"), 3); + + assert_eq!(*writeback.body, Bytes::from_static(b"abc")); + assert_eq!(writeback.content_length, 3); + assert_eq!(writeback.content_type.as_deref(), Some("application/octet-stream")); + assert_eq!(writeback.e_tag.as_deref(), Some("abc123")); + assert_eq!(writeback.last_modified.as_deref(), Some("1970-01-01T00:00:00Z")); + assert_eq!(writeback.checksum_crc32.as_deref(), Some("NSRBwg==")); + } + + #[test] + fn finalize_get_object_cache_writeback_applies_http_metadata_and_user_metadata() { + let mut info = ObjectInfo { + expires: Some(OffsetDateTime::UNIX_EPOCH), + ..Default::default() + }; + info.user_defined + .insert("cache-control".to_string(), "max-age=3600".to_string()); + info.user_defined + .insert("content-disposition".to_string(), "attachment".to_string()); + info.user_defined.insert("content-language".to_string(), "en-US".to_string()); + + let writeback = finalize_get_object_cache_writeback( + &info, + GetObjectCacheWriteback { + body: Arc::new(Bytes::from_static(b"abc")), + content_length: 3, + content_type: None, + content_encoding: None, + cache_control: None, + content_disposition: None, + content_language: None, + expires: None, + storage_class: None, + version_id: None, + delete_marker: false, + user_metadata: HashMap::new(), + e_tag: None, + last_modified: None, + checksum_crc32: None, + checksum_crc32c: None, + checksum_sha1: None, + checksum_sha256: None, + checksum_crc64nvme: None, + checksum_type: None, + }, + HashMap::from([(String::from("custom"), String::from("value"))]), + ); + + assert_eq!(writeback.cache_control.as_deref(), Some("max-age=3600")); + assert_eq!(writeback.content_disposition.as_deref(), Some("attachment")); + assert_eq!(writeback.content_language.as_deref(), Some("en-US")); + assert_eq!(writeback.expires.as_deref(), Some("1970-01-01T00:00:00Z")); + assert_eq!(writeback.user_metadata.get("custom").map(String::as_str), Some("value")); + } + + #[test] + fn frozen_get_object_body_reuses_same_shared_bytes_for_cache_writeback() { + let frozen = FrozenGetObjectBody::new(Bytes::from_static(b"abc")); + let shared = Arc::clone(frozen.shared_body()); + assert_eq!(*shared, Bytes::from_static(b"abc")); + assert!(Arc::ptr_eq(&shared, frozen.shared_body())); + } + + #[test] + fn build_output_version_id_maps_nil_uuid_to_null() { + let nil = uuid::Uuid::nil(); + let version_id = build_output_version_id(true, Some(&nil)); + + assert_eq!(version_id.as_deref(), Some("null")); + } + + #[test] + fn build_get_object_output_context_preserves_copy_mode_override() { + let info = ObjectInfo { + version_id: Some(uuid::Uuid::nil()), + ..Default::default() + }; + let output_context = build_get_object_output_context( + None, + info, + ObjectInfo::default(), + None, + None, + 8, + None, + None, + None, + None, + None, + &GetObjectChecksums::default(), + None, + true, + 4096, + Some(rustfs_io_metrics::CopyMode::Reconstructed), + ); + + assert_eq!(output_context.output.version_id.as_deref(), Some("null")); + assert_eq!(output_context.copy_mode_override, Some(rustfs_io_metrics::CopyMode::Reconstructed)); + assert_eq!(output_context.optimal_buffer_size, 4096); + } + + #[test] + fn build_cached_get_object_flow_result_from_source_builds_plain_mode() { + let result = build_cached_get_object_flow_result_from_source( + "bucket", + "key", + &MockCachedSource { + body: Arc::new(Bytes::from_static(b"abc")), + content_length: 3, + content_type: None, + e_tag: None, + last_modified: None, + cache_control: None, + content_disposition: None, + content_encoding: None, + content_language: None, + storage_class: None, + version_id: None, + delete_marker: false, + tag_count: None, + user_metadata: HashMap::new(), + checksum_crc32: Some("crc32".to_string()), + checksum_crc32c: None, + checksum_sha1: None, + checksum_sha256: None, + checksum_crc64nvme: None, + checksum_type: Some(ChecksumType::from_static(ChecksumType::FULL_OBJECT)), + }, + "vid".to_string(), + ); + + assert!(matches!(result.response_mode, GetObjectResponseMode::Plain)); + assert_eq!(result.version_id_for_event, "vid"); + assert_eq!(result.event_info.bucket, "bucket"); + assert_eq!(result.event_info.name, "key"); + assert_eq!(result.output.checksum_crc32.as_deref(), Some("crc32")); + assert_eq!(result.output.checksum_type, Some(ChecksumType::from_static(ChecksumType::FULL_OBJECT))); + } + + #[test] + fn build_cors_wrapped_get_object_flow_result_uses_wrapped_mode() { + let result = build_cors_wrapped_get_object_flow_result( + GetObjectOutputContext { + output: GetObjectOutput::default(), + event_info: ObjectInfo::default(), + response_content_length: 1, + optimal_buffer_size: 1024, + copy_mode_override: None, + }, + "vid".to_string(), + ); + + assert!(matches!(result.response_mode, GetObjectResponseMode::CorsWrapped)); + assert_eq!(result.version_id_for_event, "vid"); + } + + #[test] + fn finalize_chunk_read_setup_preserves_body_source_and_io_path() { + let chunk_result = GetObjectChunkResult { + stream: Box::pin(futures_util::stream::empty::>()), + path: GetObjectChunkPath::Direct, + copy_mode: GetObjectChunkCopyMode::Reconstructed, + }; + let plan = ChunkReadPlan { + rs: Some(HTTPRangeSpec { + is_suffix_length: false, + start: 0, + end: 7, + }), + content_type: None, + last_modified: None, + response_content_length: 8, + content_range: Some("bytes 0-7/8".to_string()), + }; + + let result = finalize_chunk_read_setup(ObjectInfo::default(), ObjectInfo::default(), chunk_result, plan); + + assert_eq!(result.io_path, rustfs_io_metrics::IoPath::Fast); + match result.read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert_eq!(path, GetObjectChunkPath::Direct); + assert_eq!(copy_mode, rustfs_io_metrics::CopyMode::Reconstructed); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + assert_eq!(result.read_setup.response_content_length, 8); + assert_eq!(result.read_setup.content_range.as_deref(), Some("bytes 0-7/8")); + } +} diff --git a/crates/object-io/src/lib.rs b/crates/object-io/src/lib.rs new file mode 100644 index 000000000..9de88dbe3 --- /dev/null +++ b/crates/object-io/src/lib.rs @@ -0,0 +1,16 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod get; +pub mod put; diff --git a/crates/object-io/src/put.rs b/crates/object-io/src/put.rs new file mode 100644 index 000000000..7085a6468 --- /dev/null +++ b/crates/object-io/src/put.rs @@ -0,0 +1,1564 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytes::Buf; +use futures_util::{Stream, StreamExt}; +use http::HeaderMap; +use rustfs_ecstore::compress::{MIN_COMPRESSIBLE_SIZE, is_compressible}; +use rustfs_ecstore::store_api::ObjectOptions; +use rustfs_rio::{ + BlockReadable, BoxReadBlockFuture, Checksum, EtagResolvable, HashReader, HashReaderDetector, Reader, TryGetIndex, WarpReader, +}; +use rustfs_utils::http::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM; +use rustfs_utils::http::headers::{ + AMZ_DECODED_CONTENT_LENGTH, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, AMZ_MINIO_SNOWBALL_PREFIX, + AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, AMZ_RUSTFS_SNOWBALL_PREFIX, AMZ_SERVER_SIDE_ENCRYPTION, + AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_SNOWBALL_EXTRACT, AMZ_SNOWBALL_IGNORE_DIRS, AMZ_SNOWBALL_IGNORE_ERRORS, + AMZ_SNOWBALL_PREFIX, +}; +use s3s::dto::{ChecksumAlgorithm, PutObjectInput, ServerSideEncryption}; +use s3s::{S3Error, s3_error}; +use std::collections::HashMap; +use std::pin::Pin; +use tokio::io::AsyncRead; +use tokio_tar::Archive; +use tokio_util::io::StreamReader; + +pub const AMZ_SNOWBALL_EXTRACT_COMPAT: &str = "X-Amz-Snowball-Auto-Extract"; +pub const AMZ_SNOWBALL_PREFIX_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix"; +pub const AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Dirs"; +pub const AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Errors"; + +const AMZ_META_PREFIX_LOWER: &str = "x-amz-meta-"; +const SNOWBALL_PREFIX_SUFFIX_LOWER: &str = "snowball-prefix"; +const SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER: &str = "snowball-ignore-dirs"; +const SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER: &str = "snowball-ignore-errors"; +const SNOWBALL_PREFIX_HEADER_KEYS: &[&str] = &[AMZ_MINIO_SNOWBALL_PREFIX, AMZ_SNOWBALL_PREFIX, AMZ_RUSTFS_SNOWBALL_PREFIX]; +const SNOWBALL_IGNORE_DIRS_HEADER_KEYS: &[&str] = &[ + AMZ_MINIO_SNOWBALL_IGNORE_DIRS, + AMZ_SNOWBALL_IGNORE_DIRS, + AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, +]; +const SNOWBALL_IGNORE_ERRORS_HEADER_KEYS: &[&str] = &[ + AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, + AMZ_SNOWBALL_IGNORE_ERRORS, + AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, +]; +pub const PUT_REDUCED_COPY_MIN_SIZE_BYTES: i64 = 1024 * 1024; + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PutObjectChecksums { + pub crc32: Option, + pub crc32c: Option, + pub sha1: Option, + pub sha256: Option, + pub crc64nvme: Option, +} + +impl PutObjectChecksums { + pub fn merge_from_map(&mut self, checksums: &HashMap) { + for (key, checksum) in checksums { + match rustfs_rio::ChecksumType::from_string(key.as_str()) { + rustfs_rio::ChecksumType::CRC32 => self.crc32 = Some(checksum.clone()), + rustfs_rio::ChecksumType::CRC32C => self.crc32c = Some(checksum.clone()), + rustfs_rio::ChecksumType::SHA1 => self.sha1 = Some(checksum.clone()), + rustfs_rio::ChecksumType::SHA256 => self.sha256 = Some(checksum.clone()), + rustfs_rio::ChecksumType::CRC64_NVME => self.crc64nvme = Some(checksum.clone()), + _ => {} + } + } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PutObjectTransformStage { + compression_applied: bool, + encryption_applied: bool, +} + +impl PutObjectTransformStage { + pub fn mark_compression(&mut self) { + self.compression_applied = true; + } + + pub fn mark_encryption(&mut self) { + self.encryption_applied = true; + } + + #[must_use] + pub const fn compression_applied(self) -> bool { + self.compression_applied + } + + #[must_use] + pub const fn encryption_applied(self) -> bool { + self.encryption_applied + } + + #[must_use] + pub fn effective_copy_mode(self) -> rustfs_io_metrics::CopyMode { + resolve_put_effective_copy_mode(self.compression_applied, self.encryption_applied) + } + + #[must_use] + pub fn metric_kind(self) -> Option<&'static str> { + resolve_put_transform_metric_kind(self.compression_applied, self.encryption_applied) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PutObjectCompatIngressKind { + BufferedStreamCompat, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PutObjectCompatIngressPlan { + pub kind: PutObjectCompatIngressKind, + pub buffer_size: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PutObjectIngressKind { + LegacyCompat, + ReducedCopyCandidate, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PutObjectIngressPlan { + pub kind: PutObjectIngressKind, + pub compat: PutObjectCompatIngressPlan, + pub enable_zero_copy: bool, +} + +pub type BoxIngressStream = Pin> + Send + Sync + 'static>>; +pub type PutObjectCompatIngressStream = futures_util::stream::Map) -> std::io::Result>; +pub type PutObjectCompatIngress = tokio::io::BufReader, B>>; + +pub fn box_put_object_ingress_stream(body: S) -> BoxIngressStream +where + S: Stream> + Send + Sync + 'static, +{ + Box::pin(body) +} + +pub struct PutObjectReducedCopyIngress { + body: BoxIngressStream, + compat: PutObjectCompatIngressPlan, +} + +impl PutObjectReducedCopyIngress { + pub const fn new(body: BoxIngressStream, compat: PutObjectCompatIngressPlan) -> Self { + Self { body, compat } + } + + pub fn from_stream(body: S, compat: PutObjectCompatIngressPlan) -> Self + where + S: Stream> + Send + Sync + 'static, + { + Self::new(box_put_object_ingress_stream(body), compat) + } + + pub const fn compat_plan(&self) -> PutObjectCompatIngressPlan { + self.compat + } + + pub fn into_body(self) -> BoxIngressStream { + self.body + } +} + +impl PutObjectReducedCopyIngress +where + B: Buf, + E: std::fmt::Display, + PutObjectCompatIngress, B, E>: Send + Sync + Unpin + 'static, +{ + pub fn into_compat_reader(self) -> Box { + build_put_object_compat_reader(self.body, self.compat) + } +} + +pub enum PutObjectIngressSource { + LegacyCompat(Box), + ReducedCopyCandidate(PutObjectReducedCopyIngress), +} + +struct PutObjectReducedCopyReader { + body: S, + current_chunk: Option, + pending_error: Option, +} + +impl PutObjectReducedCopyReader { + fn new(body: S) -> Self { + Self { + body, + current_chunk: None, + pending_error: None, + } + } + + fn copy_chunk_into_slice(chunk: &mut B, buf: &mut [u8]) -> usize + where + B: Buf, + { + let mut copied = 0; + let to_copy = chunk.remaining().min(buf.len()); + + while copied < to_copy { + let slice = chunk.chunk(); + if !slice.is_empty() { + let len = (to_copy - copied).min(slice.len()); + buf[copied..copied + len].copy_from_slice(&slice[..len]); + chunk.advance(len); + copied += len; + continue; + } + + let dest = &mut buf[copied..to_copy]; + chunk.copy_to_slice(dest); + copied = to_copy; + } + + copied + } + + fn copy_chunk_into_read_buf(chunk: &mut B, buf: &mut tokio::io::ReadBuf<'_>) -> usize + where + B: Buf, + { + let to_copy = chunk.remaining().min(buf.remaining()); + let dest = &mut buf.initialize_unfilled()[..to_copy]; + let copied = Self::copy_chunk_into_slice(chunk, dest); + buf.advance(copied); + copied + } + + async fn read_into_slice(&mut self, buf: &mut [u8]) -> std::io::Result + where + S: Stream> + Unpin, + B: Buf + Unpin, + E: std::fmt::Display, + { + if let Some(err) = self.pending_error.take() { + return Err(err); + } + + if buf.is_empty() { + return Ok(0); + } + + let mut copied = 0; + + loop { + if copied == buf.len() { + return Ok(copied); + } + + if let Some(chunk) = self.current_chunk.as_mut() { + if !chunk.has_remaining() { + self.current_chunk = None; + continue; + } + + copied += Self::copy_chunk_into_slice(chunk, &mut buf[copied..]); + if chunk.has_remaining() || copied == buf.len() { + return Ok(copied); + } + + self.current_chunk = None; + continue; + } + + match self.body.next().await { + Some(Ok(chunk)) => { + if chunk.remaining() == 0 { + continue; + } + self.current_chunk = Some(chunk); + } + Some(Err(err)) => { + let err = std::io::Error::other(err.to_string()); + if copied > 0 { + self.pending_error = Some(err); + return Ok(copied); + } + return Err(err); + } + None => return Ok(copied), + } + } + } +} + +impl AsyncRead for PutObjectReducedCopyReader +where + S: Stream> + Unpin, + B: Buf + Unpin, + E: std::fmt::Display, +{ + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + + if let Some(err) = this.pending_error.take() { + return std::task::Poll::Ready(Err(err)); + } + + let mut read_any = false; + + loop { + if buf.remaining() == 0 { + return std::task::Poll::Ready(Ok(())); + } + + if let Some(chunk) = this.current_chunk.as_mut() { + if !chunk.has_remaining() { + this.current_chunk = None; + continue; + } + + if Self::copy_chunk_into_read_buf(chunk, buf) > 0 { + read_any = true; + } + + if chunk.has_remaining() || buf.remaining() == 0 { + return std::task::Poll::Ready(Ok(())); + } + + this.current_chunk = None; + continue; + } + + match std::pin::Pin::new(&mut this.body).poll_next(cx) { + std::task::Poll::Ready(Some(Ok(chunk))) => { + if chunk.remaining() == 0 { + continue; + } + this.current_chunk = Some(chunk); + } + std::task::Poll::Ready(Some(Err(err))) => { + let err = std::io::Error::other(err.to_string()); + if read_any { + this.pending_error = Some(err); + return std::task::Poll::Ready(Ok(())); + } + return std::task::Poll::Ready(Err(err)); + } + std::task::Poll::Ready(None) => return std::task::Poll::Ready(Ok(())), + std::task::Poll::Pending => { + if read_any { + return std::task::Poll::Ready(Ok(())); + } + return std::task::Poll::Pending; + } + } + } + } +} + +impl BlockReadable for PutObjectReducedCopyReader +where + S: Stream> + Unpin + Send + Sync, + B: Buf + Unpin + Send + Sync, + E: std::fmt::Display + Send + Sync, +{ + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + Box::pin(async move { self.read_into_slice(buf).await }) + } +} + +impl EtagResolvable for PutObjectReducedCopyReader {} + +impl HashReaderDetector for PutObjectReducedCopyReader {} + +impl TryGetIndex for PutObjectReducedCopyReader {} + +fn build_put_object_reduced_copy_reader(candidate: PutObjectReducedCopyIngress) -> Box +where + B: Buf + Send + Sync + Unpin + 'static, + E: std::fmt::Display + Send + Sync + 'static, +{ + Box::new(PutObjectReducedCopyReader::new(candidate.into_body())) +} + +impl PutObjectIngressSource +where + B: Buf, + E: std::fmt::Display, + PutObjectCompatIngress, B, E>: Send + Sync + Unpin + 'static, +{ + pub fn into_compat_reader(self) -> Box { + match self { + Self::LegacyCompat(reader) => reader, + Self::ReducedCopyCandidate(candidate) => candidate.into_compat_reader(), + } + } + + pub fn into_reader(self) -> Box + where + B: Buf + Send + Sync + Unpin + 'static, + E: std::fmt::Display + Send + Sync + 'static, + { + match self { + Self::LegacyCompat(reader) => reader, + Self::ReducedCopyCandidate(candidate) => build_put_object_reduced_copy_reader(candidate), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PutObjectPlainBodyKind { + LegacyCompat, + ReducedCopyCandidate, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PutObjectBodyKind { + Plain(PutObjectPlainBodyKind), + Compressed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PutObjectBodyPlan { + pub ingress: PutObjectIngressPlan, + pub kind: PutObjectBodyKind, +} + +impl PutObjectBodyPlan { + pub const fn should_compress(&self) -> bool { + matches!(self.kind, PutObjectBodyKind::Compressed) + } + + pub const fn plain_body_kind(&self) -> Option { + match self.kind { + PutObjectBodyKind::Plain(kind) => Some(kind), + PutObjectBodyKind::Compressed => None, + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PutObjectExtractOptions { + pub prefix: Option, + pub ignore_dirs: bool, + pub ignore_errors: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PutObjectLegacyHashValues { + pub md5hex: Option, + pub sha256hex: Option, +} + +impl PutObjectLegacyHashValues { + pub fn clear_for_transformed_body(&mut self) { + self.md5hex = None; + self.sha256hex = None; + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PutObjectLegacyHashStagePlan { + pub size: i64, + pub actual_size: i64, + pub apply_s3_checksum: bool, + pub ignore_s3_checksum_value: bool, +} + +pub struct PutObjectHashStage { + pub reader: HashReader, + pub want_checksum: Option, + pub ingress_kind: PutObjectIngressKind, +} + +fn build_put_object_hash_stage( + reader: Box, + ingress_kind: PutObjectIngressKind, + hash_values: PutObjectLegacyHashValues, + plan: PutObjectLegacyHashStagePlan, + headers: &HeaderMap, + trailing_headers: Option, +) -> std::io::Result { + let mut reader = HashReader::new(reader, plan.size, plan.actual_size, hash_values.md5hex, hash_values.sha256hex, false)?; + let requested_checksum_type = rustfs_rio::ChecksumType::from_header(headers); + let want_checksum = if plan.apply_s3_checksum { + reader.add_checksum_from_s3s(headers, trailing_headers, plan.ignore_s3_checksum_value)?; + if requested_checksum_type.is_set() && reader.checksum().is_none() { + reader.enable_auto_checksum(requested_checksum_type)?; + } + reader.checksum() + } else { + None + }; + + Ok(PutObjectHashStage { + reader, + want_checksum, + ingress_kind, + }) +} + +pub fn apply_trailing_checksums( + algorithm: Option<&str>, + trailing_headers: &Option, + checksums: &mut PutObjectChecksums, +) { + let Some(alg) = algorithm else { return }; + let Some(checksum_str) = trailing_headers.as_ref().and_then(|trailer| { + let key = match alg { + ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(), + ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(), + ChecksumAlgorithm::SHA1 => rustfs_rio::ChecksumType::SHA1.key(), + ChecksumAlgorithm::SHA256 => rustfs_rio::ChecksumType::SHA256.key(), + ChecksumAlgorithm::CRC64NVME => rustfs_rio::ChecksumType::CRC64_NVME.key(), + _ => return None, + }; + trailer.read(|headers| { + headers + .get(key.unwrap_or_default()) + .and_then(|value| value.to_str().ok().map(|s| s.to_string())) + }) + }) else { + return; + }; + + match alg { + ChecksumAlgorithm::CRC32 => checksums.crc32 = checksum_str, + ChecksumAlgorithm::CRC32C => checksums.crc32c = checksum_str, + ChecksumAlgorithm::SHA1 => checksums.sha1 = checksum_str, + ChecksumAlgorithm::SHA256 => checksums.sha256 = checksum_str, + ChecksumAlgorithm::CRC64NVME => checksums.crc64nvme = checksum_str, + _ => (), + } +} + +pub fn resolve_put_body_size(content_length: Option, headers: &HeaderMap) -> s3s::S3Result { + let size = match content_length { + Some(c) => c, + None => { + if let Some(val) = headers.get(AMZ_DECODED_CONTENT_LENGTH) { + match atoi::atoi::(val.as_bytes()) { + Some(x) => x, + None => return Err(s3_error!(UnexpectedContent)), + } + } else { + return Err(s3_error!(UnexpectedContent)); + } + } + }; + + if size == -1 { + return Err(s3_error!(UnexpectedContent)); + } + + Ok(size) +} + +pub fn should_use_zero_copy(size: i64, headers: &HeaderMap) -> bool { + const ZERO_COPY_MIN_SIZE: i64 = 1024 * 1024; + + if size <= ZERO_COPY_MIN_SIZE { + return false; + } + + !has_put_encryption_headers(headers) && !put_request_is_compressible(headers) +} + +fn has_put_encryption_headers(headers: &HeaderMap) -> bool { + headers.get(AMZ_SERVER_SIDE_ENCRYPTION).is_some() + || headers.get(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM).is_some() + || headers.get(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID).is_some() +} + +fn put_request_is_compressible(headers: &HeaderMap) -> bool { + if let Some(content_type) = headers.get("content-type") + && let Ok(ct) = content_type.to_str() + { + let compressible_types = [ + "text/plain", + "text/html", + "text/css", + "text/javascript", + "application/javascript", + "application/json", + "application/xml", + "text/xml", + ]; + return compressible_types.iter().any(|ty| ct.contains(ty)); + } + + false +} + +fn should_use_put_reduced_copy_candidate( + size: i64, + headers: &HeaderMap, + encryption_enabled: bool, + compression_enabled: bool, +) -> bool { + if size <= PUT_REDUCED_COPY_MIN_SIZE_BYTES { + return false; + } + + if !encryption_enabled && has_put_encryption_headers(headers) { + return false; + } + + compression_enabled || !put_request_is_compressible(headers) +} + +fn map_put_object_ingress_error(result: Result) -> std::io::Result +where + E: std::fmt::Display, +{ + result.map_err(|err| std::io::Error::other(err.to_string())) +} + +pub fn build_put_object_compat_ingress(body: S, plan: PutObjectCompatIngressPlan) -> PutObjectCompatIngress +where + S: Stream>, + B: Buf, + E: std::fmt::Display, +{ + match plan.kind { + PutObjectCompatIngressKind::BufferedStreamCompat => tokio::io::BufReader::with_capacity( + plan.buffer_size, + StreamReader::new(body.map(map_put_object_ingress_error:: as fn(Result) -> std::io::Result)), + ), + } +} + +pub fn build_put_object_compat_reader(body: S, plan: PutObjectCompatIngressPlan) -> Box +where + S: Stream>, + B: Buf, + E: std::fmt::Display, + PutObjectCompatIngress: Send + Sync + Unpin + 'static, +{ + Box::new(WarpReader::new(build_put_object_compat_ingress(body, plan))) +} + +pub fn build_put_object_ingress_source(body: S, plan: PutObjectBodyPlan) -> PutObjectIngressSource +where + S: Stream> + Send + Sync + 'static, + B: Buf + 'static, + E: std::fmt::Display + 'static, + PutObjectCompatIngress, B, E>: Send + Sync + Unpin + 'static, +{ + match (plan.kind, plan.ingress.kind) { + (PutObjectBodyKind::Compressed, PutObjectIngressKind::ReducedCopyCandidate) + | (PutObjectBodyKind::Plain(PutObjectPlainBodyKind::ReducedCopyCandidate), PutObjectIngressKind::ReducedCopyCandidate) => { + PutObjectIngressSource::ReducedCopyCandidate(PutObjectReducedCopyIngress::from_stream(body, plan.ingress.compat)) + } + (PutObjectBodyKind::Compressed, _) + | (PutObjectBodyKind::Plain(PutObjectPlainBodyKind::LegacyCompat), _) + | (PutObjectBodyKind::Plain(PutObjectPlainBodyKind::ReducedCopyCandidate), PutObjectIngressKind::LegacyCompat) => { + PutObjectIngressSource::LegacyCompat(build_put_object_compat_reader( + box_put_object_ingress_stream(body), + plan.ingress.compat, + )) + } + } +} + +pub fn build_put_object_legacy_hash_stage( + reader: Box, + hash_values: PutObjectLegacyHashValues, + plan: PutObjectLegacyHashStagePlan, + headers: &HeaderMap, + trailing_headers: Option, +) -> std::io::Result { + build_put_object_hash_stage(reader, PutObjectIngressKind::LegacyCompat, hash_values, plan, headers, trailing_headers) +} + +pub fn build_put_object_plain_hash_stage( + ingress: PutObjectIngressSource, + hash_values: PutObjectLegacyHashValues, + plan: PutObjectLegacyHashStagePlan, + headers: &HeaderMap, + trailing_headers: Option, +) -> std::io::Result +where + B: Buf + Send + Sync + Unpin + 'static, + E: std::fmt::Display + Send + Sync + 'static, +{ + match ingress { + PutObjectIngressSource::LegacyCompat(reader) => { + build_put_object_hash_stage(reader, PutObjectIngressKind::LegacyCompat, hash_values, plan, headers, trailing_headers) + } + PutObjectIngressSource::ReducedCopyCandidate(candidate) => build_put_object_hash_stage( + build_put_object_reduced_copy_reader(candidate), + PutObjectIngressKind::ReducedCopyCandidate, + hash_values, + plan, + headers, + trailing_headers, + ), + } +} + +pub fn plan_put_object_ingress(size: i64, headers: &HeaderMap, buffer_size: usize) -> PutObjectIngressPlan { + plan_put_object_ingress_with_transforms(size, headers, buffer_size, false, false) +} + +pub fn plan_put_object_ingress_with_transforms( + size: i64, + headers: &HeaderMap, + buffer_size: usize, + encryption_enabled: bool, + compression_enabled: bool, +) -> PutObjectIngressPlan { + let enable_zero_copy = should_use_put_reduced_copy_candidate(size, headers, encryption_enabled, compression_enabled); + PutObjectIngressPlan { + kind: if enable_zero_copy { + PutObjectIngressKind::ReducedCopyCandidate + } else { + PutObjectIngressKind::LegacyCompat + }, + compat: PutObjectCompatIngressPlan { + kind: PutObjectCompatIngressKind::BufferedStreamCompat, + buffer_size, + }, + enable_zero_copy, + } +} + +pub fn plan_put_object_body(size: i64, headers: &HeaderMap, key: &str, buffer_size: usize) -> PutObjectBodyPlan { + plan_put_object_body_with_transforms(size, headers, key, buffer_size, false) +} + +pub fn plan_put_object_body_with_transforms( + size: i64, + headers: &HeaderMap, + key: &str, + buffer_size: usize, + encryption_enabled: bool, +) -> PutObjectBodyPlan { + let compression_enabled = size > MIN_COMPRESSIBLE_SIZE as i64 && is_compressible(headers, key); + let ingress = plan_put_object_ingress_with_transforms(size, headers, buffer_size, encryption_enabled, compression_enabled); + let kind = if compression_enabled { + PutObjectBodyKind::Compressed + } else { + PutObjectBodyKind::Plain(match ingress.kind { + PutObjectIngressKind::LegacyCompat => PutObjectPlainBodyKind::LegacyCompat, + PutObjectIngressKind::ReducedCopyCandidate => PutObjectPlainBodyKind::ReducedCopyCandidate, + }) + }; + + PutObjectBodyPlan { ingress, kind } +} + +pub fn resolve_put_effective_copy_mode(applied_compression: bool, applied_encryption: bool) -> rustfs_io_metrics::CopyMode { + if applied_compression || applied_encryption { + rustfs_io_metrics::CopyMode::Transformed + } else { + rustfs_io_metrics::CopyMode::SingleCopy + } +} + +pub fn resolve_put_transform_metric_kind(applied_compression: bool, applied_encryption: bool) -> Option<&'static str> { + match (applied_compression, applied_encryption) { + (true, true) => Some("compression_encryption"), + (true, false) => Some("compression"), + (false, true) => Some("encryption"), + (false, false) => None, + } +} + +pub fn resolve_put_transformed_fallback_reason( + ingress_kind: PutObjectIngressKind, + compressed: bool, + encryption_enabled: bool, +) -> Option { + if ingress_kind == PutObjectIngressKind::ReducedCopyCandidate { + return None; + } + + match (compressed, encryption_enabled) { + (true, true) => Some(rustfs_io_metrics::FallbackReason::TransformCompressionEncryptionLegacy), + (true, false) => Some(rustfs_io_metrics::FallbackReason::TransformCompressionLegacy), + (false, true) => Some(rustfs_io_metrics::FallbackReason::TransformEncryptionLegacy), + (false, false) => None, + } +} + +pub fn header_value_is_true(headers: &HeaderMap, key: &str) -> bool { + headers + .get(key) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case("true")) +} + +pub fn is_put_object_extract_requested(headers: &HeaderMap) -> bool { + header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_COMPAT) +} + +fn trimmed_header_value(headers: &HeaderMap, key: &str) -> Option { + headers + .get(key) + .and_then(|value| value.to_str().ok()) + .map(|value| value.trim().to_string()) +} + +fn is_exact_snowball_meta_key(key: &str, exact_keys: &[&str]) -> bool { + exact_keys.iter().any(|exact_key| key.eq_ignore_ascii_case(exact_key)) +} + +fn snowball_meta_value_by_suffix(headers: &HeaderMap, suffix_lower: &str, exact_keys: &[&str]) -> Option { + for (name, value) in headers { + let key = name.as_str(); + if key.starts_with(AMZ_META_PREFIX_LOWER) + && key.ends_with(suffix_lower) + && !is_exact_snowball_meta_key(key, exact_keys) + && let Ok(parsed) = value.to_str() + { + return Some(parsed.trim().to_string()); + } + } + + None +} + +fn snowball_meta_value(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> Option { + for key in exact_keys { + if let Some(value) = trimmed_header_value(headers, key) { + return Some(value); + } + } + + snowball_meta_value_by_suffix(headers, suffix_lower, exact_keys) +} + +fn snowball_meta_flag(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> bool { + snowball_meta_value(headers, exact_keys, suffix_lower).is_some_and(|value| value.eq_ignore_ascii_case("true")) +} + +pub fn normalize_snowball_prefix(prefix: &str) -> Option { + let normalized = prefix.trim().trim_matches('/'); + if normalized.is_empty() { + return None; + } + + Some(normalized.to_string()) +} + +pub fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: bool) -> String { + let path = path.trim_matches('/'); + let mut key = match prefix { + Some(prefix) if !path.is_empty() => format!("{prefix}/{path}"), + Some(prefix) => prefix.to_string(), + None => path.to_string(), + }; + + if is_dir && !key.ends_with('/') { + key.push('/'); + } + + key +} + +pub fn resolve_put_object_extract_options(headers: &HeaderMap) -> PutObjectExtractOptions { + let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER) + .and_then(|value| normalize_snowball_prefix(&value)); + let ignore_dirs = snowball_meta_flag(headers, SNOWBALL_IGNORE_DIRS_HEADER_KEYS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER); + let ignore_errors = snowball_meta_flag(headers, SNOWBALL_IGNORE_ERRORS_HEADER_KEYS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER); + + PutObjectExtractOptions { + prefix, + ignore_dirs, + ignore_errors, + } +} + +pub fn map_extract_archive_error(err: impl std::fmt::Display) -> S3Error { + s3_error!(InvalidArgument, "Failed to process archive entry: {}", err) +} + +pub async fn apply_extract_entry_pax_extensions( + entry: &mut tokio_tar::Entry>, + metadata: &mut HashMap, + opts: &mut ObjectOptions, +) -> s3s::S3Result<()> +where + R: AsyncRead + Send + Unpin + 'static, +{ + let Some(extensions) = entry.pax_extensions().await.map_err(map_extract_archive_error)? else { + return Ok(()); + }; + + for ext in extensions { + let ext = ext.map_err(map_extract_archive_error)?; + let key = ext.key().map_err(map_extract_archive_error)?; + let value = ext.value().map_err(map_extract_archive_error)?; + + if let Some(meta_key) = key.strip_prefix("minio.metadata.") { + let meta_key = meta_key.strip_prefix("x-amz-meta-").unwrap_or(meta_key); + if !meta_key.is_empty() { + metadata.insert(meta_key.to_string(), value.to_string()); + } + continue; + } + + if key == "minio.versionId" && !value.is_empty() { + opts.version_id = Some(value.to_string()); + } + } + + Ok(()) +} + +pub fn is_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool { + input + .server_side_encryption + .as_ref() + .is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) + || input.ssekms_key_id.is_some() + || headers + .get(AMZ_SERVER_SIDE_ENCRYPTION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) + || headers.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID) +} + +pub fn is_post_object_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool { + is_sse_kms_requested(input, headers) +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use http::{HeaderMap, HeaderName, HeaderValue}; + use std::io::Cursor; + use tokio::io::AsyncReadExt; + + #[test] + fn should_use_zero_copy_accepts_large_unencrypted_binary_payload() { + let mut headers = HeaderMap::new(); + headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); + assert!(should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_small_payloads() { + assert!(!should_use_zero_copy(512 * 1024, &HeaderMap::new())); + } + + #[test] + fn resolve_put_body_size_uses_content_length_when_present() { + assert_eq!(resolve_put_body_size(Some(123), &HeaderMap::new()).unwrap(), 123); + } + + #[test] + fn resolve_put_body_size_uses_decoded_content_length_header() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_DECODED_CONTENT_LENGTH, "456".parse().unwrap()); + assert_eq!(resolve_put_body_size(None, &headers).unwrap(), 456); + } + + #[test] + fn should_use_zero_copy_rejects_encrypted_payloads() { + let mut headers = HeaderMap::new(); + headers.insert("x-amz-server-side-encryption", HeaderValue::from_static("AES256")); + assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_compressible_payloads() { + let mut headers = HeaderMap::new(); + headers.insert("content-type", HeaderValue::from_static("application/json")); + assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_boundary_at_1mb() { + let headers = HeaderMap::new(); + + assert!(!should_use_zero_copy(1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_small_objects() { + let headers = HeaderMap::new(); + + assert!(!should_use_zero_copy(1024 * 1024 - 1, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_one_megabyte() { + let headers = HeaderMap::new(); + + assert!(!should_use_zero_copy(1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_encrypted_requests() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("AES256")); + + assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_encrypted_requests_with_sse_customer_algorithm() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, HeaderValue::from_static("AES256")); + + assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_encrypted_requests_with_kms_key_id() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, HeaderValue::from_static("test-kms-key-id")); + + assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_rejects_compressible_content_types() { + let mut headers = HeaderMap::new(); + headers.insert( + rustfs_utils::http::CONTENT_TYPE, + HeaderValue::from_static("application/json; charset=utf-8"), + ); + + assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn should_use_zero_copy_allows_large_unencrypted_binary_objects() { + let mut headers = HeaderMap::new(); + headers.insert(rustfs_utils::http::CONTENT_TYPE, HeaderValue::from_static("application/octet-stream")); + + assert!(should_use_zero_copy(2 * 1024 * 1024, &headers)); + } + + #[test] + fn plan_put_object_ingress_preserves_buffer_size_and_fast_path_decision() { + let mut headers = HeaderMap::new(); + headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); + + let plan = plan_put_object_ingress(2 * 1024 * 1024, &headers, 256 * 1024); + + assert_eq!(plan.kind, PutObjectIngressKind::ReducedCopyCandidate); + assert_eq!(plan.compat.kind, PutObjectCompatIngressKind::BufferedStreamCompat); + assert_eq!(plan.compat.buffer_size, 256 * 1024); + assert!(plan.enable_zero_copy); + } + + #[test] + fn plan_put_object_body_disables_compression_for_small_payloads() { + let plan = plan_put_object_body(1024, &HeaderMap::new(), "small.bin", 64 * 1024); + + assert_eq!(plan.ingress.kind, PutObjectIngressKind::LegacyCompat); + assert_eq!(plan.ingress.compat.buffer_size, 64 * 1024); + assert!(!plan.ingress.enable_zero_copy); + assert_eq!(plan.kind, PutObjectBodyKind::Plain(PutObjectPlainBodyKind::LegacyCompat)); + assert!(!plan.should_compress()); + } + + #[test] + fn plan_put_object_body_marks_large_plain_payload_as_reduced_copy_candidate() { + let mut headers = HeaderMap::new(); + headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); + + let plan = plan_put_object_body(2 * 1024 * 1024, &headers, "large.bin", 256 * 1024); + + assert_eq!(plan.kind, PutObjectBodyKind::Plain(PutObjectPlainBodyKind::ReducedCopyCandidate)); + assert_eq!(plan.plain_body_kind(), Some(PutObjectPlainBodyKind::ReducedCopyCandidate)); + assert!(!plan.should_compress()); + } + + #[test] + fn plan_put_object_body_with_transforms_allows_encrypted_large_binary_payloads() { + let mut headers = HeaderMap::new(); + headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); + headers.insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("AES256")); + + let plan = plan_put_object_body_with_transforms(2 * 1024 * 1024, &headers, "large.bin", 256 * 1024, true); + + assert_eq!(plan.kind, PutObjectBodyKind::Plain(PutObjectPlainBodyKind::ReducedCopyCandidate)); + assert_eq!(plan.plain_body_kind(), Some(PutObjectPlainBodyKind::ReducedCopyCandidate)); + assert!(plan.ingress.enable_zero_copy); + } + + #[test] + fn build_put_object_ingress_source_preserves_reduced_copy_candidate_for_compressed_body() { + let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"compressed"))]); + let plan = PutObjectBodyPlan { + ingress: PutObjectIngressPlan { + kind: PutObjectIngressKind::ReducedCopyCandidate, + compat: PutObjectCompatIngressPlan { + kind: PutObjectCompatIngressKind::BufferedStreamCompat, + buffer_size: 256 * 1024, + }, + enable_zero_copy: true, + }, + kind: PutObjectBodyKind::Compressed, + }; + + let source = build_put_object_ingress_source(stream, plan); + assert!(matches!(source, PutObjectIngressSource::ReducedCopyCandidate(_))); + } + + #[test] + fn put_object_body_plan_reports_compressed_kind() { + let plan = PutObjectBodyPlan { + ingress: PutObjectIngressPlan { + kind: PutObjectIngressKind::LegacyCompat, + compat: PutObjectCompatIngressPlan { + kind: PutObjectCompatIngressKind::BufferedStreamCompat, + buffer_size: 256 * 1024, + }, + enable_zero_copy: false, + }, + kind: PutObjectBodyKind::Compressed, + }; + + assert_eq!(plan.plain_body_kind(), None); + assert!(plan.should_compress()); + } + + #[test] + fn resolve_put_effective_copy_mode_marks_transformed_paths() { + assert_eq!(resolve_put_effective_copy_mode(false, false), rustfs_io_metrics::CopyMode::SingleCopy); + assert_eq!(resolve_put_effective_copy_mode(true, false), rustfs_io_metrics::CopyMode::Transformed); + assert_eq!(resolve_put_effective_copy_mode(false, true), rustfs_io_metrics::CopyMode::Transformed); + } + + #[test] + fn put_object_transform_stage_tracks_transform_shape() { + let mut stage = PutObjectTransformStage::default(); + assert_eq!(stage.effective_copy_mode(), rustfs_io_metrics::CopyMode::SingleCopy); + assert_eq!(stage.metric_kind(), None); + + stage.mark_compression(); + assert!(stage.compression_applied()); + assert_eq!(stage.effective_copy_mode(), rustfs_io_metrics::CopyMode::Transformed); + assert_eq!(stage.metric_kind(), Some("compression")); + + stage.mark_encryption(); + assert!(stage.encryption_applied()); + assert_eq!(stage.metric_kind(), Some("compression_encryption")); + } + + #[test] + fn resolve_put_transform_metric_kind_reports_transform_shape() { + assert_eq!(resolve_put_transform_metric_kind(false, false), None); + assert_eq!(resolve_put_transform_metric_kind(true, false), Some("compression")); + assert_eq!(resolve_put_transform_metric_kind(false, true), Some("encryption")); + assert_eq!(resolve_put_transform_metric_kind(true, true), Some("compression_encryption")); + } + + #[test] + fn resolve_put_transformed_fallback_reason_isolated_from_plain_path() { + assert_eq!( + resolve_put_transformed_fallback_reason(PutObjectIngressKind::LegacyCompat, true, false), + Some(rustfs_io_metrics::FallbackReason::TransformCompressionLegacy) + ); + assert_eq!( + resolve_put_transformed_fallback_reason(PutObjectIngressKind::LegacyCompat, false, true), + Some(rustfs_io_metrics::FallbackReason::TransformEncryptionLegacy) + ); + assert_eq!( + resolve_put_transformed_fallback_reason(PutObjectIngressKind::LegacyCompat, true, true), + Some(rustfs_io_metrics::FallbackReason::TransformCompressionEncryptionLegacy) + ); + assert_eq!( + resolve_put_transformed_fallback_reason(PutObjectIngressKind::ReducedCopyCandidate, true, true), + None + ); + } + + #[test] + fn is_put_object_extract_requested_accepts_meta_header() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + assert!(is_put_object_extract_requested(&headers)); + } + + #[test] + fn is_put_object_extract_requested_accepts_compat_header_case_insensitive() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_EXTRACT_COMPAT, HeaderValue::from_static(" TRUE ")); + assert!(is_put_object_extract_requested(&headers)); + } + + #[test] + fn is_put_object_extract_requested_rejects_missing_or_false_value() { + let mut headers = HeaderMap::new(); + assert!(!is_put_object_extract_requested(&headers)); + headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("false")); + assert!(!is_put_object_extract_requested(&headers)); + } + + #[test] + fn normalize_snowball_prefix_trims_slashes_and_whitespace() { + assert_eq!(normalize_snowball_prefix(" /batch/incoming/ "), Some("batch/incoming".to_string())); + assert_eq!(normalize_snowball_prefix("///"), None); + } + + #[test] + fn normalize_extract_entry_key_applies_prefix_and_directory_suffix() { + assert_eq!( + normalize_extract_entry_key("nested/path.txt", Some("imports"), false), + "imports/nested/path.txt" + ); + assert_eq!(normalize_extract_entry_key("nested/dir/", Some("imports"), true), "imports/nested/dir/"); + assert_eq!(normalize_extract_entry_key("top-level", None, false), "top-level"); + } + + #[test] + fn resolve_put_object_extract_options_defaults_when_headers_missing() { + let headers = HeaderMap::new(); + let options = resolve_put_object_extract_options(&headers); + assert_eq!( + options, + PutObjectExtractOptions { + prefix: None, + ignore_dirs: false, + ignore_errors: false + } + ); + } + + #[test] + fn resolve_put_object_extract_options_accepts_internal_headers() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_PREFIX_INTERNAL, HeaderValue::from_static("/internal/prefix/")); + headers.insert(AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL, HeaderValue::from_static("true")); + headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL, HeaderValue::from_static("TRUE")); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("internal/prefix")); + assert!(options.ignore_dirs); + assert!(options.ignore_errors); + } + + #[test] + fn resolve_put_object_extract_options_accepts_standard_headers() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static(" /standard/prefix/ ")); + headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static(" true ")); + headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("TRUE")); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("standard/prefix")); + assert!(options.ignore_dirs); + assert!(options.ignore_errors); + } + + #[test] + fn resolve_put_object_extract_options_accepts_suffix_compatible_headers() { + let mut headers = HeaderMap::new(); + headers.insert( + HeaderName::from_static("x-amz-meta-acme-snowball-prefix"), + HeaderValue::from_static(" /partner/import "), + ); + headers.insert( + HeaderName::from_static("x-amz-meta-acme-snowball-ignore-dirs"), + HeaderValue::from_static(" true "), + ); + headers.insert( + HeaderName::from_static("x-amz-meta-acme-snowball-ignore-errors"), + HeaderValue::from_static("TRUE"), + ); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("partner/import")); + assert!(options.ignore_dirs); + assert!(options.ignore_errors); + } + + #[test] + fn resolve_put_object_extract_options_prefers_exact_headers_over_suffix_fallback() { + let mut headers = HeaderMap::new(); + headers.insert("x-amz-meta-acme-snowball-prefix", HeaderValue::from_static("/fallback/prefix/")); + headers.insert(AMZ_RUSTFS_SNOWBALL_PREFIX, HeaderValue::from_static("/internal/prefix/")); + headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static("/standard/prefix/")); + headers.insert(AMZ_MINIO_SNOWBALL_PREFIX, HeaderValue::from_static("/minio/prefix/")); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("minio/prefix")); + } + + #[test] + fn resolve_put_object_extract_options_exact_flags_override_suffix_fallback() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static("false")); + headers.insert("x-amz-meta-acme-snowball-ignore-dirs", HeaderValue::from_static("true")); + headers.insert(AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("false")); + headers.insert("x-amz-meta-acme-snowball-ignore-errors", HeaderValue::from_static("true")); + + let options = resolve_put_object_extract_options(&headers); + assert!(!options.ignore_dirs); + assert!(!options.ignore_errors); + } + + #[tokio::test] + async fn build_put_object_compat_ingress_reads_buffered_stream() { + let stream = futures_util::stream::iter([ + Ok::(Bytes::from_static(b"abc")), + Ok::(Bytes::from_static(b"def")), + ]); + let plan = PutObjectCompatIngressPlan { + kind: PutObjectCompatIngressKind::BufferedStreamCompat, + buffer_size: 8, + }; + + let mut reader = build_put_object_compat_ingress(stream, plan); + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await.unwrap(); + + assert_eq!(buf, b"abcdef"); + } + + #[tokio::test] + async fn build_put_object_compat_ingress_maps_stream_errors() { + let stream = futures_util::stream::iter([Err::("boom")]); + let plan = PutObjectCompatIngressPlan { + kind: PutObjectCompatIngressKind::BufferedStreamCompat, + buffer_size: 8, + }; + + let mut reader = build_put_object_compat_ingress(stream, plan); + let err = reader.read_to_end(&mut Vec::new()).await.unwrap_err(); + + assert_eq!(err.kind(), std::io::ErrorKind::Other); + assert!(err.to_string().contains("boom")); + } + + #[tokio::test] + async fn build_put_object_compat_reader_wraps_ingress_as_reader() { + let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"reader"))]); + let plan = PutObjectCompatIngressPlan { + kind: PutObjectCompatIngressKind::BufferedStreamCompat, + buffer_size: 16, + }; + + let mut reader = build_put_object_compat_reader(stream, plan); + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await.unwrap(); + + assert_eq!(buf, b"reader"); + } + + #[tokio::test] + async fn build_put_object_ingress_source_keeps_plain_candidate_as_reduced_copy_source() { + let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"candidate"))]); + let mut headers = HeaderMap::new(); + headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); + let plan = plan_put_object_body(2 * 1024 * 1024, &headers, "large.bin", 16); + + let source = build_put_object_ingress_source(stream, plan); + let candidate = match source { + PutObjectIngressSource::ReducedCopyCandidate(candidate) => candidate, + PutObjectIngressSource::LegacyCompat(_) => panic!("expected reduced-copy candidate"), + }; + + assert_eq!(candidate.compat_plan().kind, PutObjectCompatIngressKind::BufferedStreamCompat); + assert_eq!(candidate.compat_plan().buffer_size, 16); + + let mut reader = candidate.into_compat_reader(); + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await.unwrap(); + + assert_eq!(buf, b"candidate"); + } + + #[tokio::test] + async fn build_put_object_ingress_source_routes_compressed_body_to_legacy_compat_reader() { + let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"compressed"))]); + let mut headers = HeaderMap::new(); + headers.insert("content-type", HeaderValue::from_static("application/json")); + let plan = plan_put_object_body(2 * 1024 * 1024, &headers, "large.json", 32); + + let source = build_put_object_ingress_source(stream, plan); + let mut reader = match source { + PutObjectIngressSource::LegacyCompat(reader) => reader, + PutObjectIngressSource::ReducedCopyCandidate(_) => panic!("expected legacy compat reader"), + }; + + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await.unwrap(); + + assert_eq!(buf, b"compressed"); + } + + #[tokio::test] + async fn build_put_object_reduced_copy_reader_reads_across_multiple_chunks() { + let stream = futures_util::stream::iter([ + Ok::(Bytes::from_static(b"ab")), + Ok::(Bytes::from_static(b"cd")), + Ok::(Bytes::from_static(b"ef")), + ]); + + let mut reader = build_put_object_reduced_copy_reader(PutObjectReducedCopyIngress::from_stream( + stream, + PutObjectCompatIngressPlan { + kind: PutObjectCompatIngressKind::BufferedStreamCompat, + buffer_size: 16, + }, + )); + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await.unwrap(); + + assert_eq!(buf, b"abcdef"); + } + + #[tokio::test] + async fn build_put_object_reduced_copy_reader_defers_stream_error_until_next_read() { + let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"ab")), Err::("boom")]); + + let mut reader = build_put_object_reduced_copy_reader(PutObjectReducedCopyIngress::from_stream( + stream, + PutObjectCompatIngressPlan { + kind: PutObjectCompatIngressKind::BufferedStreamCompat, + buffer_size: 16, + }, + )); + + let mut buf = [0_u8; 8]; + let n = reader.read(&mut buf).await.unwrap(); + assert_eq!(n, 2); + assert_eq!(&buf[..n], b"ab"); + + let err = reader.read(&mut buf).await.unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::Other); + assert!(err.to_string().contains("boom")); + } + + #[tokio::test] + async fn build_put_object_reduced_copy_reader_supports_direct_block_reads() { + let stream = futures_util::stream::iter([ + Ok::(Bytes::from_static(b"ab")), + Ok::(Bytes::from_static(b"cd")), + Ok::(Bytes::from_static(b"ef")), + ]); + + let mut reader = build_put_object_reduced_copy_reader(PutObjectReducedCopyIngress::from_stream( + stream, + PutObjectCompatIngressPlan { + kind: PutObjectCompatIngressKind::BufferedStreamCompat, + buffer_size: 16, + }, + )); + + let mut first = [0_u8; 4]; + let mut second = [0_u8; 4]; + assert_eq!(reader.read_block(&mut first).await.unwrap(), 4); + assert_eq!(&first, b"abcd"); + assert_eq!(reader.read_block(&mut second).await.unwrap(), 2); + assert_eq!(&second[..2], b"ef"); + assert_eq!(reader.read_block(&mut second).await.unwrap(), 0); + } + + #[tokio::test] + async fn build_put_object_plain_hash_stage_reports_reduced_copy_candidate_boundary() { + let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"plain"))]); + let mut headers = HeaderMap::new(); + headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); + let plan = plan_put_object_body(2 * 1024 * 1024, &headers, "large.bin", 32); + + let stage = build_put_object_plain_hash_stage( + build_put_object_ingress_source(stream, plan), + PutObjectLegacyHashValues::default(), + PutObjectLegacyHashStagePlan { + size: 5, + actual_size: 5, + apply_s3_checksum: false, + ignore_s3_checksum_value: false, + }, + &headers, + None, + ) + .unwrap(); + + assert_eq!(stage.ingress_kind, PutObjectIngressKind::ReducedCopyCandidate); + + let mut reader = stage.reader; + let mut buf = [0_u8; 8]; + let n = reader.read_block(&mut buf).await.unwrap(); + assert_eq!(n, 5); + assert_eq!(&buf[..n], b"plain"); + assert_eq!(reader.read_block(&mut buf).await.unwrap(), 0); + } + + #[tokio::test] + async fn build_put_object_plain_hash_stage_preserves_legacy_compat_boundary() { + let stream = futures_util::stream::iter([Ok::(Bytes::from_static(b"legacy"))]); + let plan = plan_put_object_body(1024, &HeaderMap::new(), "small.bin", 32); + + let stage = build_put_object_plain_hash_stage( + build_put_object_ingress_source(stream, plan), + PutObjectLegacyHashValues::default(), + PutObjectLegacyHashStagePlan { + size: 6, + actual_size: 6, + apply_s3_checksum: false, + ignore_s3_checksum_value: false, + }, + &HeaderMap::new(), + None, + ) + .unwrap(); + + assert_eq!(stage.ingress_kind, PutObjectIngressKind::LegacyCompat); + + let mut reader = stage.reader; + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await.unwrap(); + + assert_eq!(buf, b"legacy"); + } + + #[test] + fn put_object_legacy_hash_values_clear_for_transformed_body_resets_hashes() { + let mut hash_values = PutObjectLegacyHashValues { + md5hex: Some("md5".to_string()), + sha256hex: Some("sha256".to_string()), + }; + + hash_values.clear_for_transformed_body(); + + assert_eq!(hash_values, PutObjectLegacyHashValues::default()); + } + + #[test] + fn build_put_object_legacy_hash_stage_preserves_legacy_compat_boundary() { + let reader: Box = Box::new(WarpReader::new(Cursor::new(Vec::from(&b"abc"[..])))); + let stage = build_put_object_legacy_hash_stage( + reader, + PutObjectLegacyHashValues::default(), + PutObjectLegacyHashStagePlan { + size: 3, + actual_size: 3, + apply_s3_checksum: false, + ignore_s3_checksum_value: false, + }, + &HeaderMap::new(), + None, + ) + .unwrap(); + + assert_eq!(stage.reader.size(), 3); + assert_eq!(stage.reader.actual_size(), 3); + assert!(stage.want_checksum.is_none()); + } +} diff --git a/crates/protocols/src/swift/object.rs b/crates/protocols/src/swift/object.rs index 1c59da07f..763a8f202 100644 --- a/crates/protocols/src/swift/object.rs +++ b/crates/protocols/src/swift/object.rs @@ -55,7 +55,7 @@ use super::{SwiftError, SwiftResult}; use axum::http::HeaderMap; use rustfs_credentials::Credentials; use rustfs_ecstore::new_object_layer_fn; -use rustfs_ecstore::store_api::{BucketOperations, BucketOptions, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}; +use rustfs_ecstore::store_api::{BucketOperations, BucketOptions, ChunkNativePutData, ObjectIO, ObjectOperations, ObjectOptions}; use rustfs_rio::HashReader; use std::collections::HashMap; use tracing::debug; @@ -381,8 +381,8 @@ where let hash_reader = HashReader::from_stream(buf_reader, content_length, content_length, None, None, false) .map_err(|e| sanitize_storage_error("Hash reader creation", e))?; - // 15. Wrap in PutObjReader as expected by storage layer - let mut put_reader = PutObjReader::new(hash_reader); + // 15. Hand the hash reader to the chunk-native PUT data wrapper + let mut put_reader = ChunkNativePutData::new(hash_reader); // 16. Upload object to storage let obj_info = store @@ -464,8 +464,8 @@ where let hash_reader = HashReader::from_stream(buf_reader, content_length, content_length, None, None, false) .map_err(|e| sanitize_storage_error("Hash reader creation", e))?; - // Wrap in PutObjReader - let mut put_reader = PutObjReader::new(hash_reader); + // Hand the hash reader to the chunk-native PUT data wrapper + let mut put_reader = ChunkNativePutData::new(hash_reader); // Upload object to storage let obj_info = store diff --git a/crates/rio/src/checksum.rs b/crates/rio/src/checksum.rs index 86867cc15..b09916bf8 100644 --- a/crates/rio/src/checksum.rs +++ b/crates/rio/src/checksum.rs @@ -29,11 +29,21 @@ pub const RUSTFS_MULTIPART_CHECKSUM: &str = "x-rustfs-multipart-checksum"; /// RustFS multipart checksum type metadata key pub const RUSTFS_MULTIPART_CHECKSUM_TYPE: &str = "x-rustfs-multipart-checksum-type"; +const AMZ_CHECKSUM_ALGORITHM: &str = "x-amz-checksum-algorithm"; +const AMZ_SDK_CHECKSUM_ALGORITHM: &str = "x-amz-sdk-checksum-algorithm"; + /// Checksum type enumeration with flags #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct ChecksumType(pub u32); impl ChecksumType { + fn algorithm_from_headers(headers: &HeaderMap) -> Option<&str> { + headers + .get(AMZ_CHECKSUM_ALGORITHM) + .and_then(|v| v.to_str().ok()) + .or_else(|| headers.get(AMZ_SDK_CHECKSUM_ALGORITHM).and_then(|v| v.to_str().ok())) + } + /// Checksum will be sent in trailing header pub const TRAILING: ChecksumType = ChecksumType(1 << 0); @@ -156,10 +166,7 @@ impl ChecksumType { pub fn from_header(headers: &HeaderMap) -> Self { Self::from_string_with_obj_type( - headers - .get("x-amz-checksum-algorithm") - .and_then(|v| v.to_str().ok()) - .unwrap_or(""), + Self::algorithm_from_headers(headers).unwrap_or(""), headers.get("x-amz-checksum-type").and_then(|v| v.to_str().ok()).unwrap_or(""), ) } @@ -573,7 +580,7 @@ pub fn get_content_checksum(headers: &HeaderMap) -> Result, std fn get_content_checksum_direct(headers: &HeaderMap) -> (ChecksumType, String) { let mut checksum_type = ChecksumType::NONE; - if let Some(alg) = headers.get("x-amz-checksum-algorithm").and_then(|v| v.to_str().ok()) { + if let Some(alg) = ChecksumType::algorithm_from_headers(headers) { checksum_type = ChecksumType::from_string_with_obj_type( alg, headers.get("x-amz-checksum-type").and_then(|s| s.to_str().ok()).unwrap_or(""), @@ -1131,7 +1138,8 @@ fn crc64_combine(poly: u64, crc1: u64, crc2: u64, len2: i64) -> u64 { #[cfg(test)] mod tests { - use super::{Checksum, ChecksumType}; + use super::{AMZ_SDK_CHECKSUM_ALGORITHM, Checksum, ChecksumType, get_content_checksum_direct}; + use http::{HeaderMap, HeaderValue}; #[test] fn crc64_nvme_add_part_matches_full_object_checksum() { @@ -1186,4 +1194,24 @@ mod tests { assert_eq!(combined.encoded, expected.encoded); assert_eq!(combined.raw, expected.raw); } + + #[test] + fn checksum_type_from_header_supports_sdk_checksum_algorithm_header() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SDK_CHECKSUM_ALGORITHM, HeaderValue::from_static("CRC32")); + + assert_eq!(ChecksumType::from_header(&headers), ChecksumType::CRC32); + } + + #[test] + fn get_content_checksum_direct_supports_sdk_checksum_algorithm_header() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SDK_CHECKSUM_ALGORITHM, HeaderValue::from_static("CRC32")); + headers.insert("x-amz-checksum-crc32", HeaderValue::from_static("nct/nQ==")); + + let (checksum_type, checksum_value) = get_content_checksum_direct(&headers); + + assert_eq!(checksum_type, ChecksumType::CRC32); + assert_eq!(checksum_value, "nct/nQ=="); + } } diff --git a/crates/rio/src/compress_reader.rs b/crates/rio/src/compress_reader.rs index 418373a89..e0d961234 100644 --- a/crates/rio/src/compress_reader.rs +++ b/crates/rio/src/compress_reader.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::compress_index::{Index, TryGetIndex}; +use crate::{BlockReadable, BoxReadBlockFuture, Reader}; use pin_project_lite::pin_project; use rustfs_utils::compress::{CompressionAlgorithm, compress_block, decompress_block}; use rustfs_utils::{put_uvarint, uvarint}; @@ -85,6 +86,31 @@ where read_buffer: vec![0u8; block_size], } } + + fn copy_buffered(&mut self, buf: &mut [u8]) -> usize { + if self.pos >= self.buffer.len() || buf.is_empty() { + return 0; + } + + let to_copy = min(buf.len(), self.buffer.len() - self.pos); + buf[..to_copy].copy_from_slice(&self.buffer[self.pos..self.pos + to_copy]); + self.pos += to_copy; + if self.pos == self.buffer.len() { + self.buffer.clear(); + self.pos = 0; + } + to_copy + } + + fn queue_compressed_block(&mut self, uncompressed_data: &[u8]) -> io::Result<()> { + let out = build_compressed_block(uncompressed_data, self.compression_algorithm); + self.written += out.len(); + self.uncomp_written += uncompressed_data.len(); + self.index.add(self.written as i64, self.uncomp_written as i64)?; + self.buffer = out; + self.pos = 0; + Ok(()) + } } impl TryGetIndex for CompressReader { @@ -170,6 +196,50 @@ where delegate_reader_capabilities_generic_no_index!(CompressReader, inner); +impl BlockReadable for CompressReader +where + R: Reader, +{ + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + Box::pin(async move { + if buf.is_empty() { + return Ok(0); + } + + let mut written = self.copy_buffered(buf); + while written < buf.len() { + if self.done { + break; + } + + self.temp_buffer.resize(self.block_size, 0); + let n = { + let inner = &mut self.inner; + let temp = &mut self.temp_buffer[..self.block_size]; + match inner.read_block(temp).await { + Ok(n) => n, + Err(err) if err.kind() == io::ErrorKind::UnexpectedEof => 0, + Err(err) => return Err(err), + } + }; + + if n == 0 { + self.done = true; + self.temp_buffer.clear(); + break; + } + + let block = self.temp_buffer[..n].to_vec(); + self.temp_buffer.clear(); + self.queue_compressed_block(&block)?; + written += self.copy_buffered(&mut buf[written..]); + } + + Ok(written) + }) + } +} + pin_project! { /// A reader wrapper that decompresses data on the fly using DEFLATE algorithm. /// Header format: @@ -390,6 +460,7 @@ fn build_compressed_block(uncompressed_data: &[u8], compression_algorithm: Compr #[cfg(test)] mod tests { use super::*; + use crate::{BlockReadable, WarpReader}; use rand::RngExt; use std::io::Cursor; use tokio::io::{AsyncReadExt, BufReader}; @@ -479,4 +550,27 @@ mod tests { assert_eq!(&decompressed, &data); } + + #[tokio::test] + async fn test_compress_reader_read_block_round_trips() { + let data = b"hello world, hello world, hello world!"; + let reader = Cursor::new(data.to_vec()); + let mut compress_reader = CompressReader::new(WarpReader::new(reader), CompressionAlgorithm::Gzip); + let mut compressed = Vec::new(); + let mut buf = [0u8; 19]; + + loop { + let n = compress_reader.read_block(&mut buf).await.unwrap(); + if n == 0 { + break; + } + compressed.extend_from_slice(&buf[..n]); + } + + let mut decompress_reader = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::Gzip); + let mut decompressed = Vec::new(); + decompress_reader.read_to_end(&mut decompressed).await.unwrap(); + + assert_eq!(&decompressed, data); + } } diff --git a/crates/rio/src/encrypt_reader.rs b/crates/rio/src/encrypt_reader.rs index 4b8e275cf..d815a75d3 100644 --- a/crates/rio/src/encrypt_reader.rs +++ b/crates/rio/src/encrypt_reader.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::compress_index::{Index, TryGetIndex}; +use crate::{BlockReadable, BoxReadBlockFuture, Reader}; use aes_gcm::aead::Aead; use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; use pin_project_lite::pin_project; @@ -59,6 +60,69 @@ where } } +fn encrypt_segment_bytes(cipher: &Aes256Gcm, nonce_bytes: &[u8; 12], plaintext: &[u8]) -> std::io::Result> { + let nonce = Nonce::try_from(nonce_bytes.as_slice()).map_err(|_| Error::other("invalid nonce length"))?; + let plaintext_len = plaintext.len(); + let crc = { + let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + hasher.update(plaintext); + hasher.finalize() as u32 + }; + let ciphertext = cipher + .encrypt(&nonce, plaintext) + .map_err(|e| Error::other(format!("encrypt error: {e}")))?; + let int_len = put_uvarint_len(plaintext_len as u64); + let clen = int_len + ciphertext.len() + 4; + let mut header = [0u8; 8]; + header[0] = 0x00; + header[1] = (clen & 0xFF) as u8; + header[2] = ((clen >> 8) & 0xFF) as u8; + header[3] = ((clen >> 16) & 0xFF) as u8; + header[4] = (crc & 0xFF) as u8; + header[5] = ((crc >> 8) & 0xFF) as u8; + header[6] = ((crc >> 16) & 0xFF) as u8; + header[7] = ((crc >> 24) & 0xFF) as u8; + debug!( + "encrypt block header typ=0 len={} header={:?} plaintext_len={} ciphertext_len={}", + clen, + header, + plaintext_len, + ciphertext.len() + ); + let mut out = Vec::with_capacity(8 + int_len + ciphertext.len()); + out.extend_from_slice(&header); + let mut plaintext_len_buf = vec![0u8; int_len]; + put_uvarint(&mut plaintext_len_buf, plaintext_len as u64); + out.extend_from_slice(&plaintext_len_buf); + out.extend_from_slice(&ciphertext); + Ok(out) +} + +impl EncryptReader +where + R: Reader, +{ + fn copy_buffered(&mut self, buf: &mut [u8]) -> usize { + if self.buffer_pos >= self.buffer.len() || buf.is_empty() { + return 0; + } + + let to_copy = buf.len().min(self.buffer.len() - self.buffer_pos); + buf[..to_copy].copy_from_slice(&self.buffer[self.buffer_pos..self.buffer_pos + to_copy]); + self.buffer_pos += to_copy; + if self.buffer_pos == self.buffer.len() { + self.buffer.clear(); + self.buffer_pos = 0; + } + to_copy + } + + fn encrypt_segment(&self, plaintext: &[u8]) -> std::io::Result> { + let nonce = derive_block_nonce(&self.base_nonce, self.block_index); + encrypt_segment_bytes(&self.cipher, &nonce, plaintext) + } +} + impl AsyncRead for EncryptReader where R: AsyncRead + Unpin + Send + Sync, @@ -97,49 +161,8 @@ where *this.buffer_pos += to_copy; Poll::Ready(Ok(())) } else { - // Encrypt the chunk let block_nonce = derive_block_nonce(this.base_nonce, *this.block_index); - let nonce = Nonce::try_from(block_nonce.as_slice()).map_err(|_| Error::other("invalid nonce length"))?; - let plaintext = &this.read_buffer[..n]; - let plaintext_len = plaintext.len(); - let crc = { - let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); - hasher.update(plaintext); - hasher.finalize() as u32 - }; - let ciphertext = this - .cipher - .encrypt(&nonce, plaintext) - .map_err(|e| Error::other(format!("encrypt error: {e}")))?; - let int_len = put_uvarint_len(plaintext_len as u64); - let clen = int_len + ciphertext.len() + 4; - // Header: 8 bytes - // 0: type (0 = encrypted, 0xFF = end) - // 1-3: length (little endian u24, ciphertext length) - // 4-7: CRC32 of ciphertext (little endian u32) - let mut header = [0u8; 8]; - header[0] = 0x00; // 0 = encrypted - header[1] = (clen & 0xFF) as u8; - header[2] = ((clen >> 8) & 0xFF) as u8; - header[3] = ((clen >> 16) & 0xFF) as u8; - header[4] = (crc & 0xFF) as u8; - header[5] = ((crc >> 8) & 0xFF) as u8; - header[6] = ((crc >> 16) & 0xFF) as u8; - header[7] = ((crc >> 24) & 0xFF) as u8; - debug!( - "encrypt block header typ=0 len={} header={:?} plaintext_len={} ciphertext_len={}", - clen, - header, - plaintext_len, - ciphertext.len() - ); - let mut out = Vec::with_capacity(8 + int_len + ciphertext.len()); - out.extend_from_slice(&header); - let mut plaintext_len_buf = [0u8; 10]; - let encoded_len = put_uvarint(&mut plaintext_len_buf, plaintext_len as u64); - out.extend_from_slice(&plaintext_len_buf[..encoded_len]); - out.extend_from_slice(&ciphertext); - *this.buffer = out; + *this.buffer = encrypt_segment_bytes(this.cipher, &block_nonce, &this.read_buffer[..n])?; *this.buffer_pos = 0; *this.block_index += 1; let to_copy = std::cmp::min(buf.remaining(), this.buffer.len()); @@ -164,6 +187,50 @@ where } } +impl BlockReadable for EncryptReader +where + R: Reader, +{ + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + Box::pin(async move { + if buf.is_empty() { + return Ok(0); + } + + let mut written = self.copy_buffered(buf); + while written < buf.len() { + if self.finished { + break; + } + + let mut plaintext = vec![0u8; 8 * 1024]; + let n = match self.inner.read_block(&mut plaintext).await { + Ok(n) => n, + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => 0, + Err(err) => return Err(err), + }; + if n == 0 { + self.buffer = [0xFF, 0, 0, 0, 0, 0, 0, 0].to_vec(); + self.buffer_pos = 0; + self.finished = true; + } else { + self.buffer = self.encrypt_segment(&plaintext[..n])?; + self.buffer_pos = 0; + } + + let copied = self.copy_buffered(&mut buf[written..]); + written += copied; + + if copied == 0 && self.finished { + break; + } + } + + Ok(written) + }) + } +} + pin_project! { /// A reader wrapper that decrypts data on the fly using AES-256-GCM. /// This is a demonstration. For production, use a secure and audited crypto library. @@ -486,7 +553,7 @@ mod tests { use std::pin::Pin; use std::task::{Context, Poll}; - use crate::HardLimitReader; + use crate::{BlockReadable, HardLimitReader, WarpReader}; use super::*; use futures::StreamExt; @@ -674,6 +741,35 @@ mod tests { assert_eq!(&decrypted, &data); } + #[tokio::test] + async fn test_encrypt_reader_read_block_round_trips() { + let data = b"hello sse encrypt via blocks"; + let mut key = [0u8; 32]; + let mut nonce = [0u8; 12]; + rand::rng().fill_bytes(&mut key); + rand::rng().fill_bytes(&mut nonce); + + let reader = Cursor::new(data.to_vec()); + let mut encrypt_reader = EncryptReader::new(WarpReader::new(reader), key, nonce); + let mut encrypted = Vec::new(); + let mut buf = [0u8; 17]; + + loop { + let n = encrypt_reader.read_block(&mut buf).await.unwrap(); + if n == 0 { + break; + } + encrypted.extend_from_slice(&buf[..n]); + } + + let reader = Cursor::new(encrypted); + let mut decrypt_reader = DecryptReader::new(WarpReader::new(reader), key, nonce); + let mut decrypted = Vec::new(); + decrypt_reader.read_to_end(&mut decrypted).await.unwrap(); + + assert_eq!(&decrypted, data); + } + #[tokio::test] async fn test_decrypt_reader_large_with_small_chunks() { let size = 1024 * 1024; diff --git a/crates/rio/src/etag_reader.rs b/crates/rio/src/etag_reader.rs index ba1638069..d6f410ae9 100644 --- a/crates/rio/src/etag_reader.rs +++ b/crates/rio/src/etag_reader.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::compress_index::{Index, TryGetIndex}; -use crate::{EtagResolvable, HashReaderDetector, HashReaderMut}; +use crate::{BlockReadable, BoxReadBlockFuture, EtagResolvable, HashReaderDetector, HashReaderMut, Reader}; use md5::{Digest, Md5}; use pin_project_lite::pin_project; use std::pin::Pin; @@ -135,9 +135,41 @@ where } } +impl BlockReadable for EtagReader +where + R: Reader, +{ + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + Box::pin(async move { + let n = match self.inner.read_block(buf).await { + Ok(n) => n, + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => 0, + Err(err) => return Err(err), + }; + if n > 0 { + self.md5.update(&buf[..n]); + return Ok(n); + } + + self.finished = true; + if let Some(checksum) = &self.checksum { + let etag = self.md5.clone().finalize().to_vec(); + let etag_hex = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower); + if checksum != &etag_hex { + error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag_hex); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Checksum mismatch")); + } + } + + Ok(0) + }) + } +} + #[cfg(test)] mod tests { use super::*; + use crate::{BlockReadable, WarpReader}; use rand::RngExt; use std::io::Cursor; use tokio::io::{AsyncReadExt, BufReader}; @@ -180,6 +212,24 @@ mod tests { assert_eq!(etag, Some(expected)); } + #[tokio::test] + async fn test_etag_reader_read_block_updates_checksum() { + let data = b"hello world"; + let mut hasher = Md5::new(); + hasher.update(data); + let expected = faster_hex::hex_string(hasher.finalize().as_slice()).to_string(); + let reader = BufReader::new(&data[..]); + let reader = Box::new(WarpReader::new(reader)); + let mut etag_reader = EtagReader::new(reader, Some(expected.clone())); + + let mut buf = [0_u8; 32]; + let n = etag_reader.read_block(&mut buf).await.unwrap(); + assert_eq!(n, data.len()); + assert_eq!(&buf[..n], data); + assert_eq!(etag_reader.read_block(&mut buf).await.unwrap(), 0); + assert_eq!(etag_reader.try_resolve_etag(), Some(expected)); + } + #[tokio::test] async fn test_etag_reader_multiple_get() { let data = b"abc123"; diff --git a/crates/rio/src/hardlimit_reader.rs b/crates/rio/src/hardlimit_reader.rs index e50b052f5..c74f37d89 100644 --- a/crates/rio/src/hardlimit_reader.rs +++ b/crates/rio/src/hardlimit_reader.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::{BlockReadable, BoxReadBlockFuture, Reader}; use pin_project_lite::pin_project; use std::io::{Error, Result}; use std::pin::Pin; @@ -61,11 +62,51 @@ where delegate_reader_capabilities_generic!(HardLimitReader, inner); +impl BlockReadable for HardLimitReader +where + R: Reader, +{ + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + Box::pin(async move { + if self.remaining < 0 { + return Err(Error::other("input provided more bytes than specified")); + } + + let max_len = match usize::try_from(self.remaining) { + Ok(remaining) => remaining.min(buf.len()), + Err(_) => buf.len(), + }; + + if max_len == 0 { + let mut probe = [0_u8; 1]; + match self.inner.read_block(&mut probe).await { + Ok(0) => return Ok(0), + Ok(n) => { + self.remaining -= n as i64; + return Err(Error::other("input provided more bytes than specified")); + } + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(0), + Err(err) => return Err(err), + } + } + + let n = self.inner.read_block(&mut buf[..max_len]).await?; + self.remaining -= n as i64; + if self.remaining < 0 { + return Err(Error::other("input provided more bytes than specified")); + } + + Ok(n) + }) + } +} + #[cfg(test)] mod tests { use std::vec; use super::*; + use crate::{BlockReadable, WarpReader}; use rustfs_utils::read_full; use tokio::io::{AsyncReadExt, BufReader}; @@ -128,4 +169,20 @@ mod tests { assert_eq!(n, 0); assert_eq!(&buf, data); } + + #[tokio::test] + async fn test_hardlimit_reader_read_block_enforces_limit() { + let data = b"abcdef"; + let reader = BufReader::new(&data[..]); + let reader = Box::new(WarpReader::new(reader)); + let mut hardlimit = HardLimitReader::new(reader, 3); + + let mut buf = [0_u8; 8]; + let n = hardlimit.read_block(&mut buf).await.unwrap(); + assert_eq!(n, 3); + assert_eq!(&buf[..n], b"abc"); + + let err = hardlimit.read_block(&mut buf).await.unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::Other); + } } diff --git a/crates/rio/src/hash_reader.rs b/crates/rio/src/hash_reader.rs index aee0a50d6..092eb3786 100644 --- a/crates/rio/src/hash_reader.rs +++ b/crates/rio/src/hash_reader.rs @@ -90,7 +90,10 @@ use crate::ChecksumType; use crate::Sha256Hasher; use crate::compress_index::{Index, TryGetIndex}; use crate::get_content_checksum; -use crate::{DynReader, EtagReader, EtagResolvable, HardLimitReader, HashReaderDetector, WarpReader, boxed_reader, wrap_reader}; +use crate::{ + BlockReadable, BoxReadBlockFuture, DynReader, EtagReader, EtagResolvable, HardLimitReader, HashReaderDetector, WarpReader, + boxed_reader, wrap_reader, +}; use base64::Engine; use base64::engine::general_purpose; use http::HeaderMap; @@ -408,6 +411,23 @@ impl HashReader { Ok(()) } + pub fn enable_auto_checksum(&mut self, checksum_type: ChecksumType) -> Result<(), std::io::Error> { + if !checksum_type.is_set() { + return Ok(()); + } + + let Some(hasher) = checksum_type.hasher() else { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid checksum type")); + }; + + self.content_hash = Some(Checksum { + checksum_type, + ..Default::default() + }); + self.content_hasher = Some(hasher); + Ok(()) + } + pub fn checksum(&self) -> Option { if self .content_hash @@ -449,6 +469,97 @@ impl HashReader { } map } + + pub fn finalize_content_hash(&mut self) -> std::io::Result> { + self.finish_checksum_validation()?; + Ok(self.content_hash.clone()) + } + + fn update_read_state(&mut self, data: &[u8]) -> std::io::Result<()> { + self.bytes_read += data.len() as u64; + + if data.is_empty() { + return Ok(()); + } + + if let Some(hasher) = self.content_sha256_hasher.as_mut() { + hasher.write_all(data)?; + } + + if let Some(hasher) = self.content_hasher.as_mut() { + hasher.write_all(data)?; + } + + Ok(()) + } + + fn finish_checksum_validation(&mut self) -> std::io::Result<()> { + if self.checksum_on_finish { + return Ok(()); + } + + if let (Some(mut hasher), Some(expected_sha256)) = (self.content_sha256_hasher.take(), self.content_sha256.as_ref()) { + let sha256 = hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower); + if sha256 != *expected_sha256 { + error!("SHA256 mismatch, expected={:?}, actual={:?}", expected_sha256, sha256); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "SHA256 mismatch")); + } + } + + if let Some(mut expected_content_hash) = self.content_hash.clone() + && let Some(mut hasher) = self.content_hasher.take() + { + if expected_content_hash.checksum_type.trailing() + && let Some(trailer) = self.trailer_s3s.as_ref() + && let Some(Some(checksum_str)) = trailer.read(|headers| { + expected_content_hash + .checksum_type + .key() + .and_then(|key| headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string()))) + }) + { + expected_content_hash.encoded = checksum_str; + expected_content_hash.raw = general_purpose::STANDARD + .decode(&expected_content_hash.encoded) + .map_err(|_| std::io::Error::other("Invalid base64 checksum"))?; + + if expected_content_hash.raw.is_empty() { + return Err(std::io::Error::other("Content hash mismatch")); + } + } + + let content_hash = hasher.finalize(); + if expected_content_hash.encoded.is_empty() { + expected_content_hash.raw = content_hash.clone(); + expected_content_hash.encoded = general_purpose::STANDARD.encode(&content_hash); + self.content_hash = Some(expected_content_hash); + } else if content_hash != expected_content_hash.raw { + let expected_hex = hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower); + let actual_hex = hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower); + error!( + "Content hash mismatch, type={:?}, encoded={:?}, expected={:?}, actual={:?}", + expected_content_hash.checksum_type, expected_content_hash.encoded, expected_hex, actual_hex + ); + let checksum_err = crate::errors::ChecksumMismatch { + want: expected_hex, + got: actual_hex, + }; + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, checksum_err)); + } + } + + self.checksum_on_finish = true; + Ok(()) + } + + pub async fn read_block(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read_block(buf).await?; + self.update_read_state(&buf[..n])?; + if n == 0 { + self.finish_checksum_validation()?; + } + Ok(n) + } } impl HashReaderMut for HashReader { @@ -508,84 +619,23 @@ impl HashReaderMut for HashReader { impl AsyncRead for HashReader { fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - let this = self.project(); + let this = self.get_mut(); let before = buf.filled().len(); - match this.inner.poll_read(cx, buf) { + match Pin::new(&mut this.inner).poll_read(cx, buf) { Poll::Pending => Poll::Pending, Poll::Ready(Ok(())) => { let data = &buf.filled()[before..]; let filled = data.len(); - - *this.bytes_read += filled as u64; - - if filled > 0 { - // Update SHA256 hasher - if let Some(hasher) = this.content_sha256_hasher - && let Err(e) = hasher.write_all(data) - { - error!("SHA256 hasher write error, error={:?}", e); - return Poll::Ready(Err(std::io::Error::other(e))); - } - - // Update content hasher - if let Some(hasher) = this.content_hasher - && let Err(e) = hasher.write_all(data) - { - return Poll::Ready(Err(std::io::Error::other(e))); - } + if let Err(e) = this.update_read_state(data) { + error!("hash reader state update error, error={:?}", e); + return Poll::Ready(Err(std::io::Error::other(e))); } - if filled == 0 && !*this.checksum_on_finish { - // check SHA256 - if let (Some(hasher), Some(expected_sha256)) = (this.content_sha256_hasher, this.content_sha256) { - let sha256 = hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower); - if sha256 != *expected_sha256 { - error!("SHA256 mismatch, expected={:?}, actual={:?}", expected_sha256, sha256); - return Poll::Ready(Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "SHA256 mismatch"))); - } - } - - // check content hasher - if let (Some(hasher), Some(expected_content_hash)) = (this.content_hasher, this.content_hash) { - if expected_content_hash.checksum_type.trailing() - && let Some(trailer) = this.trailer_s3s.as_ref() - && let Some(Some(checksum_str)) = trailer.read(|headers| { - expected_content_hash - .checksum_type - .key() - .and_then(|key| headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string()))) - }) - { - expected_content_hash.encoded = checksum_str; - expected_content_hash.raw = general_purpose::STANDARD - .decode(&expected_content_hash.encoded) - .map_err(|_| std::io::Error::other("Invalid base64 checksum"))?; - - if expected_content_hash.raw.is_empty() { - return Poll::Ready(Err(std::io::Error::other("Content hash mismatch"))); - } - } - - let content_hash = hasher.finalize(); - - if content_hash != expected_content_hash.raw { - let expected_hex = hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower); - let actual_hex = hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower); - error!( - "Content hash mismatch, type={:?}, encoded={:?}, expected={:?}, actual={:?}", - expected_content_hash.checksum_type, expected_content_hash.encoded, expected_hex, actual_hex - ); - // Use ChecksumMismatch error so that API layer can return BadDigest - let checksum_err = crate::errors::ChecksumMismatch { - want: expected_hex, - got: actual_hex, - }; - return Poll::Ready(Err(std::io::Error::new(std::io::ErrorKind::InvalidData, checksum_err))); - } - } - - *this.checksum_on_finish = true; + if filled == 0 + && let Err(e) = this.finish_checksum_validation() + { + return Poll::Ready(Err(e)); } Poll::Ready(Ok(())) } @@ -623,13 +673,39 @@ impl TryGetIndex for HashReader { } } +impl BlockReadable for HashReader { + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + Box::pin(async move { self.read_block(buf).await }) + } +} + #[cfg(test)] mod tests { use super::*; use crate::{DecryptReader, EncryptReader, encrypt_reader, wrap_reader}; use rand::RngExt; use std::io::Cursor; - use tokio::io::{AsyncReadExt, BufReader}; + use std::pin::Pin; + use std::task::{Context, Poll}; + use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf}; + + struct UnexpectedEofReader; + + impl AsyncRead for UnexpectedEofReader { + fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + + impl BlockReadable for UnexpectedEofReader { + fn read_block<'a>(&'a mut self, _buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + Box::pin(async { Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "synthetic unexpected eof")) }) + } + } + + impl EtagResolvable for UnexpectedEofReader {} + impl HashReaderDetector for UnexpectedEofReader {} + impl TryGetIndex for UnexpectedEofReader {} #[tokio::test] async fn test_hashreader_wrapping_logic() { @@ -742,6 +818,37 @@ mod tests { assert_eq!(buf, data); } + #[tokio::test] + async fn test_hashreader_read_block_reads_full_and_tail_block() { + let data = b"hello block reader"; + let reader = BufReader::new(Cursor::new(&data[..])); + let reader = Box::new(WarpReader::new(reader)); + let mut hash_reader = HashReader::new(reader, data.len() as i64, data.len() as i64, None, None, false).unwrap(); + + let mut first = [0_u8; 8]; + let n1 = hash_reader.read_block(&mut first).await.unwrap(); + assert_eq!(n1, 8); + assert_eq!(&first[..n1], b"hello bl"); + + let mut second = [0_u8; 32]; + let n2 = hash_reader.read_block(&mut second).await.unwrap(); + assert_eq!(n2, data.len() - n1); + assert_eq!(&second[..n2], b"ock reader"); + + let n3 = hash_reader.read_block(&mut second).await.unwrap(); + assert_eq!(n3, 0); + } + + #[tokio::test] + async fn test_hashreader_read_block_propagates_unexpected_eof() { + let mut hash_reader = HashReader::new(Box::new(UnexpectedEofReader), 0, 0, None, None, true).unwrap(); + let mut buf = [0_u8; 8]; + + let err = hash_reader.read_block(&mut buf).await.unwrap_err(); + + assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof); + } + #[tokio::test] async fn test_hashreader_new_logic() { let data = b"test data"; diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index 39ef43ecd..b207fbc95 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -23,7 +23,6 @@ use rustfs_utils::get_env_opt_str; use std::io::IoSlice; use std::io::{self, Error}; use std::net::IpAddr; -use std::ops::Not as _; use std::pin::Pin; use std::sync::LazyLock; use std::task::{Context, Poll}; @@ -137,6 +136,71 @@ fn get_http_client(url: &str) -> Client { CLIENT.clone() } +type HttpByteStream = Pin> + Send + Sync>>; + +async fn request_http_byte_stream( + url: String, + method: Method, + headers: HeaderMap, + body: Option>, + meter_stream_recv_bytes: bool, +) -> io::Result<(bool, HttpByteStream)> { + let track_internode_metrics = is_internode_rpc_url(&url); + let client = get_http_client(&url); + let mut request: RequestBuilder = client.request(method, url.clone()).headers(headers); + if let Some(body) = body { + request = request.body(body); + } + + let resp = request.send().await.map_err(|e| { + if track_internode_metrics { + global_internode_metrics().record_error(); + } + Error::other(format!("HttpReader HTTP request error: {e}")) + })?; + + if !resp.status().is_success() { + if track_internode_metrics { + global_internode_metrics().record_error(); + } + return Err(Error::other(format!( + "HttpReader HTTP request failed with non-200 status {}", + resp.status() + ))); + } + + if track_internode_metrics { + global_internode_metrics().record_outgoing_request(); + } + + let stream = resp + .bytes_stream() + .map_ok(move |bytes| { + if track_internode_metrics && meter_stream_recv_bytes { + global_internode_metrics().record_recv_bytes(bytes.len()); + } + bytes + }) + .map_err(move |e| { + if track_internode_metrics { + global_internode_metrics().record_error(); + } + Error::other(format!("HttpReader stream error: {e}")) + }); + + Ok((track_internode_metrics, Box::pin(stream))) +} + +pub async fn open_http_byte_stream( + url: String, + method: Method, + headers: HeaderMap, + body: Option>, +) -> io::Result { + let (_track_internode_metrics, stream) = request_http_byte_stream(url, method, headers, body, true).await?; + Ok(stream) +} + pin_project! { pub struct HttpReader { url:String, @@ -161,43 +225,11 @@ impl HttpReader { body: Option>, _read_buf_size: usize, ) -> io::Result { - let track_internode_metrics = is_internode_rpc_url(&url); - let client = get_http_client(&url); - let mut request: RequestBuilder = client.request(method.clone(), url.clone()).headers(headers.clone()); - if let Some(body) = body { - request = request.body(body); - } - - let resp = request.send().await.map_err(|e| { - if track_internode_metrics { - global_internode_metrics().record_error(); - } - Error::other(format!("HttpReader HTTP request error: {e}")) - })?; - - if resp.status().is_success().not() { - if track_internode_metrics { - global_internode_metrics().record_error(); - } - return Err(Error::other(format!( - "HttpReader HTTP request failed with non-200 status {}", - resp.status() - ))); - } - - if track_internode_metrics { - global_internode_metrics().record_outgoing_request(); - } - - let stream = resp.bytes_stream().map_err(move |e| { - if track_internode_metrics { - global_internode_metrics().record_error(); - } - Error::other(format!("HttpReader stream error: {e}")) - }); + let (track_internode_metrics, stream) = + request_http_byte_stream(url.clone(), method.clone(), headers.clone(), body, false).await?; Ok(Self { - inner: StreamReader::new(Box::pin(stream)), + inner: StreamReader::new(stream), url, method, headers, diff --git a/crates/rio/src/lib.rs b/crates/rio/src/lib.rs index 9663f133d..6e9040382 100644 --- a/crates/rio/src/lib.rs +++ b/crates/rio/src/lib.rs @@ -15,6 +15,10 @@ // Default encryption block size - aligned with system default read buffer size (1MB) pub const DEFAULT_ENCRYPTION_BLOCK_SIZE: usize = 1024 * 1024; +use std::future::Future; +use std::pin::Pin; +use tokio::io::AsyncReadExt; + macro_rules! delegate_reader_capabilities_generic { ($name:ident<$inner_ty:ident>, $inner:ident) => { impl<$inner_ty> crate::EtagResolvable for $name<$inner_ty> @@ -114,14 +118,39 @@ pub use compress_index::{Index, TryGetIndex}; mod etag; +pub type BoxReadBlockFuture<'a> = Pin> + Send + 'a>>; + +pub trait BlockReadable { + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a>; +} + +fn read_block_via_async_read<'a, R>(reader: &'a mut R, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> +where + R: tokio::io::AsyncRead + Unpin + Send + Sync + 'a, +{ + Box::pin(async move { + let mut total = 0; + + while total < buf.len() { + match reader.read(&mut buf[total..]).await { + Ok(0) => return Ok(total), + Ok(n) => total += n, + Err(err) => return Err(err), + } + } + + Ok(total) + }) +} + pub trait ReadStream: tokio::io::AsyncRead + Unpin + Send + Sync {} impl ReadStream for T where T: tokio::io::AsyncRead + Unpin + Send + Sync {} pub trait ReaderCapabilities: EtagResolvable + HashReaderDetector + TryGetIndex {} impl ReaderCapabilities for T where T: EtagResolvable + HashReaderDetector + TryGetIndex {} -pub trait Reader: ReadStream + ReaderCapabilities {} -impl Reader for T where T: ReadStream + ReaderCapabilities {} +pub trait Reader: ReadStream + ReaderCapabilities + BlockReadable {} +impl Reader for T where T: ReadStream + ReaderCapabilities + BlockReadable {} pub type DynReader = Box; @@ -154,6 +183,42 @@ pub trait HashReaderDetector { } } +impl BlockReadable for crate::WarpReader +where + R: tokio::io::AsyncRead + Unpin + Send + Sync, +{ + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + read_block_via_async_read(self, buf) + } +} + +impl BlockReadable for tokio::io::BufReader +where + R: tokio::io::AsyncRead + Unpin + Send + Sync, +{ + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + read_block_via_async_read(self, buf) + } +} + +impl BlockReadable for crate::LimitReader +where + R: Reader, +{ + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + read_block_via_async_read(self, buf) + } +} + +impl BlockReadable for crate::DecryptReader +where + R: Reader, +{ + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + read_block_via_async_read(self, buf) + } +} + pub fn boxed_reader(reader: R) -> DynReader where R: Reader + 'static, @@ -198,3 +263,74 @@ where self.as_ref().try_get_index() } } + +impl BlockReadable for Box +where + T: BlockReadable + ?Sized, +{ + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + self.as_mut().read_block(buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::VecDeque; + use std::io::{self, ErrorKind}; + use std::pin::Pin; + use std::task::{Context, Poll}; + use tokio::io::{AsyncRead, ReadBuf}; + + enum ReadStep { + Data(Vec), + Error(ErrorKind), + Eof, + } + + struct StepReader { + steps: VecDeque, + } + + impl StepReader { + fn new(steps: impl IntoIterator) -> Self { + Self { + steps: steps.into_iter().collect(), + } + } + } + + impl AsyncRead for StepReader { + fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + match self.steps.pop_front().unwrap_or(ReadStep::Eof) { + ReadStep::Data(data) => { + buf.put_slice(&data); + Poll::Ready(Ok(())) + } + ReadStep::Error(kind) => Poll::Ready(Err(io::Error::new(kind, "synthetic read failure"))), + ReadStep::Eof => Poll::Ready(Ok(())), + } + } + } + + #[tokio::test] + async fn test_read_block_via_async_read_preserves_midstream_error_kind() { + let reader = StepReader::new([ReadStep::Data(b"ab".to_vec()), ReadStep::Error(ErrorKind::ConnectionReset)]); + let mut reader = WarpReader::new(reader); + let mut buf = [0_u8; 4]; + + let err = reader.read_block(&mut buf).await.unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::ConnectionReset); + } + + #[tokio::test] + async fn test_read_block_via_async_read_returns_zero_on_initial_eof() { + let mut reader = WarpReader::new(StepReader::new([ReadStep::Eof])); + let mut buf = [0_u8; 4]; + + let n = reader.read_block(&mut buf).await.unwrap(); + + assert_eq!(n, 0); + } +} diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index b8e8304f7..ca0d475ed 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -24,7 +24,7 @@ use rustfs_ecstore::{ pools::path2_bucket_object_with_base_path, store::ECStore, store_api::{ - BucketOperations, MakeBucketOptions, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, + BucketOperations, ChunkNativePutData, MakeBucketOptions, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, }, tier::{ tier_config::{TierConfig, TierMinIO, TierType}, @@ -235,7 +235,7 @@ async fn create_test_lock_bucket(ecstore: &Arc, bucket_name: &str) { /// Test helper: Upload test object async fn upload_test_object(ecstore: &Arc, bucket: &str, object: &str, data: &[u8]) { - let mut reader = PutObjReader::from_vec(data.to_vec()); + let mut reader = ChunkNativePutData::from_vec(data.to_vec()); let object_info = (**ecstore) .put_object(bucket, object, &mut reader, &ObjectOptions::default()) .await @@ -781,7 +781,7 @@ mod serial_tests { .await .expect("Failed to set lifecycle configuration"); - let mut reader = PutObjReader::from_vec(put_payload.to_vec()); + let mut reader = ChunkNativePutData::from_vec(put_payload.to_vec()); let mut metadata = HashMap::new(); metadata.insert("content-type".to_string(), "text/plain".to_string()); ecstore @@ -838,7 +838,7 @@ mod serial_tests { .expect("Failed to create multipart upload"); let part_data = b"multipart immediate transition"; - let mut reader = PutObjReader::from_vec(part_data.to_vec()); + let mut reader = ChunkNativePutData::from_vec(part_data.to_vec()); let part = ecstore .put_object_part( multipart_bucket.as_str(), @@ -903,7 +903,7 @@ mod serial_tests { .get_object_info(src_bucket.as_str(), src_object, &ObjectOptions::default()) .await .expect("Failed to load source object info"); - src_info.put_object_reader = Some(PutObjReader::from_vec(payload.to_vec())); + src_info.put_object_reader = Some(ChunkNativePutData::from_vec(payload.to_vec())); ecstore .copy_object( @@ -969,7 +969,7 @@ mod serial_tests { .await .expect("Failed to create multipart upload"); - let mut part1_reader = PutObjReader::from_vec(part1); + let mut part1_reader = ChunkNativePutData::from_vec(part1); let uploaded_part1 = ecstore .put_object_part( bucket_name.as_str(), @@ -982,7 +982,7 @@ mod serial_tests { .await .expect("Failed to upload first multipart part"); - let mut part2_reader = PutObjReader::from_vec(part2); + let mut part2_reader = ChunkNativePutData::from_vec(part2); let uploaded_part2 = ecstore .put_object_part( bucket_name.as_str(), diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 16a27178c..ff923e174 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -86,6 +86,7 @@ rustfs-utils = { workspace = true, features = ["full"] } rustfs-zip = { workspace = true } rustfs-io-core = { workspace = true } rustfs-io-metrics = { workspace = true } +rustfs-object-io = { workspace = true } rustfs-concurrency = { workspace = true } rustfs-scanner = { workspace = true } diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index ae144b880..8ed844525 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -27,8 +27,8 @@ use rustfs_ecstore::{ global::GLOBAL_TierConfigMgr, store::ECStore, store_api::{ - BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, - PutObjReader, + BucketOperations, BucketOptions, ChunkNativePutData, MakeBucketOptions, MultipartOperations, ObjectIO, ObjectOperations, + ObjectOptions, }, tier::{ tier_config::{TierConfig, TierType}, @@ -148,7 +148,7 @@ async fn upload_test_object( object: &str, data: &[u8], ) -> rustfs_ecstore::store_api::ObjectInfo { - let mut reader = PutObjReader::from_vec(data.to_vec()); + let mut reader = ChunkNativePutData::from_vec(data.to_vec()); (**ecstore) .put_object(bucket, object, &mut reader, &ObjectOptions::default()) .await @@ -446,7 +446,7 @@ async fn complete_multipart_upload_transitions_immediately_via_usecase() { .await .expect("Failed to create multipart upload"); - let mut reader = PutObjReader::from_vec(payload.to_vec()); + let mut reader = ChunkNativePutData::from_vec(payload.to_vec()); let uploaded_part = ecstore .put_object_part(bucket.as_str(), object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) .await diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 0f97f1fe1..b370fc89e 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -44,10 +44,13 @@ use rustfs_ecstore::compress::is_compressible; use rustfs_ecstore::error::{StorageError, is_err_object_not_found, is_err_version_not_found}; use rustfs_ecstore::new_object_layer_fn; use rustfs_ecstore::set_disk::{MAX_PARTS_COUNT, is_valid_storage_class}; -use rustfs_ecstore::store_api::{CompletePart, HTTPRangeSpec, MultipartUploadResult, ObjectIO, ObjectOptions, PutObjReader}; +use rustfs_ecstore::store_api::{ + ChunkNativePutData, CompletePart, HTTPRangeSpec, MultipartUploadResult, ObjectIO, ObjectOptions, +}; use rustfs_ecstore::store_api::{MultipartOperations, ObjectOperations}; use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; -use rustfs_rio::{CompressReader, HashReader}; +use rustfs_object_io::put::PutObjectChecksums; +use rustfs_rio::{CompressReader, HashReader, Reader, WarpReader}; use rustfs_s3_common::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; @@ -719,6 +722,13 @@ impl DefaultMultipartUsecase { .map_err(ApiError::from)?; let mut size = size.ok_or_else(|| s3_error!(UnexpectedContent))?; + let mut requested_checksum_type = rustfs_rio::ChecksumType::from_header(&req.headers); + if !requested_checksum_type.is_set() + && let Some(checksum_algo) = fi.user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM) + && let Some(checksum_type) = fi.user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE) + { + requested_checksum_type = rustfs_rio::ChecksumType::from_string_with_obj_type(checksum_algo, checksum_type); + } // Apply adaptive buffer sizing based on part size for optimal streaming performance. // Uses workload profile configuration (enabled by default) to select appropriate buffer size. @@ -731,6 +741,8 @@ impl DefaultMultipartUsecase { let is_compressible = rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); + let mut reader: Box = Box::new(WarpReader::new(body)); + let actual_size = size; let mut md5hex = if let Some(base64_md5) = input.content_md5 { @@ -744,31 +756,32 @@ impl DefaultMultipartUsecase { let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query()); - let mut reader = if is_compressible { - let mut hrd = HashReader::from_stream(body, size, actual_size, md5hex.take(), sha256hex.take(), false) - .map_err(ApiError::from)?; + if is_compressible { + let mut hrd = + HashReader::new(reader, size, actual_size, md5hex.take(), sha256hex.take(), false).map_err(ApiError::from)?; if let Err(err) = hrd.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) { return Err(ApiError::from(err).into()); } + if requested_checksum_type.is_set() && hrd.checksum().is_none() { + hrd.enable_auto_checksum(requested_checksum_type).map_err(ApiError::from)?; + } + let compress_reader = CompressReader::new(hrd, CompressionAlgorithm::default()); + reader = Box::new(compress_reader); size = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - size, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? - } else { - HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? - }; + md5hex = None; + sha256hex = None; + } + + let mut reader = HashReader::new(reader, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?; if let Err(err) = reader.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), size < 0) { return Err(ApiError::from(err).into()); } + if requested_checksum_type.is_set() && reader.checksum().is_none() { + reader.enable_auto_checksum(requested_checksum_type).map_err(ApiError::from)?; + } let has_ssec = sse_customer_algorithm.is_some(); // When SSE-C headers are present, skip managed-encryption metadata to avoid @@ -818,9 +831,8 @@ impl DefaultMultipartUsecase { let requested_kms_key_id = material.kms_key_id.clone(); let encrypted_reader = material.wrap_reader(reader); - reader = - HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) - .map_err(ApiError::from)?; + reader = HashReader::new(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) + .map_err(ApiError::from)?; fi.user_defined.extend(material.metadata); @@ -829,18 +841,20 @@ impl DefaultMultipartUsecase { None => (None, None), }; - let mut reader = PutObjReader::new(reader); + let mut reader = ChunkNativePutData::new(reader); let info = store .put_object_part(&bucket, &key, &upload_id, part_id, &mut reader, &opts) .await .map_err(ApiError::from)?; - let mut checksum_crc32 = input.checksum_crc32; - let mut checksum_crc32c = input.checksum_crc32c; - let mut checksum_sha1 = input.checksum_sha1; - let mut checksum_sha256 = input.checksum_sha256; - let mut checksum_crc64nvme = input.checksum_crc64nvme; + let mut checksums = PutObjectChecksums { + crc32: input.checksum_crc32, + crc32c: input.checksum_crc32c, + sha1: input.checksum_sha1, + sha256: input.checksum_sha256, + crc64nvme: input.checksum_crc64nvme, + }; if let Some(alg) = &input.checksum_algorithm && let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| { @@ -860,25 +874,26 @@ impl DefaultMultipartUsecase { }) { match alg.as_str() { - ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str, - ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str, - ChecksumAlgorithm::SHA1 => checksum_sha1 = checksum_str, - ChecksumAlgorithm::SHA256 => checksum_sha256 = checksum_str, - ChecksumAlgorithm::CRC64NVME => checksum_crc64nvme = checksum_str, + ChecksumAlgorithm::CRC32 => checksums.crc32 = checksum_str, + ChecksumAlgorithm::CRC32C => checksums.crc32c = checksum_str, + ChecksumAlgorithm::SHA1 => checksums.sha1 = checksum_str, + ChecksumAlgorithm::SHA256 => checksums.sha256 = checksum_str, + ChecksumAlgorithm::CRC64NVME => checksums.crc64nvme = checksum_str, _ => (), } } + checksums.merge_from_map(&reader.content_crc()); let output = UploadPartOutput { server_side_encryption: requested_sse, ssekms_key_id: requested_kms_key_id, sse_customer_algorithm, sse_customer_key_md5, - checksum_crc32, - checksum_crc32c, - checksum_sha1, - checksum_sha256, - checksum_crc64nvme, + checksum_crc32: checksums.crc32, + checksum_crc32c: checksums.crc32c, + checksum_sha1: checksums.sha1, + checksum_sha256: checksums.sha256, + checksum_crc64nvme: checksums.crc64nvme, e_tag: info.etag.map(|etag| to_s3s_etag(&etag)), ..Default::default() }; @@ -1116,6 +1131,8 @@ impl DefaultMultipartUsecase { let is_compressible = rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); + let mut reader: Box = Box::new(WarpReader::new(src_stream)); + let src_decryption_request = DecryptionRequest { bucket: &src_bucket, key: &src_key, @@ -1127,74 +1144,23 @@ impl DefaultMultipartUsecase { etag: src_info.etag.as_deref(), }; + if let Some(material) = sse_decryption(src_decryption_request).await? { + reader = material.wrap_single_reader(reader); + if let Some(original) = material.original_size { + src_info.actual_size = original; + } + } + let actual_size = length; let mut size = length; - let mut reader = match sse_decryption(src_decryption_request).await? { - Some(material) => { - if let Some(original) = material.original_size { - src_info.actual_size = original; - } + if is_compressible { + let hrd = HashReader::new(reader, size, actual_size, None, None, false).map_err(ApiError::from)?; + reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default())); + size = HashReader::SIZE_PRESERVE_LAYER; + } - if material.is_multipart { - let (decrypted_stream, plaintext_size) = - material.wrap_reader(src_stream, size).await.map_err(ApiError::from)?; - size = plaintext_size; - - if is_compressible { - let hrd = HashReader::from_reader(decrypted_stream, size, actual_size, None, None, false) - .map_err(ApiError::from)?; - size = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - size, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? - } else { - HashReader::from_reader(decrypted_stream, size, actual_size, None, None, false).map_err(ApiError::from)? - } - } else if is_compressible { - let hrd = - HashReader::from_stream(material.wrap_single_reader(src_stream), size, actual_size, None, None, false) - .map_err(ApiError::from)?; - size = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - size, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? - } else { - HashReader::from_stream(material.wrap_single_reader(src_stream), size, actual_size, None, None, false) - .map_err(ApiError::from)? - } - } - None => { - if is_compressible { - let hrd = - HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)?; - size = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - size, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? - } else { - HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)? - } - } - }; + let mut reader = HashReader::new(reader, size, actual_size, None, None, false).map_err(ApiError::from)?; let server_side_encryption = mp_info .user_defined @@ -1235,9 +1201,8 @@ impl DefaultMultipartUsecase { let requested_kms_key_id = material.kms_key_id.clone(); let encrypted_reader = material.wrap_reader(reader); - reader = - HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) - .map_err(ApiError::from)?; + reader = HashReader::new(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) + .map_err(ApiError::from)?; mp_info.user_defined.extend(material.metadata); @@ -1246,7 +1211,7 @@ impl DefaultMultipartUsecase { None => (None, None), }; - let mut reader = PutObjReader::new(reader); + let mut reader = ChunkNativePutData::new(reader); let dst_opts = ObjectOptions { user_defined: mp_info.user_defined.clone(), diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index efebe94e4..c94347b5f 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -14,22 +14,31 @@ //! Object application use-case contracts. +mod app_adapters; +mod get_object_flow; +mod get_object_zero_copy; +mod put_object_extract; +mod put_object_flow; +mod types; +#[cfg(test)] +mod zero_copy_tests; +use self::app_adapters::*; +use self::get_object_flow::{GetObjectBootstrap, GetObjectFlowRuntime}; +use self::types::*; + use crate::app::context::{AppContext, default_notify_interface, get_global_app_context}; use crate::capacity::capacity_manager::get_capacity_manager; use crate::config::RustFSBufferConfig; use crate::error::ApiError; use crate::storage::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut}; -use crate::storage::concurrency::{ - CachedGetObject, ConcurrencyManager, GetObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, -}; +use crate::storage::concurrency::{CachedGetObject, ConcurrencyManager, GetObjectGuard, get_concurrency_manager}; use crate::storage::ecfs::*; use crate::storage::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children}; -use crate::storage::helper::{OperationHelper, spawn_background, spawn_background_with_context}; +use crate::storage::helper::OperationHelper; use crate::storage::options::{ copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime_with_object_name, filter_object_metadata, get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, put_opts, }; -use crate::storage::request_context::spawn_traced; use crate::storage::s3_api::multipart::parse_list_parts_params; use crate::storage::s3_api::{acl, restore, select}; use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig}; @@ -43,8 +52,6 @@ use http::{HeaderMap, HeaderValue, StatusCode}; use md5::Context as Md5Context; use metrics::{counter, histogram}; use pin_project_lite::pin_project; -// Performance metrics recording (with zero-copy-metrics integration) -use rustfs_concurrency::GetObjectQueueSnapshot; use rustfs_ecstore::bucket::quota::checker::QuotaChecker; use rustfs_ecstore::bucket::{ lifecycle::{ @@ -77,8 +84,8 @@ use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object use rustfs_ecstore::new_object_layer_fn; use rustfs_ecstore::set_disk::is_valid_storage_class; use rustfs_ecstore::store_api::{ - BucketOperations, BucketOptions, HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, - PutObjReader, + BucketOperations, BucketOptions, ChunkNativePutData, HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, + ObjectToDelete, }; use rustfs_filemeta::{ REPLICATE_INCOMING_DELETE, ReplicationStatusType, ReplicationType, RestoreStatusOps, VersionPurgeStatusType, @@ -86,26 +93,22 @@ use rustfs_filemeta::{ }; use rustfs_io_metrics; use rustfs_notify::EventArgsBuilder; -use rustfs_policy::policy::action::{Action, S3Action}; -use rustfs_rio::{CompressReader, DynReader, HashReader, wrap_reader}; -use rustfs_s3_common::S3Operation; -use rustfs_s3select_api::{ - object_store::bytes_stream, - query::{Context, Query}, +use rustfs_object_io::put::{ + is_post_object_sse_kms_requested, is_put_object_extract_requested, is_sse_kms_requested, resolve_put_body_size, }; +use rustfs_policy::policy::action::{Action, S3Action}; +use rustfs_rio::{CompressReader, EtagReader, HashReader, Reader, WarpReader}; +use rustfs_s3_common::S3Operation; +use rustfs_s3select_api::query::{Context, Query}; use rustfs_s3select_query::get_global_db; use rustfs_targets::EventName; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, headers::{ - AMZ_DECODED_CONTENT_LENGTH, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, AMZ_MINIO_SNOWBALL_PREFIX, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, - AMZ_RESTORE_REQUEST_DATE, AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, AMZ_RUSTFS_SNOWBALL_PREFIX, - AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, - AMZ_SNOWBALL_EXTRACT, AMZ_SNOWBALL_IGNORE_DIRS, AMZ_SNOWBALL_IGNORE_ERRORS, AMZ_SNOWBALL_PREFIX, AMZ_STORAGE_CLASS, - AMZ_TAG_COUNT, + AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, }, insert_str, remove_str, }; @@ -119,7 +122,6 @@ use s3s::dto::*; use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use std::collections::HashMap; -use std::convert::Infallible; use std::ops::Add; use std::path::Path; use std::str::FromStr; @@ -131,7 +133,7 @@ use tokio::sync::RwLock; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tokio_tar::Archive; -use tokio_util::io::{ReaderStream, StreamReader}; +use tokio_util::io::StreamReader; use tracing::{debug, error, info, instrument, warn}; use uuid::Uuid; @@ -155,70 +157,6 @@ impl Drop for DeadlockRequestGuard { } } -struct GetObjectBootstrap { - timeout_config: TimeoutConfig, - wrapper: RequestTimeoutWrapper, - request_start: std::time::Instant, - request_guard: GetObjectGuard, - _deadlock_request_guard: DeadlockRequestGuard, - concurrent_requests: usize, -} - -struct GetObjectIoPlanning<'a> { - _disk_permit: tokio::sync::SemaphorePermit<'a>, - permit_wait_duration: Duration, - queue_status: concurrency::IoQueueStatus, - queue_utilization: f64, -} - -struct GetObjectRequestContext { - bucket: String, - key: String, - cache_key: String, - version_id_for_event: String, - part_number: Option, - rs: Option, - opts: ObjectOptions, -} - -struct GetObjectReadSetup { - info: ObjectInfo, - event_info: ObjectInfo, - final_stream: DynReader, - rs: Option, - content_type: Option, - last_modified: Option, - response_content_length: i64, - content_range: Option, - server_side_encryption: Option, - sse_customer_algorithm: Option, - sse_customer_key_md5: Option, - ssekms_key_id: Option, - encryption_applied: bool, -} - -struct GetObjectPreparedRead<'a> { - io_planning: GetObjectIoPlanning<'a>, - read_setup: GetObjectReadSetup, -} - -struct GetObjectStrategyContext { - io_strategy: concurrency::IoStrategy, - optimal_buffer_size: usize, -} - -struct GetObjectCachedHit { - output: GetObjectOutput, - event_info: ObjectInfo, -} - -struct GetObjectOutputContext { - output: GetObjectOutput, - event_info: ObjectInfo, - response_content_length: i64, - optimal_buffer_size: usize, -} - async fn enqueue_transitioned_delete_cleanup(bucket: &str, object: &str, opts: &ObjectOptions, existing: Option<&ObjectInfo>) { let Some(existing) = existing else { return; @@ -303,61 +241,6 @@ impl AsyncRead for ExtractArchiveEtagReader { } } -/// Determine if zero-copy write should be used for this PutObject operation. -/// -/// Zero-copy is beneficial for large objects without encryption or compression. -/// -/// # Arguments -/// -/// * `size` - Object size in bytes -/// * `headers` - HTTP headers (to check for encryption/compression) -/// -/// # Returns -/// -/// `true` if zero-copy should be used, `false` otherwise -fn should_use_zero_copy(size: i64, headers: &HeaderMap) -> bool { - // Only use zero-copy for objects larger than 1MB - const ZERO_COPY_MIN_SIZE: i64 = 1024 * 1024; - - if size <= ZERO_COPY_MIN_SIZE { - return false; - } - - // Don't use zero-copy if encryption is requested - if headers.get(AMZ_SERVER_SIDE_ENCRYPTION).is_some() - || headers.get(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM).is_some() - || headers.get(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID).is_some() - { - return false; - } - - // Don't use zero-copy if compression is likely (compressible content types) - // The compression check happens later in the flow - if let Some(content_type) = headers.get(CONTENT_TYPE) - && let Ok(ct) = content_type.to_str() - { - // Skip zero-copy for easily compressible content types - // since compression will be applied - let compressible_types = [ - "text/plain", - "text/html", - "text/css", - "text/javascript", - "application/javascript", - "application/json", - "application/xml", - "text/xml", - ]; - for ct_type in compressible_types { - if ct.contains(ct_type) { - return false; - } - } - } - - true -} - #[cfg(test)] mod deadlock_request_guard_tests { use super::DeadlockRequestGuard; @@ -387,60 +270,6 @@ async fn maybe_enqueue_transition_immediate(obj_info: &ObjectInfo, src: LcEventS enqueue_transition_immediate(obj_info, src).await; } -/// Extract trailing-header checksum values, overriding the corresponding input fields. -fn apply_trailing_checksums( - algorithm: Option<&str>, - trailing_headers: &Option, - checksums: &mut PutObjectChecksums, -) { - let Some(alg) = algorithm else { return }; - let Some(checksum_str) = trailing_headers.as_ref().and_then(|trailer| { - let key = match alg { - ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(), - ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(), - ChecksumAlgorithm::SHA1 => rustfs_rio::ChecksumType::SHA1.key(), - ChecksumAlgorithm::SHA256 => rustfs_rio::ChecksumType::SHA256.key(), - ChecksumAlgorithm::CRC64NVME => rustfs_rio::ChecksumType::CRC64_NVME.key(), - _ => return None, - }; - trailer.read(|headers| { - headers - .get(key.unwrap_or_default()) - .and_then(|value| value.to_str().ok().map(|s| s.to_string())) - }) - }) else { - return; - }; - - match alg { - ChecksumAlgorithm::CRC32 => checksums.crc32 = checksum_str, - ChecksumAlgorithm::CRC32C => checksums.crc32c = checksum_str, - ChecksumAlgorithm::SHA1 => checksums.sha1 = checksum_str, - ChecksumAlgorithm::SHA256 => checksums.sha256 = checksum_str, - ChecksumAlgorithm::CRC64NVME => checksums.crc64nvme = checksum_str, - _ => (), - } -} - -#[derive(Default)] -struct GetObjectChecksums { - crc32: Option, - crc32c: Option, - sha1: Option, - sha256: Option, - crc64nvme: Option, - checksum_type: Option, -} - -#[derive(Default)] -struct PutObjectChecksums { - crc32: Option, - crc32c: Option, - sha1: Option, - sha256: Option, - crc64nvme: Option, -} - fn normalize_delete_objects_version_id(version_id: Option) -> Result<(Option, Option), String> { let version_id = version_id.map(|v| v.trim().to_string()).filter(|v| !v.is_empty()); match version_id { @@ -471,148 +300,6 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option, - ignore_dirs: bool, - ignore_errors: bool, -} - -fn header_value_is_true(headers: &HeaderMap, key: &str) -> bool { - headers - .get(key) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.trim().eq_ignore_ascii_case("true")) -} - -fn is_put_object_extract_requested(headers: &HeaderMap) -> bool { - header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_COMPAT) -} - -fn trimmed_header_value(headers: &HeaderMap, key: &str) -> Option { - headers - .get(key) - .and_then(|value| value.to_str().ok()) - .map(|value| value.trim().to_string()) -} - -fn is_exact_snowball_meta_key(key: &str, exact_keys: &[&str]) -> bool { - exact_keys.iter().any(|exact_key| key.eq_ignore_ascii_case(exact_key)) -} - -fn snowball_meta_value_by_suffix(headers: &HeaderMap, suffix_lower: &str, exact_keys: &[&str]) -> Option { - for (name, value) in headers { - let key = name.as_str(); - if key.starts_with(AMZ_META_PREFIX_LOWER) - && key.ends_with(suffix_lower) - && !is_exact_snowball_meta_key(key, exact_keys) - && let Ok(parsed) = value.to_str() - { - return Some(parsed.trim().to_string()); - } - } - - None -} - -fn snowball_meta_value(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> Option { - for key in exact_keys { - if let Some(value) = trimmed_header_value(headers, key) { - return Some(value); - } - } - - snowball_meta_value_by_suffix(headers, suffix_lower, exact_keys) -} - -fn snowball_meta_flag(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> bool { - snowball_meta_value(headers, exact_keys, suffix_lower).is_some_and(|value| value.eq_ignore_ascii_case("true")) -} - -fn normalize_snowball_prefix(prefix: &str) -> Option { - let normalized = prefix.trim().trim_matches('/'); - if normalized.is_empty() { - return None; - } - - Some(normalized.to_string()) -} - -fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: bool) -> String { - let path = path.trim_matches('/'); - let mut key = match prefix { - Some(prefix) if !path.is_empty() => format!("{prefix}/{path}"), - Some(prefix) => prefix.to_string(), - None => path.to_string(), - }; - - if is_dir && !key.ends_with('/') { - key.push('/'); - } - - key -} - -fn map_extract_archive_error(err: impl std::fmt::Display) -> S3Error { - s3_error!(InvalidArgument, "Failed to process archive entry: {}", err) -} - -async fn apply_extract_entry_pax_extensions( - entry: &mut tokio_tar::Entry>, - metadata: &mut HashMap, - opts: &mut ObjectOptions, -) -> S3Result<()> -where - R: AsyncRead + Send + Unpin + 'static, -{ - let Some(extensions) = entry.pax_extensions().await.map_err(map_extract_archive_error)? else { - return Ok(()); - }; - - for ext in extensions { - let ext = ext.map_err(map_extract_archive_error)?; - let key = ext.key().map_err(map_extract_archive_error)?; - let value = ext.value().map_err(map_extract_archive_error)?; - - if let Some(meta_key) = key.strip_prefix("minio.metadata.") { - let meta_key = meta_key.strip_prefix("x-amz-meta-").unwrap_or(meta_key); - if !meta_key.is_empty() { - metadata.insert(meta_key.to_string(), value.to_string()); - } - continue; - } - - if key == "minio.versionId" && !value.is_empty() { - opts.version_id = Some(value.to_string()); - } - } - - Ok(()) -} - #[allow(clippy::too_many_arguments)] fn apply_put_request_metadata( metadata: &mut HashMap, @@ -643,7 +330,7 @@ fn apply_put_request_metadata( metadata.insert("content-language".to_string(), content_language.to_string()); } if let Some(content_type) = content_type { - metadata.insert("content-type".to_string(), content_type.to_string()); + metadata.insert(CONTENT_TYPE.to_string(), content_type.to_string()); } if let Some(expires) = expires { let mut formatted = Vec::new(); @@ -831,36 +518,6 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool { opts.version_id.is_none() && opts.versioned && !opts.version_suspended } -fn resolve_put_object_extract_options(headers: &HeaderMap) -> PutObjectExtractOptions { - let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER) - .and_then(|value| normalize_snowball_prefix(&value)); - let ignore_dirs = snowball_meta_flag(headers, SNOWBALL_IGNORE_DIRS_HEADER_KEYS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER); - let ignore_errors = snowball_meta_flag(headers, SNOWBALL_IGNORE_ERRORS_HEADER_KEYS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER); - - PutObjectExtractOptions { - prefix, - ignore_dirs, - ignore_errors, - } -} - -fn is_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool { - input - .server_side_encryption - .as_ref() - .is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) - || input.ssekms_key_id.is_some() - || headers - .get(AMZ_SERVER_SIDE_ENCRYPTION) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.trim().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) - || headers.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID) -} - -fn is_post_object_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool { - is_sse_kms_requested(input, headers) -} - async fn resolve_put_object_expiration(bucket: &str, obj_info: &ObjectInfo) -> Option { let Ok((lifecycle_config, _)) = metadata_sys::get_lifecycle_config(bucket).await else { debug!("resolve_put_object_expiration: lifecycle config not found for bucket {bucket}"); @@ -929,711 +586,22 @@ impl DefaultObjectUsecase { fn spawn_cache_invalidation(bucket: String, key: String, version_id: Option) { let manager = get_concurrency_manager(); - spawn_traced(async move { + crate::storage::request_context::spawn_traced(async move { manager.invalidate_cache_versioned(&bucket, &key, version_id.as_deref()).await; }); } - fn build_cached_get_object_output(cached: &CachedGetObject) -> GetObjectOutput { - let body_data = cached.body.clone(); - let body = Some(StreamingBlob::wrap::<_, Infallible>(futures::stream::once(async move { - Ok((*body_data).clone()) - }))); - - let last_modified = cached - .last_modified - .as_ref() - .and_then(|s| match OffsetDateTime::parse(s, &Rfc3339) { - Ok(dt) => Some(Timestamp::from(dt)), - Err(e) => { - warn!("Failed to parse cached last_modified '{}': {}", s, e); - None - } - }); - - let content_type = cached.content_type.as_ref().and_then(|ct| ContentType::from_str(ct).ok()); - - GetObjectOutput { - body, - content_length: Some(cached.content_length), - accept_ranges: Some("bytes".to_string()), - e_tag: cached.e_tag.as_ref().map(|etag| to_s3s_etag(etag)), - last_modified, - content_type, - cache_control: cached.cache_control.clone(), - content_disposition: cached.content_disposition.clone(), - content_encoding: cached.content_encoding.clone(), - content_language: cached.content_language.clone(), - version_id: cached.version_id.clone(), - delete_marker: Some(cached.delete_marker), - tag_count: cached.tag_count, - metadata: if cached.user_metadata.is_empty() { - None - } else { - Some(cached.user_metadata.clone()) - }, - ..Default::default() - } - } - - fn build_cached_get_object_event_info(bucket: &str, key: &str, cached: &CachedGetObject) -> ObjectInfo { - ObjectInfo { - bucket: bucket.to_string(), - name: key.to_string(), - storage_class: cached.storage_class.clone(), - mod_time: cached - .last_modified - .as_ref() - .and_then(|s| OffsetDateTime::parse(s, &Rfc3339).ok()), - size: cached.content_length, - actual_size: cached.content_length, - is_dir: false, - user_defined: cached.user_metadata.clone(), - version_id: cached.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok()), - delete_marker: cached.delete_marker, - content_type: cached.content_type.clone(), - content_encoding: cached.content_encoding.clone(), - etag: cached.e_tag.clone(), - ..Default::default() - } - } - - fn build_memory_blob(buf: Vec, response_content_length: i64, optimal_buffer_size: usize) -> Option { - let mem_reader = InMemoryAsyncReader::new(buf); - Some(StreamingBlob::wrap(bytes_stream( - ReaderStream::with_capacity(Box::new(mem_reader), optimal_buffer_size), - response_content_length as usize, - ))) - } - - fn build_reader_blob(reader: R, response_content_length: i64, optimal_buffer_size: usize) -> Option - where - R: AsyncRead + Send + Sync + 'static, - { - Some(StreamingBlob::wrap(bytes_stream( - ReaderStream::with_capacity(reader, optimal_buffer_size), - response_content_length as usize, - ))) - } - - fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result { - let timeout_config = TimeoutConfig::from_env(); - let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.to_string()); - let request_start = std::time::Instant::now(); - let request_guard = ConcurrencyManager::track_request(); - let concurrent_requests = GetObjectGuard::concurrent_requests(); - - let deadlock_detector = deadlock_detector::get_deadlock_detector(); - let request_id = wrapper.request_id().to_string(); - deadlock_detector.register_request(&request_id, format!("GetObject {bucket}/{key}")); - let deadlock_request_guard = DeadlockRequestGuard::new(deadlock_detector, request_id); - - if wrapper.is_timeout() { - warn!( - bucket = %bucket, - key = %key, - timeout_secs = timeout_config.get_object_timeout.as_secs(), - elapsed_ms = wrapper.elapsed().as_millis(), - "GetObject request timed out before processing" - ); - return Err(s3_error!(InternalError, "Request timeout before processing")); - } - - rustfs_io_metrics::record_get_object_request_start(concurrent_requests); - - debug!( - "GetObject request started with {} concurrent requests, timeout={:?}", - concurrent_requests, timeout_config.get_object_timeout - ); - - Ok(GetObjectBootstrap { - timeout_config, - wrapper, - request_start, - request_guard, - _deadlock_request_guard: deadlock_request_guard, - concurrent_requests, - }) - } - - async fn acquire_get_object_io_planning<'a>( - manager: &'a ConcurrencyManager, - wrapper: &RequestTimeoutWrapper, - timeout_config: &TimeoutConfig, - bucket: &str, - key: &str, - ) -> S3Result> { - let permit_wait_start = std::time::Instant::now(); - let disk_permit = manager - .acquire_disk_read_permit() - .await - .map_err(|_| s3_error!(InternalError, "disk read semaphore closed"))?; - let permit_wait_duration = permit_wait_start.elapsed(); - - if wrapper.is_timeout() { - warn!( - bucket = %bucket, - key = %key, - wait_ms = permit_wait_duration.as_millis(), - timeout_secs = timeout_config.get_object_timeout.as_secs(), - elapsed_ms = wrapper.elapsed().as_millis(), - "GetObject request timed out while waiting for disk permit" - ); - - rustfs_io_metrics::record_get_object_timeout(Some("disk_permit"), Some(wrapper.elapsed().as_secs_f64())); - return Err(s3_error!(InternalError, "Request timeout while waiting for disk permit")); - } - - let queue_status = manager.io_queue_status(); - let queue_snapshot = GetObjectQueueSnapshot::from_available_permits( - queue_status.total_permits, - queue_status.total_permits.saturating_sub(queue_status.permits_in_use), - ); - let queue_utilization = queue_snapshot.utilization_percent(); - - if queue_snapshot.is_congested(80.0) { - warn!( - bucket = %bucket, - key = %key, - queue_utilization = format!("{:.1}%", queue_utilization), - permits_in_use = queue_status.permits_in_use, - total_permits = queue_status.total_permits, - "I/O queue congestion detected" - ); - - rustfs_io_metrics::record_io_queue_congestion(); - } - - if wrapper.is_timeout() { - warn!( - bucket = %bucket, - key = %key, - timeout_secs = timeout_config.get_object_timeout.as_secs(), - elapsed_ms = wrapper.elapsed().as_millis(), - "GetObject request timed out before reading object" - ); - rustfs_io_metrics::record_get_object_timeout(Some("before_read"), Some(wrapper.elapsed().as_secs_f64())); - return Err(s3_error!(InternalError, "Request timeout before reading object")); - } - - Ok(GetObjectIoPlanning { - _disk_permit: disk_permit, - permit_wait_duration, - queue_status, - queue_utilization, - }) - } - - async fn prepare_get_object_request_context(req: &S3Request) -> S3Result { - let GetObjectInput { - bucket, - key, - version_id, - part_number, - range, - .. - } = req.input.clone(); - - validate_object_key(&key, "GET")?; - - let part_number = part_number.map(|v| v as usize); - - if let Some(part_num) = part_number - && part_num == 0 - { - return Err(s3_error!(InvalidArgument, "Invalid part number: part number must be greater than 0")); - } - - let rs = range.map(|v| match v { - Range::Int { first, last } => HTTPRangeSpec { - is_suffix_length: false, - start: first as i64, - end: if let Some(last) = last { last as i64 } else { -1 }, - }, - Range::Suffix { length } => HTTPRangeSpec { - is_suffix_length: true, - start: length as i64, - end: -1, - }, - }); - - if rs.is_some() && part_number.is_some() { - return Err(s3_error!(InvalidArgument, "range and part_number invalid")); - } - - let opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), part_number, &req.headers) - .await - .map_err(ApiError::from)?; - - Ok(GetObjectRequestContext { - cache_key: ConcurrencyManager::make_cache_key(&bucket, &key, version_id.as_deref()), - version_id_for_event: version_id.unwrap_or_default(), - bucket, - key, - part_number, - rs, - opts, - }) - } - #[allow(clippy::too_many_arguments)] - async fn prepare_get_object_read_execution<'a>( - req: &S3Request, - manager: &'a ConcurrencyManager, - wrapper: &RequestTimeoutWrapper, - timeout_config: &TimeoutConfig, - bucket: &str, - key: &str, - rs: Option, - opts: &ObjectOptions, - part_number: Option, - ) -> S3Result> { - let h = HeaderMap::new(); - let io_planning = Self::acquire_get_object_io_planning(manager, wrapper, timeout_config, bucket, key).await?; - let store = get_validated_store(bucket).await?; - - let read_start = std::time::Instant::now(); - let read_setup = - Self::prepare_get_object_read(req, &store, manager, bucket, key, rs, h, opts, part_number, read_start).await?; - - Ok(GetObjectPreparedRead { io_planning, read_setup }) - } - - #[allow(clippy::too_many_arguments)] - async fn prepare_get_object_read( - req: &S3Request, - store: &rustfs_ecstore::store::ECStore, - manager: &ConcurrencyManager, - bucket: &str, - key: &str, - mut rs: Option, - h: HeaderMap, - opts: &ObjectOptions, - part_number: Option, - read_start: std::time::Instant, - ) -> S3Result { - let reader = store - .get_object_reader(bucket, key, rs.clone(), h, opts) - .await - .map_err(ApiError::from)?; - - let info = reader.object_info; - - use rustfs_io_metrics::{record_memory_copy_saved, record_zero_copy_read}; - let read_duration = read_start.elapsed(); - let estimated_saved = (info.size * 2) as usize; - record_zero_copy_read(info.size as usize, read_duration.as_secs_f64() * 1000.0); - record_memory_copy_saved(estimated_saved); - - manager.record_disk_operation(info.size as u64, read_duration, true).await; - - check_preconditions(&req.headers, &info)?; - - debug!(object_size = info.size, part_count = info.parts.len(), "GET object metadata snapshot"); - for part in &info.parts { - debug!( - part_number = part.number, - part_size = part.size, - part_actual_size = part.actual_size, - "GET object part details" - ); - } - - let event_info = info.clone(); - let content_type = if let Some(content_type) = &info.content_type { - match ContentType::from_str(content_type) { - Ok(res) => Some(res), - Err(err) => { - error!("parse content-type err {} {:?}", content_type, err); - None - } - } - } else { - None - }; - let last_modified = info.mod_time.map(Timestamp::from); - - if let Some(part_number) = part_number - && rs.is_none() - { - rs = HTTPRangeSpec::from_object_info(&info, part_number); - } - - validate_sse_headers_for_read(&info.user_defined, &req.headers)?; - - let mut content_length = info.get_actual_size().map_err(ApiError::from)?; - let content_range = if let Some(rs) = &rs { - let total_size = content_length; - let (start, length) = rs.get_offset_length(total_size).map_err(ApiError::from)?; - content_length = length; - Some(format!("bytes {}-{}/{}", start, start as i64 + length - 1, total_size)) - } else { - None - }; - - debug!( - "GET object metadata check: parts={}, provided_sse_key={:?}", - info.parts.len(), - req.input.sse_customer_key.is_some() - ); - - let decryption_request = DecryptionRequest { - bucket, - key, - metadata: &info.user_defined, - sse_customer_key: req.input.sse_customer_key.as_ref(), - sse_customer_key_md5: req.input.sse_customer_key_md5.as_ref(), - part_number: None, - parts: &info.parts, - etag: info.etag.as_deref(), - }; - - let mut response_content_length = content_length; - let encrypted_stream = reader.stream; - - let ( - server_side_encryption, - sse_customer_algorithm, - sse_customer_key_md5, - ssekms_key_id, - encryption_applied, - final_stream, - ) = match sse_decryption(decryption_request).await? { - Some(material) => { - let server_side_encryption = Some(material.server_side_encryption.clone()); - let sse_customer_algorithm = Some(material.algorithm.clone()); - let sse_customer_key_md5 = material.customer_key_md5.clone(); - let ssekms_key_id = material.kms_key_id.clone(); - - let (decrypted_stream, plaintext_size) = material - .wrap_reader(encrypted_stream, content_length) - .await - .map_err(ApiError::from)?; - - response_content_length = plaintext_size; - - ( - server_side_encryption, - sse_customer_algorithm, - sse_customer_key_md5, - ssekms_key_id, - true, - decrypted_stream, - ) - } - None => (None, None, None, None, false, wrap_reader(encrypted_stream)), - }; - - Ok(GetObjectReadSetup { - info, - event_info, - final_stream, - rs, - content_type, - last_modified, - response_content_length, - content_range, - server_side_encryption, - sse_customer_algorithm, - sse_customer_key_md5, - ssekms_key_id, - encryption_applied, - }) - } - #[allow(clippy::too_many_arguments)] - fn finalize_get_object_strategy( - &self, - manager: &ConcurrencyManager, - bucket: &str, - key: &str, - info: &ObjectInfo, - rs: Option<&HTTPRangeSpec>, - response_content_length: i64, - permit_wait_duration: Duration, - queue_utilization: f64, - queue_status: &concurrency::IoQueueStatus, - concurrent_requests: usize, - ) -> GetObjectStrategyContext { - let base_buffer_size = self.base_buffer_size(); - - let is_sequential_hint = if rs.is_none() { - true - } else if let Some(range_spec) = rs { - range_spec.start == 0 && !range_spec.is_suffix_length - } else { - false - }; - - if let Some(range_spec) = rs - && range_spec.start >= 0 - { - manager.record_access(range_spec.start as u64, response_content_length as u64); - } - - if response_content_length > 0 { - manager.record_transfer(response_content_length as u64, permit_wait_duration); - } - - let io_strategy = - manager.calculate_io_strategy_with_context(info.size, base_buffer_size, permit_wait_duration, is_sequential_hint); - - debug!( - wait_ms = permit_wait_duration.as_millis() as u64, - load_level = ?io_strategy.load_level, - buffer_size = io_strategy.buffer_size, - buffer_multiplier = io_strategy.buffer_multiplier, - readahead = io_strategy.enable_readahead, - cache_wb = io_strategy.cache_writeback_enabled, - storage_media = ?io_strategy.storage_media, - access_pattern = ?io_strategy.access_pattern, - bandwidth_tier = ?io_strategy.bandwidth_tier, - concurrent_requests = io_strategy.concurrent_requests, - file_size = info.size, - is_sequential = is_sequential_hint, - "Enhanced multi-factor I/O strategy calculated" - ); - - let io_priority = manager.get_io_priority(response_content_length); - - if manager.is_priority_scheduling_enabled() { - debug!( - bucket = %bucket, - key = %key, - priority = %io_priority, - request_size = response_content_length, - "I/O priority assigned (based on actual request size)" - ); - - rustfs_io_metrics::record_io_priority_assignment(io_priority.as_str()); - } - - rustfs_io_metrics::record_get_object_io_state( - permit_wait_duration.as_secs_f64(), - queue_utilization, - queue_status.permits_in_use, - queue_status.total_permits.saturating_sub(queue_status.permits_in_use), - io_strategy.load_level.as_str(), - io_strategy.buffer_multiplier, - ); - rustfs_io_metrics::record_io_priority_assignment(io_priority.as_str()); - - debug!( - actual_request_size = response_content_length, - priority = %io_priority.as_str(), - "I/O priority finalized with actual request size" - ); - - let base_buffer_size = get_buffer_size_opt_in(response_content_length); - let optimal_buffer_size = if io_strategy.buffer_size > 0 { - io_strategy.buffer_size.min(base_buffer_size) - } else { - get_concurrency_aware_buffer_size(response_content_length, base_buffer_size) - }; - - debug!( - "GetObject buffer sizing: file_size={}, base={}, optimal={}, concurrent_requests={}, io_strategy={:?}", - response_content_length, base_buffer_size, optimal_buffer_size, concurrent_requests, io_strategy.load_level - ); - - GetObjectStrategyContext { - io_strategy, - optimal_buffer_size, - } - } - - fn build_get_object_checksums( - info: &ObjectInfo, - headers: &HeaderMap, - part_number: Option, - rs: Option<&HTTPRangeSpec>, - ) -> S3Result { - let mut checksums = GetObjectChecksums::default(); - - if let Some(checksum_mode) = headers.get(AMZ_CHECKSUM_MODE) - && checksum_mode.to_str().unwrap_or_default() == "ENABLED" - && rs.is_none() - { - let (decrypted_checksums, _is_multipart) = - info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| { - error!("decrypt_checksums error: {}", e); - ApiError::from(e) - })?; - - for (key, checksum) in decrypted_checksums { - if key == AMZ_CHECKSUM_TYPE { - checksums.checksum_type = Some(ChecksumType::from(checksum)); - continue; - } - - match rustfs_rio::ChecksumType::from_string(key.as_str()) { - rustfs_rio::ChecksumType::CRC32 => checksums.crc32 = Some(checksum), - rustfs_rio::ChecksumType::CRC32C => checksums.crc32c = Some(checksum), - rustfs_rio::ChecksumType::SHA1 => checksums.sha1 = Some(checksum), - rustfs_rio::ChecksumType::SHA256 => checksums.sha256 = Some(checksum), - rustfs_rio::ChecksumType::CRC64_NVME => checksums.crc64nvme = Some(checksum), - _ => (), - } - } - } - - Ok(checksums) - } - #[allow(clippy::too_many_arguments)] - async fn build_get_object_body( - mut final_stream: R, - info: &ObjectInfo, - cache_key: &str, - response_content_length: i64, - optimal_buffer_size: usize, - part_number: Option, - has_range: bool, - encryption_applied: bool, - cache_writeback_enabled: bool, - ) -> S3Result> - where - R: AsyncRead + Send + Sync + Unpin + 'static, - { - let manager = get_concurrency_manager(); - let cache_eligibility = manager.get_object_cache_eligibility( - cache_writeback_enabled, - part_number.is_some(), - has_range, - encryption_applied, - response_content_length, - ); - let should_cache = cache_eligibility.should_cache(); - - let body = if should_cache { - debug!( - "Reading object into memory for caching: key={} size={}", - cache_key, response_content_length - ); - - let mut buf = Vec::with_capacity(response_content_length as usize); - if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { - error!("Failed to read object into memory for caching: {}", e); - return Err(ApiError::from(StorageError::other(format!("Failed to read object for caching: {e}"))).into()); - } - - if buf.len() != response_content_length as usize { - warn!( - "Object size mismatch during cache read: expected={} actual={}", - response_content_length, - buf.len() - ); - } - - let last_modified_str = info.mod_time.and_then(|t| match t.format(&Rfc3339) { - Ok(s) => Some(s), - Err(e) => { - warn!("Failed to format last_modified for cache writeback: {}", e); - None - } - }); - - let cached_response = CachedGetObject::new(Bytes::from(buf.clone()), response_content_length) - .with_content_type(info.content_type.clone().unwrap_or_default()) - .with_e_tag(info.etag.clone().unwrap_or_default()) - .with_last_modified(last_modified_str.unwrap_or_default()); - - let cache_key_clone = cache_key.to_string(); - spawn_traced(async move { - let manager = get_concurrency_manager(); - manager.put_cached_object(cache_key_clone.clone(), cached_response).await; - debug!("Object cached successfully with metadata: {}", cache_key_clone); - }); - - rustfs_io_metrics::record_object_cache_writeback(); - Self::build_memory_blob(buf, response_content_length, optimal_buffer_size) - } else if encryption_applied { - let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD; - let should_buffer_encrypted_object = response_content_length > 0 - && response_content_length <= seekable_object_size_threshold as i64 - && part_number.is_none() - && !has_range; - - if should_buffer_encrypted_object { - let mut buf = Vec::with_capacity(response_content_length as usize); - if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { - error!("Failed to read decrypted object into memory: {}", e); - return Err(ApiError::from(StorageError::other(format!("Failed to read decrypted object: {e}"))).into()); - } - - if buf.len() != response_content_length as usize { - warn!( - "Encrypted object size mismatch during read: expected={} actual={}", - response_content_length, - buf.len() - ); - } - - Self::build_memory_blob(buf, response_content_length, optimal_buffer_size) - } else { - info!( - "Encrypted object: Using unlimited stream for decryption with buffer size {}", - optimal_buffer_size - ); - Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size) - } - } else { - let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD; - - let should_provide_seek_support = response_content_length > 0 - && response_content_length <= seekable_object_size_threshold as i64 - && part_number.is_none() - && !has_range; - - if should_provide_seek_support { - debug!( - "Reading small object into memory for seek support: key={} size={}", - cache_key, response_content_length - ); - - let mut buf = Vec::with_capacity(response_content_length as usize); - match tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { - Ok(_) => { - if buf.len() != response_content_length as usize { - warn!( - "Object size mismatch during seek support read: expected={} actual={}", - response_content_length, - buf.len() - ); - } - - Self::build_memory_blob(buf, response_content_length, optimal_buffer_size) - } - Err(e) => { - error!("Failed to read object into memory for seek support: {}", e); - Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size) - } - } - } else { - Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size) - } - }; - - Ok(body) - } - - fn put_object_execution_context(req: &S3Request) -> (EventName, QuotaOperation, &'static str) { - if req.extensions.get::().is_some() { - (EventName::ObjectCreatedPost, QuotaOperation::PostObject, "POST") - } else { - (EventName::ObjectCreatedPut, QuotaOperation::PutObject, "PUT") - } - } - #[instrument(level = "debug", skip(self, _fs, req))] pub async fn execute_put_object(&self, _fs: &FS, req: S3Request) -> S3Result> { - let start_time = std::time::Instant::now(); - let mut req = req; - if let Some(context) = &self.context { let _ = context.object_store(); } - let (event_name, quota_operation, request_method_name) = Self::put_object_execution_context(&req); - if req.extensions.get::().is_some() && is_post_object_sse_kms_requested(&req.input, &req.headers) - { + let request_context = prepare_put_object_request_context(&req); + let (event_name, quota_operation, request_method_name) = put_object_execution_context(&req); + let helper = new_operation_helper(&req, event_name, S3Operation::PutObject, false); + + if request_context.is_post_object && is_post_object_sse_kms_requested(&req.input, &request_context.headers) { return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for POST object uploads")); } if let Some(ref storage_class) = req.input.storage_class @@ -1641,380 +609,19 @@ impl DefaultObjectUsecase { { return Err(s3_error!(InvalidStorageClass)); } - if is_put_object_extract_requested(&req.headers) { + if is_put_object_extract_requested(&request_context.headers) { return self.execute_put_object_extract(req).await; } - let input = std::mem::take(&mut req.input); + let resolved_size = resolve_put_body_size(req.input.content_length, &request_context.headers)?; + self.check_bucket_quota(&req.input.bucket, quota_operation, resolved_size as u64) + .await?; - let PutObjectInput { - body, - bucket, - cache_control, - key, - content_length, - content_disposition, - content_encoding, - content_language, - content_type, - expires, - tagging, - metadata, - version_id, - server_side_encryption, - sse_customer_algorithm, - sse_customer_key, - sse_customer_key_md5, - ssekms_key_id, - content_md5, - object_lock_legal_hold_status, - object_lock_mode, - object_lock_retain_until_date, - storage_class, - website_redirect_location, - .. - } = input; - - // Merge SSE-C params from headers (fallback when S3 layer does not populate input) - let (h_algo, h_key, h_md5) = extract_ssec_params_from_headers(&req.headers)?; - let sse_customer_algorithm = sse_customer_algorithm.or(h_algo); - let sse_customer_key = sse_customer_key.or(h_key); - let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5); - - // Merge server_side_encryption from headers (fallback when S3 layer does not populate input) - let server_side_encryption = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); - - // Validate object key - validate_object_key(&key, request_method_name)?; - - if let Some(size) = content_length { - self.check_bucket_quota(&bucket, quota_operation, size as u64).await?; - } - - let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; - - let mut size = match content_length { - Some(c) => c, - None => { - if let Some(val) = req.headers.get(AMZ_DECODED_CONTENT_LENGTH) { - match atoi::atoi::(val.as_bytes()) { - Some(x) => x, - None => return Err(s3_error!(UnexpectedContent)), - } - } else { - return Err(s3_error!(UnexpectedContent)); - } - } - }; - - if size == -1 { - return Err(s3_error!(UnexpectedContent)); - } - - // Apply adaptive buffer sizing based on file size for optimal streaming performance. - // Uses workload profile configuration (enabled by default) to select appropriate buffer size. - // Buffer sizes range from 32KB to 4MB depending on file size and configured workload profile. - let buffer_size = get_buffer_size_opt_in(size); - - // Detect zero-copy opportunity before encryption/compression decisions - // Zero-copy is beneficial for large unencrypted, uncompressed objects - let enable_zero_copy = should_use_zero_copy(size, &req.headers); - - if enable_zero_copy { - // Record zero-copy write attempt - counter!("rustfs.zero_copy.write.attempts.total").increment(1); - histogram!("rustfs.zero_copy.write.size.bytes").record(size as f64); - debug!("Zero-copy write enabled for {} byte object (bucket={}, key={})", size, bucket, key); - } - - let body = tokio::io::BufReader::with_capacity( - buffer_size, - StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), - ); - - let store = get_validated_store(&bucket).await?; - - // TDD: Get bucket default encryption configuration - let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); - debug!("TDD: bucket_sse_config={:?}", bucket_sse_config); - - // TDD: Determine effective encryption configuration (request overrides bucket default) - let original_sse = server_side_encryption.clone(); - let mut effective_sse = server_side_encryption.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - debug!("TDD: Processing bucket SSE config: {:?}", config); - config.rules.first().and_then(|rule| { - debug!("TDD: Processing SSE rule: {:?}", rule); - rule.apply_server_side_encryption_by_default.as_ref().map(|sse| { - debug!("TDD: Found SSE default: {:?}", sse); - match sse.sse_algorithm.as_str() { - "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), // fallback to AES256 - } - }) - }) - }) - }); - debug!("TDD: effective_sse={:?} (original={:?})", effective_sse, original_sse); - - let mut effective_kms_key_id = ssekms_key_id.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .and_then(|sse| sse.kms_master_key_id.clone()) - }) - }) - }); - - // Validate SSE-C headers early: reject partial/invalid combinations per S3 spec - validate_sse_headers_for_write( - effective_sse.as_ref(), - effective_kms_key_id.as_ref(), - sse_customer_algorithm.as_ref(), - sse_customer_key.as_ref(), - sse_customer_key_md5.as_ref(), - true, // PutObject requires all three: algorithm, key, key_md5 - )?; - - let mut metadata = metadata.unwrap_or_default(); - apply_put_request_metadata( - &mut metadata, - &req.headers, - &key, - cache_control, - content_disposition, - content_encoding, - content_language, - content_type, - expires, - website_redirect_location, - tagging, - storage_class.clone(), - )?; - - let mut opts: ObjectOptions = put_opts(&bucket, &key, version_id.clone(), &req.headers, metadata.clone()) - .await - .map_err(ApiError::from)?; - apply_put_request_object_lock_opts( - &bucket, - object_lock_legal_hold_status, - object_lock_mode, - object_lock_retain_until_date, - &mut opts, - ) - .await?; - - let current_opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers) - .await - .map_err(ApiError::from)?; - match store.get_object_info(&bucket, &key, ¤t_opts).await { - Ok(existing_obj_info) => validate_existing_object_lock_for_write(&existing_obj_info)?, - Err(err) => { - if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { - return Err(ApiError::from(err).into()); - } - } - } - - let actual_size = size; - - let mut md5hex = if let Some(base64_md5) = content_md5 { - let md5 = base64_simd::STANDARD - .decode_to_vec(base64_md5.as_bytes()) - .map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?; - Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower)) - } else { - None - }; - - let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query()); - - let mut reader = if is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64 { - let algorithm = CompressionAlgorithm::default(); - insert_str(&mut metadata, SUFFIX_COMPRESSION, algorithm.to_string()); - insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); - - let mut hrd = - HashReader::from_stream(body, size, size, md5hex.take(), sha256hex.take(), false).map_err(ApiError::from)?; - - if let Err(err) = hrd.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) { - return Err(ApiError::from(err).into()); - } - - opts.want_checksum = hrd.checksum(); - insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, algorithm.to_string()); - insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string()); - - size = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader(CompressReader::new(hrd, algorithm), size, actual_size, None, None, false) - .map_err(ApiError::from)? - } else { - HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? - }; - - if size >= 0 { - if let Err(err) = reader.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) { - return Err(ApiError::from(err).into()); - } - - opts.want_checksum = reader.checksum(); - } - - let mut helper = OperationHelper::new(&req, event_name, S3Operation::PutObject); - - // Apply encryption using unified SSE API. - let encryption_request = EncryptionRequest { - bucket: &bucket, - key: &key, - server_side_encryption: effective_sse.clone(), - ssekms_key_id: effective_kms_key_id.clone(), - sse_customer_algorithm: sse_customer_algorithm.clone(), - sse_customer_key, - sse_customer_key_md5: sse_customer_key_md5.clone(), - content_size: actual_size, - part_number: None, - part_key: None, - part_nonce: None, - }; - - let encryption_material = match sse_encryption(encryption_request).await { - Ok(material) => material, - Err(err) => { - let result = Err(err.into()); - let _ = helper.complete(&result); - return result; - } - }; - - if let Some(material) = encryption_material { - effective_sse = Some(material.server_side_encryption.clone()); - effective_kms_key_id = material.kms_key_id.clone(); - - let encrypted_reader = material.wrap_reader(reader); - reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) - .map_err(ApiError::from)?; - - let encryption_metadata = material.metadata; - metadata.extend(encryption_metadata.clone()); - opts.user_defined.extend(encryption_metadata); - } - - let mut reader = PutObjReader::new(reader); - - let mt2 = metadata.clone(); - opts.user_defined.extend(metadata); - - let repoptions = - get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts.clone()); - - let dsc = must_replicate(&bucket, &key, repoptions).await; - - if dsc.replicate_any() { - insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); - insert_str( - &mut opts.user_defined, - SUFFIX_REPLICATION_STATUS, - dsc.pending_status().unwrap_or_default(), - ); - } - - let obj_info = match store - .put_object(&bucket, &key, &mut reader, &opts) - .await - .map_err(ApiError::from) - { - Ok(obj_info) => obj_info, - Err(err) => { - let result: S3Result> = Err(err.into()); - let _ = helper.complete(&result); - return result; - } - }; - - maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; - - // Fast in-memory update for immediate quota consistency - rustfs_ecstore::data_usage::increment_bucket_usage_memory(&bucket, obj_info.size as u64).await; - - let raw_version = obj_info.version_id.map(|v| v.to_string()); - - helper = helper.object(obj_info.clone()); - if let Some(version_id) = &raw_version { - helper = helper.version_id(version_id.clone()); - } - - Self::spawn_cache_invalidation(bucket.clone(), key.clone(), raw_version.clone()); - - let put_version = if BucketVersioningSys::prefix_enabled(&bucket, &key).await { - raw_version - } else { - None - }; - - let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)); - - let repoptions = - get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts); - - let dsc = must_replicate(&bucket, &key, repoptions).await; - let expiration = resolve_put_object_expiration(&bucket, &obj_info).await; - - if dsc.replicate_any() { - schedule_replication(obj_info.clone(), store, dsc, ReplicationType::Object).await; - } - - let mut checksums = PutObjectChecksums { - crc32: input.checksum_crc32, - crc32c: input.checksum_crc32c, - sha1: input.checksum_sha1, - sha256: input.checksum_sha256, - crc64nvme: input.checksum_crc64nvme, - }; - apply_trailing_checksums( - input.checksum_algorithm.as_ref().map(|a| a.as_str()), - &req.trailing_headers, - &mut checksums, - ); - - let output = PutObjectOutput { - e_tag, - server_side_encryption: effective_sse, - sse_customer_algorithm: sse_customer_algorithm.clone(), - sse_customer_key_md5: sse_customer_key_md5.clone(), - ssekms_key_id: effective_kms_key_id, - expiration, - checksum_crc32: checksums.crc32, - checksum_crc32c: checksums.crc32c, - checksum_sha1: checksums.sha1, - checksum_sha256: checksums.sha256, - checksum_crc64nvme: checksums.crc64nvme, - version_id: put_version, - ..Default::default() - }; - - // For browser-based POST uploads (multipart/form-data), response status/body handling - // is decided by s3s PostObject serializer (success_action_status / redirect semantics). - - let result = Ok(S3Response::new(output)); - let _ = helper.complete(&result); - - // Record write operation for capacity management (inline to avoid per-request tokio::spawn overhead) - let manager = get_capacity_manager(); - manager.record_write_operation().await; - - // Record PutObject metrics via zero-copy-metrics - { - let duration_ms = start_time.elapsed().as_millis() as f64; - rustfs_io_metrics::record_put_object( - duration_ms, - size, - enable_zero_copy, // Track if zero-copy was enabled - ); - } - - result + let input = req.input; + let flow_result = + DefaultObjectUsecase::run_put_object_flow(input, request_context, request_method_name, resolved_size).await?; + let helper = bind_helper_object(helper, flow_result.helper_object, flow_result.helper_version_id); + complete_put_response(helper, flow_result.output) } pub async fn execute_put_object_acl(&self, req: S3Request) -> S3Result> { @@ -2370,7 +977,7 @@ impl DefaultObjectUsecase { let cache_key = ConcurrencyManager::make_cache_key(&bucket, &object, version_id.clone().as_deref()); let cache_bucket = bucket.clone(); let cache_object = object.clone(); - spawn_traced(async move { + crate::storage::request_context::spawn_traced(async move { manager .invalidate_cache_versioned(&cache_bucket, &cache_object, version_id.as_deref()) .await; @@ -2405,191 +1012,6 @@ impl DefaultObjectUsecase { result } - async fn maybe_get_cached_get_object( - manager: &ConcurrencyManager, - bucket: &str, - key: &str, - cache_key: &str, - part_number: Option, - rs: Option<&HTTPRangeSpec>, - request_start: std::time::Instant, - ) -> Option { - if !manager.is_cache_enabled() || part_number.is_some() || rs.is_some() { - return None; - } - - let cached = manager.get_cached_object(cache_key).await?; - let cache_serve_duration = request_start.elapsed(); - - debug!("Serving object from response cache: {} (latency: {:?})", cache_key, cache_serve_duration); - - rustfs_io_metrics::record_get_object_cache_served(cache_serve_duration.as_secs_f64(), cached.body.len()); - - use rustfs_io_metrics::{record_memory_copy_saved, record_zero_copy_read}; - record_zero_copy_read(cached.body.len(), cache_serve_duration.as_secs_f64() * 1000.0); - record_memory_copy_saved(cached.body.len()); - - manager.record_transfer(cached.content_length as u64, Duration::from_micros(1)); - - let output = Self::build_cached_get_object_output(&cached); - let event_info = Self::build_cached_get_object_event_info(bucket, key, &cached); - - rustfs_io_metrics::record_get_object(request_start.elapsed().as_millis() as f64, cached.content_length, true); - - Some(GetObjectCachedHit { output, event_info }) - } - - fn finalize_get_object_completion( - cache_key: &str, - wrapper: &RequestTimeoutWrapper, - timeout_config: &TimeoutConfig, - total_duration: Duration, - response_content_length: i64, - optimal_buffer_size: usize, - ) { - rustfs_io_metrics::record_get_object_completion( - total_duration.as_secs_f64(), - response_content_length, - optimal_buffer_size, - ); - - rustfs_io_metrics::record_get_object(total_duration.as_millis() as f64, response_content_length, false); - - if wrapper.is_timeout() { - warn!( - "GetObject request exceeded timeout: key={} duration={:?} timeout={:?}", - cache_key, - wrapper.elapsed(), - timeout_config.get_object_timeout - ); - rustfs_io_metrics::record_get_object_timeout(None, Some(wrapper.elapsed().as_secs_f64())); - } - - debug!( - "GetObject completed: key={} size={} duration={:?} buffer={}", - cache_key, response_content_length, total_duration, optimal_buffer_size - ); - } - - async fn finalize_get_object_response( - helper: OperationHelper, - bucket: &str, - method: &hyper::Method, - headers: &HeaderMap, - event_info: ObjectInfo, - version_id_for_event: String, - output: GetObjectOutput, - ) -> S3Result> { - let helper = helper.object(event_info).version_id(version_id_for_event); - let response = wrap_response_with_cors(bucket, method, headers, output).await; - let result = Ok(response); - let _ = helper.complete(&result); - result - } - #[allow(clippy::too_many_arguments)] - async fn build_get_object_output_context( - &self, - req: &S3Request, - cache_key: &str, - manager: &ConcurrencyManager, - bucket: &str, - key: &str, - info: ObjectInfo, - event_info: ObjectInfo, - final_stream: DynReader, - rs: Option, - content_type: Option, - last_modified: Option, - response_content_length: i64, - content_range: Option, - server_side_encryption: Option, - sse_customer_algorithm: Option, - sse_customer_key_md5: Option, - ssekms_key_id: Option, - encryption_applied: bool, - permit_wait_duration: Duration, - queue_utilization: f64, - queue_status: &concurrency::IoQueueStatus, - concurrent_requests: usize, - part_number: Option, - versioned: bool, - ) -> S3Result { - let strategy = self.finalize_get_object_strategy( - manager, - bucket, - key, - &info, - rs.as_ref(), - response_content_length, - permit_wait_duration, - queue_utilization, - queue_status, - concurrent_requests, - ); - let GetObjectStrategyContext { - io_strategy, - optimal_buffer_size, - } = strategy; - - let body = Self::build_get_object_body( - final_stream, - &info, - cache_key, - response_content_length, - optimal_buffer_size, - part_number, - rs.is_some(), - encryption_applied, - io_strategy.cache_writeback_enabled, - ) - .await?; - - let checksums = Self::build_get_object_checksums(&info, &req.headers, part_number, rs.as_ref())?; - - let output_version_id = if versioned { - info.version_id.map(|vid| { - if vid == Uuid::nil() { - "null".to_string() - } else { - vid.to_string() - } - }) - } else { - None - }; - - let output = GetObjectOutput { - body, - content_length: Some(response_content_length), - last_modified, - content_type, - content_encoding: info.content_encoding.clone(), - accept_ranges: Some("bytes".to_string()), - content_range, - e_tag: info.etag.map(|etag| to_s3s_etag(&etag)), - metadata: filter_object_metadata(&info.user_defined), - server_side_encryption, - sse_customer_algorithm, - sse_customer_key_md5, - ssekms_key_id, - checksum_crc32: checksums.crc32, - checksum_crc32c: checksums.crc32c, - checksum_sha1: checksums.sha1, - checksum_sha256: checksums.sha256, - checksum_crc64nvme: checksums.crc64nvme, - checksum_type: checksums.checksum_type, - version_id: output_version_id, - ..Default::default() - }; - - Ok(GetObjectOutputContext { - output, - event_info, - response_content_length, - optimal_buffer_size, - }) - } - #[instrument( level = "debug", skip(self, req), @@ -2605,130 +1027,29 @@ impl DefaultObjectUsecase { .get::() .map(|ctx| ctx.request_id.clone()) .unwrap_or_else(|| crate::storage::request_context::RequestContext::fallback().request_id); - let bootstrap = Self::init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?; - let timeout_config = bootstrap.timeout_config; - let wrapper = bootstrap.wrapper; - let request_start = bootstrap.request_start; - let concurrent_requests = bootstrap.concurrent_requests; - let mut request_guard = bootstrap.request_guard; - - let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject).suppress_event(); - // mc get 3 - - let request_context = Self::prepare_get_object_request_context(&req).await?; - let GetObjectRequestContext { - bucket, - key, - cache_key, - version_id_for_event, - part_number, - rs, - opts, - } = request_context; - - // Try to get from cache for small, frequently accessed objects + let bootstrap = init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?; + let request_context = prepare_get_object_request_context(&req).await?; + let base_buffer_size = self.base_buffer_size(); let manager = get_concurrency_manager(); - - if let Some(cached_hit) = - Self::maybe_get_cached_get_object(manager, &bucket, &key, &cache_key, part_number, rs.as_ref(), request_start).await - { - let GetObjectCachedHit { output, event_info } = cached_hit; - helper = helper.object(event_info).version_id(version_id_for_event.clone()); - - let result = Ok(S3Response::new(output)); - let _ = helper.complete(&result); - return result; - } - - let prepared_read = Self::prepare_get_object_read_execution( - &req, + let flow_runtime = GetObjectFlowRuntime { manager, - &wrapper, - &timeout_config, - &bucket, - &key, - rs, - &opts, - part_number, - ) - .await?; - let GetObjectPreparedRead { io_planning, read_setup } = prepared_read; - let permit_wait_duration = io_planning.permit_wait_duration; - let queue_status = io_planning.queue_status; - let queue_utilization = io_planning.queue_utilization; + bootstrap: &bootstrap, + base_buffer_size, + }; + let helper = new_operation_helper(&req, EventName::ObjectAccessedGet, S3Operation::GetObject, true); + let flow_result = get_object_flow::run_get_object_flow(request_context.clone(), flow_runtime).await; - let GetObjectReadSetup { - info, - event_info, - final_stream, - rs, - content_type, - last_modified, - response_content_length, - content_range, - server_side_encryption, - sse_customer_algorithm, - sse_customer_key_md5, - ssekms_key_id, - encryption_applied, - } = read_setup; + let GetObjectBootstrap { + mut request_guard, + _deadlock_request_guard, + .. + } = bootstrap; - let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; - let output_context = self - .build_get_object_output_context( - &req, - &cache_key, - manager, - &bucket, - &key, - info, - event_info, - final_stream, - rs, - content_type, - last_modified, - response_content_length, - content_range, - server_side_encryption, - sse_customer_algorithm, - sse_customer_key_md5, - ssekms_key_id, - encryption_applied, - permit_wait_duration, - queue_utilization, - &queue_status, - concurrent_requests, - part_number, - versioned, - ) - .await?; - let GetObjectOutputContext { - output, - event_info, - response_content_length, - optimal_buffer_size, - } = output_context; + let result = match flow_result { + Ok(flow_result) => complete_get_flow_result(helper, &request_context, flow_result).await, + Err(err) => Err(err), + }; - let total_duration = request_start.elapsed(); - Self::finalize_get_object_completion( - &cache_key, - &wrapper, - &timeout_config, - total_duration, - response_content_length, - optimal_buffer_size, - ); - - let result = Self::finalize_get_object_response( - helper, - &bucket, - &req.method, - &req.headers, - event_info, - version_id_for_event, - output, - ) - .await; if result.is_ok() { request_guard.finish_ok(); } else { @@ -3336,6 +1657,8 @@ impl DefaultObjectUsecase { src_info.metadata_only = true; } + let mut reader: Box = Box::new(WarpReader::new(gr.stream)); + let decryption_request = DecryptionRequest { bucket: &src_bucket, key: &src_key, @@ -3347,12 +1670,11 @@ impl DefaultObjectUsecase { etag: src_info.etag.as_deref(), }; - let decryption_material = sse_decryption(decryption_request).await?; - - if let Some(material) = decryption_material.as_ref() - && let Some(original) = material.original_size - { - src_info.actual_size = original; + if let Some(material) = sse_decryption(decryption_request).await? { + reader = material.wrap_single_reader(reader); + if let Some(original) = material.original_size { + src_info.actual_size = original; + } } strip_managed_encryption_metadata(&mut src_info.user_defined); @@ -3363,11 +1685,16 @@ impl DefaultObjectUsecase { let mut compress_metadata = HashMap::new(); - let should_compress = is_compressible(&req.headers, &key) && actual_size > MIN_COMPRESSIBLE_SIZE as i64; - - if should_compress { + if is_compressible(&req.headers, &key) && actual_size > MIN_COMPRESSIBLE_SIZE as i64 { insert_str(&mut compress_metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string()); insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); + + let hrd = EtagReader::new(reader, None); + + // let hrd = HashReader::new(reader, length, actual_size, None, false).map_err(ApiError::from)?; + + reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default())); + length = HashReader::SIZE_PRESERVE_LAYER; } else { remove_str(&mut src_info.user_defined, SUFFIX_COMPRESSION); remove_str(&mut src_info.user_defined, SUFFIX_ACTUAL_SIZE); @@ -3384,7 +1711,7 @@ impl DefaultObjectUsecase { } if let Some(ct) = content_type { src_info.content_type = Some(ct.clone()); - src_info.user_defined.insert("content-type".to_string(), ct); + src_info.user_defined.insert(CONTENT_TYPE.to_string(), ct); } } @@ -3399,68 +1726,7 @@ impl DefaultObjectUsecase { src_info.user_defined.extend(object_lock_metadata); } - let mut reader = match decryption_material { - Some(material) => { - if material.is_multipart { - let (decrypted_stream, plaintext_size) = - material.wrap_reader(gr.stream, length).await.map_err(ApiError::from)?; - length = plaintext_size; - - if should_compress { - let hrd = HashReader::from_reader(decrypted_stream, length, actual_size, None, None, false) - .map_err(ApiError::from)?; - length = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - length, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? - } else { - HashReader::from_reader(decrypted_stream, length, actual_size, None, None, false) - .map_err(ApiError::from)? - } - } else if should_compress { - let hrd = - HashReader::from_stream(material.wrap_single_reader(gr.stream), length, actual_size, None, None, false) - .map_err(ApiError::from)?; - length = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - length, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? - } else { - HashReader::from_stream(material.wrap_single_reader(gr.stream), length, actual_size, None, None, false) - .map_err(ApiError::from)? - } - } - None => { - if should_compress { - let hrd = - HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)?; - length = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - length, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? - } else { - HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)? - } - } - }; + let mut reader = HashReader::new(reader, length, actual_size, None, None, false).map_err(ApiError::from)?; let encryption_request = EncryptionRequest { bucket: &bucket, @@ -3481,13 +1747,13 @@ impl DefaultObjectUsecase { effective_kms_key_id = material.kms_key_id.clone(); let encrypted_reader = material.wrap_reader(reader); - reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) + reader = HashReader::new(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) .map_err(ApiError::from)?; src_info.user_defined.extend(material.metadata); } - src_info.put_object_reader = Some(PutObjReader::new(reader)); + src_info.put_object_reader = Some(ChunkNativePutData::new(reader)); // check quota @@ -3717,7 +1983,7 @@ impl DefaultObjectUsecase { let manager = get_concurrency_manager(); let bucket_clone = bucket.clone(); let deleted_objects = dobjs.clone(); - spawn_traced(async move { + crate::storage::request_context::spawn_traced(async move { for dobj in deleted_objects { manager .invalidate_cache_versioned( @@ -3830,7 +2096,7 @@ impl DefaultObjectUsecase { .as_ref() .map(|context| context.notify()) .unwrap_or_else(default_notify_interface); - spawn_background(async move { + crate::storage::helper::spawn_background(async move { for res in delete_results { if let Some(dobj) = res.delete_object { let event_name = if dobj.delete_marker { @@ -4002,7 +2268,6 @@ impl DefaultObjectUsecase { }) .await; } - // Prefix/force-delete returns empty ObjectInfo; still emit bucket notification so webhooks match S3 DELETE. helper = helper .event_name(EventName::ObjectRemovedDelete) .object(ObjectInfo { @@ -4012,7 +2277,6 @@ impl DefaultObjectUsecase { }) .version_id(String::new()); let result = Ok(S3Response::with_status(DeleteObjectOutput::default(), StatusCode::NO_CONTENT)); - // Match non-empty delete path: capacity manager write-op telemetry. let manager = get_capacity_manager(); manager.record_write_operation().await; let _ = helper.complete(&result); @@ -4120,7 +2384,7 @@ impl DefaultObjectUsecase { let version_id_clone = version_id.clone(); let cache_bucket = bucket.clone(); let cache_object = object.clone(); - spawn_traced(async move { + crate::storage::request_context::spawn_traced(async move { manager .invalidate_cache_versioned(&cache_bucket, &cache_object, version_id_clone.as_deref()) .await; @@ -4632,7 +2896,7 @@ impl DefaultObjectUsecase { let rreq_clone = rreq.clone(); let version_id_clone = version_id.clone(); - spawn_traced(async move { + crate::storage::request_context::spawn_traced(async move { let opts = ObjectOptions { transition: TransitionOptions { restore_request: rreq_clone, @@ -4725,7 +2989,7 @@ impl DefaultObjectUsecase { let (tx, rx) = mpsc::channel::>(2); let stream = ReceiverStream::new(rx); - spawn_traced(async move { + crate::storage::request_context::spawn_traced(async move { let _ = tx .send(Ok(SelectObjectContentEvent::Cont(ContinuationEvent::default()))) .await; @@ -4746,419 +3010,22 @@ impl DefaultObjectUsecase { #[instrument(level = "debug", skip(self, req))] pub async fn execute_put_object_extract(&self, req: S3Request) -> S3Result> { - let helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, S3Operation::PutObject).suppress_event(); - let auth_method = req.method.clone(); - let auth_uri = req.uri.clone(); - let auth_headers = req.headers.clone(); - let auth_extensions = req.extensions.clone(); - let auth_credentials = req.credentials.clone(); - let auth_region = req.region.clone(); - let auth_service = req.service.clone(); - let auth_trailing_headers = req.trailing_headers.clone(); - if is_sse_kms_requested(&req.input, &req.headers) { + let request_context = prepare_put_object_request_context(&req); + let helper = new_operation_helper(&req, EventName::ObjectCreatedPut, S3Operation::PutObject, true); + if is_sse_kms_requested(&req.input, &request_context.headers) { return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for extract uploads")); } - let input = req.input; - - let PutObjectInput { - body, - bucket, - key, - version_id, - cache_control, - content_disposition, - content_encoding, - content_length, - content_language, - content_type, - content_md5, - expires, - object_lock_legal_hold_status, - object_lock_mode, - object_lock_retain_until_date, - server_side_encryption, - sse_customer_algorithm, - sse_customer_key, - sse_customer_key_md5, - ssekms_key_id, - storage_class, - tagging, - website_redirect_location, - .. - } = input; - - let event_version_id = version_id; - let (h_algo, h_key, h_md5) = extract_ssec_params_from_headers(&req.headers)?; - let sse_customer_algorithm = sse_customer_algorithm.or(h_algo); - let sse_customer_key = sse_customer_key.or(h_key); - let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5); - - let original_sse = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); - let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); - let mut effective_sse = original_sse.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .map(|sse| match sse.sse_algorithm.as_str() { - "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - }) - }) - }) - }); - let mut effective_kms_key_id = ssekms_key_id.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .and_then(|sse| sse.kms_master_key_id.clone()) - }) - }) - }); - if effective_sse - .as_ref() - .is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) - { - return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for extract uploads")); - } - validate_sse_headers_for_write( - effective_sse.as_ref(), - effective_kms_key_id.as_ref(), - sse_customer_algorithm.as_ref(), - sse_customer_key.as_ref(), - sse_customer_key_md5.as_ref(), - true, - )?; - let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; - - let size = match content_length { - Some(c) => c, - None => { - if let Some(val) = req.headers.get(AMZ_DECODED_CONTENT_LENGTH) { - match atoi::atoi::(val.as_bytes()) { - Some(x) => x, - None => return Err(s3_error!(UnexpectedContent)), - } - } else { - return Err(s3_error!(UnexpectedContent)); - } - } - }; - if size == -1 { - return Err(s3_error!(UnexpectedContent)); - } - validate_object_key(&key, "PUT")?; - self.check_bucket_quota(&bucket, QuotaOperation::PutObject, size as u64) + let resolved_size = resolve_put_body_size(req.input.content_length, &request_context.headers)?; + self.check_bucket_quota(&req.input.bucket, QuotaOperation::PutObject, resolved_size as u64) .await?; - - // Apply adaptive buffer sizing based on file size for optimal streaming performance. - // Uses workload profile configuration (enabled by default) to select appropriate buffer size. - // Buffer sizes range from 32KB to 4MB depending on file size and configured workload profile. - let buffer_size = get_buffer_size_opt_in(size); - let body = tokio::io::BufReader::with_capacity( - buffer_size, - StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), - ); - - let Some(ext) = Path::new(&key).extension().and_then(|s| s.to_str()) else { - return Err(s3_error!(InvalidArgument, "key extension not found")); - }; - - let ext = ext.to_owned(); - - let md5hex = if let Some(base64_md5) = content_md5 { - let md5 = base64_simd::STANDARD - .decode_to_vec(base64_md5.as_bytes()) - .map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?; - Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower)) - } else { - None - }; - - let sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query()); - let actual_size = size; - - let mut archive_reader = - HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?; - - if let Err(err) = archive_reader.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) { - return Err(ApiError::from(err).into()); - } - - let archive_etag = Arc::new(Mutex::new(None)); - let decoder = CompressionFormat::from_extension(&ext) - .get_decoder(ExtractArchiveEtagReader::new(archive_reader, archive_etag.clone())) - .map_err(|e| { - error!("get_decoder err {:?}", e); - s3_error!(InvalidArgument, "get_decoder err") - })?; - - let mut ar = Archive::new(decoder); - let mut entries = ar.entries().map_err(|e| { - error!("get entries err {:?}", e); - s3_error!(InvalidArgument, "get entries err") - })?; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - let extract_options = resolve_put_object_extract_options(&req.headers); - let version_id = match event_version_id { - Some(v) => v.to_string(), - None => String::new(), - }; - let notify = self .context .as_ref() .map(|context| context.notify()) .unwrap_or_else(default_notify_interface); - let req_params = extract_params_header(&req.headers); - let host = get_request_host(&req.headers); - let port = get_request_port(&req.headers); - let user_agent = get_request_user_agent(&req.headers); - - while let Some(entry) = entries.next().await { - let mut f = match entry { - Ok(f) => f, - Err(e) => { - if extract_options.ignore_errors { - warn!("Skipping archive entry because read failed and ignore-errors is enabled: {e}"); - continue; - } - error!("Failed to read archive entry: {}", e); - return Err(s3_error!(InvalidArgument, "Failed to read archive entry: {:?}", e)); - } - }; - - let fpath = match f.path() { - Ok(path) => path, - Err(e) => { - if extract_options.ignore_errors { - warn!("Skipping archive entry because path decode failed and ignore-errors is enabled: {e}"); - continue; - } - return Err(s3_error!(InvalidArgument, "Failed to decode archive entry path")); - } - }; - - let is_dir = f.header().entry_type().is_dir(); - let fpath = normalize_extract_entry_key(&fpath.to_string_lossy(), extract_options.prefix.as_deref(), is_dir); - - let mut auth_req = S3Request { - input: PutObjectInput::default(), - method: auth_method.clone(), - uri: auth_uri.clone(), - headers: auth_headers.clone(), - extensions: auth_extensions.clone(), - credentials: auth_credentials.clone(), - region: auth_region.clone(), - service: auth_service.clone(), - trailing_headers: auth_trailing_headers.clone(), - }; - { - let req_info = req_info_mut(&mut auth_req)?; - req_info.bucket = Some(bucket.clone()); - req_info.object = Some(fpath.clone()); - req_info.version_id = None; - } - authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectAction)).await?; - - let mut size = f.header().size().unwrap_or_default() as i64; - let archive_entry_mod_time = f - .header() - .mtime() - .ok() - .and_then(|modified_at_secs| OffsetDateTime::from_unix_timestamp(modified_at_secs as i64).ok()); - let mut metadata = HashMap::new(); - apply_put_request_metadata( - &mut metadata, - &req.headers, - &fpath, - cache_control.clone(), - content_disposition.clone(), - content_encoding.clone(), - content_language.clone(), - content_type.clone(), - expires.clone(), - website_redirect_location.clone(), - tagging.clone(), - storage_class.clone(), - )?; - let mut opts = put_opts(&bucket, &fpath, None, &req.headers, metadata.clone()) - .await - .map_err(ApiError::from)?; - apply_extract_entry_pax_extensions(&mut f, &mut metadata, &mut opts).await?; - if archive_entry_mod_time.is_some() { - opts.mod_time = archive_entry_mod_time; - } - - debug!("Extracting file: {}, size: {} bytes", fpath, size); - - if is_dir { - if extract_options.ignore_dirs { - debug!("Skipping directory entry during archive extract: {}", fpath); - continue; - } - size = 0; - } - - let actual_size = size; - - let should_compress = !is_dir && is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64; - - let mut hrd = if is_dir { - HashReader::from_stream(std::io::Cursor::new(Vec::new()), size, actual_size, None, None, false) - .map_err(ApiError::from)? - } else if should_compress { - insert_str(&mut metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string()); - insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); - - let hrd = HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)?; - size = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - size, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? - } else { - HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)? - }; - apply_put_request_object_lock_opts( - &bucket, - object_lock_legal_hold_status.clone(), - object_lock_mode.clone(), - object_lock_retain_until_date.clone(), - &mut opts, - ) - .await?; - if let Some(material) = sse_encryption(EncryptionRequest { - bucket: &bucket, - key: &fpath, - server_side_encryption: effective_sse.clone(), - ssekms_key_id: effective_kms_key_id.clone(), - sse_customer_algorithm: sse_customer_algorithm.clone(), - sse_customer_key: sse_customer_key.clone(), - sse_customer_key_md5: sse_customer_key_md5.clone(), - content_size: actual_size, - part_number: None, - part_key: None, - part_nonce: None, - }) - .await? - { - effective_sse = Some(material.server_side_encryption.clone()); - effective_kms_key_id = material.kms_key_id.clone(); - - let encrypted_reader = material.wrap_reader(hrd); - hrd = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) - .map_err(ApiError::from)?; - - let encryption_metadata = material.metadata; - metadata.extend(encryption_metadata.clone()); - opts.user_defined.extend(encryption_metadata); - } - opts.user_defined.extend(metadata); - let mut reader = PutObjReader::new(hrd); - - let obj_info = match store.put_object(&bucket, &fpath, &mut reader, &opts).await { - Ok(info) => info, - Err(e) => { - if extract_options.ignore_errors { - warn!("Skipping archive entry because object write failed and ignore-errors is enabled: {e}"); - continue; - } - return Err(ApiError::from(e).into()); - } - }; - - let manager = get_concurrency_manager(); - let fpath_clone = fpath.clone(); - let bucket_clone = bucket.clone(); - spawn_traced(async move { - manager.invalidate_cache_versioned(&bucket_clone, &fpath_clone, None).await; - }); - - let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)); - - let output = PutObjectOutput { - e_tag, - ..Default::default() - }; - - let event_args = rustfs_notify::EventArgs { - event_name: EventName::ObjectCreatedPut, - bucket_name: bucket.clone(), - object: obj_info.clone(), - req_params: req_params.clone(), - resp_elements: extract_resp_elements(&S3Response::new(output.clone())), - version_id: version_id.clone(), - host: host.clone(), - port, - user_agent: user_agent.clone(), - }; - - let notify = notify.clone(); - let request_context = req - .extensions - .get::() - .cloned(); - spawn_background_with_context(request_context, async move { - notify.notify(event_args).await; - }); - } - - let mut checksums = PutObjectChecksums { - crc32: input.checksum_crc32, - crc32c: input.checksum_crc32c, - sha1: input.checksum_sha1, - sha256: input.checksum_sha256, - crc64nvme: input.checksum_crc64nvme, - }; - apply_trailing_checksums( - input.checksum_algorithm.as_ref().map(|a| a.as_str()), - &req.trailing_headers, - &mut checksums, - ); - - warn!( - "put object extract checksum_crc32={:?}, checksum_crc32c={:?}, checksum_sha1={:?}, checksum_sha256={:?}, checksum_crc64nvme={:?}", - checksums.crc32, checksums.crc32c, checksums.sha1, checksums.sha256, checksums.crc64nvme, - ); - - drop(entries); - let mut decoder = match ar.into_inner() { - Ok(decoder) => decoder, - Err(_) => return Err(s3_error!(InvalidArgument, "Failed to finalize archive reader")), - }; - tokio::io::copy(&mut decoder, &mut tokio::io::sink()) - .await - .map_err(map_extract_archive_error)?; - let archive_etag = archive_etag - .lock() - .ok() - .and_then(|etag| etag.clone()) - .map(|etag| to_s3s_etag(&etag)); - - let output = PutObjectOutput { - e_tag: archive_etag, - checksum_crc32: checksums.crc32, - checksum_crc32c: checksums.crc32c, - checksum_sha1: checksums.sha1, - checksum_sha256: checksums.sha256, - checksum_crc64nvme: checksums.crc64nvme, - ..Default::default() - }; - let result = Ok(S3Response::new(output)); - let _ = helper.complete(&result); - result + let input = req.input; + let output = DefaultObjectUsecase::run_put_object_extract_flow(input, request_context, notify, resolved_size).await?; + complete_put_response(helper, output) } } @@ -5174,7 +3041,7 @@ fn object_attributes_requested(object_attributes: &[ObjectAttributes], name: &'s #[cfg(test)] mod tests { use super::*; - use http::{Extensions, HeaderMap, HeaderName, HeaderValue, Method, Uri}; + use http::{Extensions, HeaderMap, Method, Uri}; fn build_request(input: T, method: Method) -> S3Request { S3Request { @@ -5199,7 +3066,7 @@ mod tests { .unwrap(); let req = build_request(input, Method::PUT); - let (event_name, quota_operation, method_name) = DefaultObjectUsecase::put_object_execution_context(&req); + let (event_name, quota_operation, method_name) = put_object_execution_context(&req); assert_eq!(event_name, EventName::ObjectCreatedPut); assert!(matches!(quota_operation, QuotaOperation::PutObject)); assert_eq!(method_name, "PUT"); @@ -5215,298 +3082,12 @@ mod tests { let mut req = build_request(input, Method::POST); req.extensions.insert(PostObjectRequestMarker); - let (event_name, quota_operation, method_name) = DefaultObjectUsecase::put_object_execution_context(&req); + let (event_name, quota_operation, method_name) = put_object_execution_context(&req); assert_eq!(event_name, EventName::ObjectCreatedPost); assert!(matches!(quota_operation, QuotaOperation::PostObject)); assert_eq!(method_name, "POST"); } - #[test] - fn is_put_object_extract_requested_accepts_meta_header() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); - - assert!(is_put_object_extract_requested(&headers)); - } - - #[test] - fn is_put_object_extract_requested_accepts_compat_header_case_insensitive() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_EXTRACT_COMPAT, HeaderValue::from_static(" TRUE ")); - - assert!(is_put_object_extract_requested(&headers)); - } - - #[test] - fn is_put_object_extract_requested_rejects_missing_or_false_value() { - let mut headers = HeaderMap::new(); - assert!(!is_put_object_extract_requested(&headers)); - - headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("false")); - assert!(!is_put_object_extract_requested(&headers)); - } - - #[test] - fn normalize_snowball_prefix_trims_slashes_and_whitespace() { - assert_eq!(normalize_snowball_prefix(" /batch/incoming/ "), Some("batch/incoming".to_string())); - assert_eq!(normalize_snowball_prefix("///"), None); - } - - #[test] - fn normalize_extract_entry_key_applies_prefix_and_directory_suffix() { - assert_eq!( - normalize_extract_entry_key("nested/path.txt", Some("imports"), false), - "imports/nested/path.txt" - ); - assert_eq!(normalize_extract_entry_key("nested/dir/", Some("imports"), true), "imports/nested/dir/"); - assert_eq!(normalize_extract_entry_key("top-level", None, false), "top-level"); - } - - #[test] - fn should_use_zero_copy_rejects_boundary_at_1mb() { - let headers = HeaderMap::new(); - - assert!(!should_use_zero_copy(1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_small_objects() { - let headers = HeaderMap::new(); - - assert!(!should_use_zero_copy(1024 * 1024 - 1, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_one_megabyte() { - let headers = HeaderMap::new(); - - assert!(!should_use_zero_copy(1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_encrypted_requests() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("AES256")); - - assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_encrypted_requests_with_sse_customer_algorithm() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, HeaderValue::from_static("AES256")); - - assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_encrypted_requests_with_kms_key_id() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, HeaderValue::from_static("test-kms-key-id")); - - assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_rejects_compressible_content_types() { - let mut headers = HeaderMap::new(); - headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json; charset=utf-8")); - - assert!(!should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn should_use_zero_copy_allows_large_unencrypted_binary_objects() { - let mut headers = HeaderMap::new(); - headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/octet-stream")); - - assert!(should_use_zero_copy(2 * 1024 * 1024, &headers)); - } - - #[test] - fn resolve_put_object_extract_options_defaults_when_headers_missing() { - let headers = HeaderMap::new(); - let options = resolve_put_object_extract_options(&headers); - assert_eq!( - options, - PutObjectExtractOptions { - prefix: None, - ignore_dirs: false, - ignore_errors: false - } - ); - } - - #[test] - fn resolve_put_object_extract_options_accepts_internal_headers() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_PREFIX_INTERNAL, HeaderValue::from_static("/internal/prefix/")); - headers.insert(AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL, HeaderValue::from_static("true")); - headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL, HeaderValue::from_static("TRUE")); - - let options = resolve_put_object_extract_options(&headers); - assert_eq!(options.prefix.as_deref(), Some("internal/prefix")); - assert!(options.ignore_dirs); - assert!(options.ignore_errors); - } - - #[test] - fn resolve_put_object_extract_options_accepts_standard_headers() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static(" /standard/prefix/ ")); - headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static(" true ")); - headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("TRUE")); - - let options = resolve_put_object_extract_options(&headers); - assert_eq!(options.prefix.as_deref(), Some("standard/prefix")); - assert!(options.ignore_dirs); - assert!(options.ignore_errors); - } - - #[test] - fn resolve_put_object_extract_options_accepts_suffix_compatible_headers() { - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("x-amz-meta-acme-snowball-prefix"), - HeaderValue::from_static(" /partner/import "), - ); - headers.insert( - HeaderName::from_static("x-amz-meta-acme-snowball-ignore-dirs"), - HeaderValue::from_static(" true "), - ); - headers.insert( - HeaderName::from_static("x-amz-meta-acme-snowball-ignore-errors"), - HeaderValue::from_static("TRUE"), - ); - - let options = resolve_put_object_extract_options(&headers); - assert_eq!(options.prefix.as_deref(), Some("partner/import")); - assert!(options.ignore_dirs); - assert!(options.ignore_errors); - } - - #[test] - fn resolve_put_object_extract_options_prefers_exact_headers_over_suffix_fallback() { - let mut headers = HeaderMap::new(); - headers.insert("x-amz-meta-acme-snowball-prefix", HeaderValue::from_static("/fallback/prefix/")); - headers.insert(AMZ_RUSTFS_SNOWBALL_PREFIX, HeaderValue::from_static("/internal/prefix/")); - headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static("/standard/prefix/")); - headers.insert(AMZ_MINIO_SNOWBALL_PREFIX, HeaderValue::from_static("/minio/prefix/")); - - let options = resolve_put_object_extract_options(&headers); - assert_eq!(options.prefix.as_deref(), Some("minio/prefix")); - } - - #[test] - fn resolve_put_object_extract_options_exact_flags_override_suffix_fallback() { - let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static("false")); - headers.insert("x-amz-meta-acme-snowball-ignore-dirs", HeaderValue::from_static("true")); - headers.insert(AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("false")); - headers.insert("x-amz-meta-acme-snowball-ignore-errors", HeaderValue::from_static("true")); - - let options = resolve_put_object_extract_options(&headers); - assert!(!options.ignore_dirs); - assert!(!options.ignore_errors); - } - - #[tokio::test] - async fn execute_put_object_rejects_post_object_sse_kms_from_input() { - let input = PutObjectInput::builder() - .bucket("test-bucket".to_string()) - .key("test-key".to_string()) - .server_side_encryption(Some(ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS))) - .build() - .unwrap(); - - let mut req = build_request(input, Method::POST); - req.extensions.insert(PostObjectRequestMarker); - - let usecase = DefaultObjectUsecase::without_context(); - let fs = FS::new(); - - let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); - assert_eq!(err.code(), &S3ErrorCode::NotImplemented); - } - - #[tokio::test] - async fn execute_put_object_rejects_extract_sse_kms() { - let input = PutObjectInput::builder() - .bucket("test-bucket".to_string()) - .key("archive.tar".to_string()) - .server_side_encryption(Some(ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS))) - .build() - .unwrap(); - - let mut req = build_request(input, Method::PUT); - req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); - - let usecase = DefaultObjectUsecase::without_context(); - let fs = FS::new(); - - let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); - assert_eq!(err.code(), &S3ErrorCode::NotImplemented); - } - - #[tokio::test] - async fn execute_put_object_extract_rejects_invalid_storage_class() { - let input = PutObjectInput::builder() - .bucket("test-bucket".to_string()) - .key("archive.tar".to_string()) - .storage_class(Some(StorageClass::from_static("INVALID"))) - .build() - .unwrap(); - - let mut req = build_request(input, Method::PUT); - req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); - - let usecase = DefaultObjectUsecase::without_context(); - let fs = FS::new(); - - let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); - assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass); - } - - #[tokio::test] - async fn execute_put_object_rejects_post_object_sse_kms_from_headers() { - let input = PutObjectInput::builder() - .bucket("test-bucket".to_string()) - .key("test-key".to_string()) - .build() - .unwrap(); - - let mut req = build_request(input, Method::POST); - req.extensions.insert(PostObjectRequestMarker); - req.headers - .insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("aws:kms")); - - let usecase = DefaultObjectUsecase::without_context(); - let fs = FS::new(); - - let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); - assert_eq!(err.code(), &S3ErrorCode::NotImplemented); - } - - #[tokio::test] - async fn execute_put_object_rejects_post_object_sse_kms_key_id_header() { - let input = PutObjectInput::builder() - .bucket("test-bucket".to_string()) - .key("test-key".to_string()) - .build() - .unwrap(); - - let mut req = build_request(input, Method::POST); - req.extensions.insert(PostObjectRequestMarker); - req.headers - .insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, HeaderValue::from_static("test-kms-key-id")); - - let usecase = DefaultObjectUsecase::without_context(); - let fs = FS::new(); - - let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); - assert_eq!(err.code(), &S3ErrorCode::NotImplemented); - } - #[tokio::test] async fn execute_put_object_rejects_invalid_storage_class() { let input = PutObjectInput::builder() @@ -5524,22 +3105,6 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass); } - #[tokio::test] - async fn execute_get_object_rejects_zero_part_number() { - let input = GetObjectInput::builder() - .bucket("test-bucket".to_string()) - .key("test-key".to_string()) - .part_number(Some(0)) - .build() - .unwrap(); - - let req = build_request(input, Method::GET); - let usecase = DefaultObjectUsecase::without_context(); - - let err = usecase.execute_get_object(req).await.unwrap_err(); - assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); - } - #[tokio::test] async fn execute_copy_object_rejects_self_copy_without_replace_directive() { let input = CopyObjectInput::builder() diff --git a/rustfs/src/app/object_usecase/app_adapters.rs b/rustfs/src/app/object_usecase/app_adapters.rs new file mode 100644 index 000000000..d424764fb --- /dev/null +++ b/rustfs/src/app/object_usecase/app_adapters.rs @@ -0,0 +1,616 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::get_object_flow::GetObjectBootstrap; +use super::*; +use crate::app::context::NotifyInterface; +use crate::storage::concurrency::{self, get_buffer_size_opt_in}; +use hashbrown::HashMap; +use rustfs_object_io::get::{ + CachedGetObjectSource as ObjectIoCachedGetObjectSource, GetObjectBodyPlan as ObjectIoGetObjectBodyPlan, + GetObjectCacheWriteback, GetObjectDataPlaneMetricContract as ObjectIoGetObjectDataPlaneMetricContract, GetObjectFlowResult, + GetObjectResponseMode, MaterializeGetObjectBodyError as ObjectIoMaterializeGetObjectBodyError, + build_cached_get_object_flow_result_from_source as object_io_build_cached_get_object_flow_result_from_source, + finalize_get_object_cache_writeback as object_io_finalize_get_object_cache_writeback, + materialize_get_object_body as object_io_materialize_get_object_body, plan_get_object_body as object_io_plan_get_object_body, + plan_get_object_strategy_layout as object_io_plan_get_object_strategy_layout, +}; + +pub(super) async fn prepare_get_object_request_context(req: &S3Request) -> S3Result { + let GetObjectInput { + bucket, + key, + version_id, + part_number, + range, + .. + } = req.input.clone(); + + validate_object_key(&key, "GET")?; + + let part_number = part_number.map(|v| v as usize); + + if let Some(part_num) = part_number + && part_num == 0 + { + return Err(s3_error!(InvalidArgument, "Invalid part number: part number must be greater than 0")); + } + + let rs = range.map(|v| match v { + Range::Int { first, last } => HTTPRangeSpec { + is_suffix_length: false, + start: first as i64, + end: if let Some(last) = last { last as i64 } else { -1 }, + }, + Range::Suffix { length } => HTTPRangeSpec { + is_suffix_length: true, + start: length as i64, + end: -1, + }, + }); + + if rs.is_some() && part_number.is_some() { + return Err(s3_error!(InvalidArgument, "range and part_number invalid")); + } + + let opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), part_number, &req.headers) + .await + .map_err(ApiError::from)?; + + Ok(GetObjectRequestContext { + cache_key: ConcurrencyManager::make_cache_key(&bucket, &key, version_id.as_deref()), + version_id_for_event: version_id.unwrap_or_default(), + bucket, + key, + part_number, + rs, + opts, + headers: req.headers.clone(), + method: req.method.clone(), + sse_customer_key: req.input.sse_customer_key.clone(), + sse_customer_key_md5: req.input.sse_customer_key_md5.clone(), + }) +} + +impl ObjectIoCachedGetObjectSource for CachedGetObject { + fn body(&self) -> &std::sync::Arc { + &self.body + } + + fn content_length(&self) -> i64 { + self.content_length + } + + fn content_type(&self) -> Option<&str> { + self.content_type.as_deref() + } + + fn e_tag(&self) -> Option<&str> { + self.e_tag.as_deref() + } + + fn last_modified(&self) -> Option<&str> { + self.last_modified.as_deref() + } + + fn cache_control(&self) -> Option<&str> { + self.cache_control.as_deref() + } + + fn content_disposition(&self) -> Option<&str> { + self.content_disposition.as_deref() + } + + fn content_encoding(&self) -> Option<&str> { + self.content_encoding.as_deref() + } + + fn content_language(&self) -> Option<&str> { + self.content_language.as_deref() + } + + fn storage_class(&self) -> Option<&str> { + self.storage_class.as_deref() + } + + fn version_id(&self) -> Option<&str> { + self.version_id.as_deref() + } + + fn delete_marker(&self) -> bool { + self.delete_marker + } + + fn tag_count(&self) -> Option { + self.tag_count + } + + fn user_metadata(&self) -> &std::collections::HashMap { + &self.user_metadata + } + + fn checksum_crc32(&self) -> Option<&str> { + self.checksum_crc32.as_deref() + } + + fn checksum_crc32c(&self) -> Option<&str> { + self.checksum_crc32c.as_deref() + } + + fn checksum_sha1(&self) -> Option<&str> { + self.checksum_sha1.as_deref() + } + + fn checksum_sha256(&self) -> Option<&str> { + self.checksum_sha256.as_deref() + } + + fn checksum_crc64nvme(&self) -> Option<&str> { + self.checksum_crc64nvme.as_deref() + } + + fn checksum_type(&self) -> Option<&ChecksumType> { + self.checksum_type.as_ref() + } +} + +pub(super) fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result { + let timeout_config = TimeoutConfig::from_env(); + let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.to_string()); + let request_start = std::time::Instant::now(); + let request_guard = ConcurrencyManager::track_request(); + let concurrent_requests = GetObjectGuard::concurrent_requests(); + + let deadlock_detector = deadlock_detector::get_deadlock_detector(); + deadlock_detector.register_request(request_id, format!("GetObject {bucket}/{key}")); + let deadlock_request_guard = DeadlockRequestGuard::new(deadlock_detector, request_id.to_string()); + + if wrapper.is_timeout() { + warn!( + bucket = %bucket, + key = %key, + timeout_secs = timeout_config.get_object_timeout.as_secs(), + elapsed_ms = wrapper.elapsed().as_millis(), + "GetObject request timed out before processing" + ); + return Err(s3_error!(InternalError, "Request timeout before processing")); + } + + rustfs_io_metrics::record_get_object_request_start(concurrent_requests); + + debug!( + "GetObject request started with {} concurrent requests, timeout={:?}", + concurrent_requests, timeout_config.get_object_timeout + ); + + Ok(GetObjectBootstrap { + timeout_config, + wrapper, + request_start, + request_guard, + _deadlock_request_guard: deadlock_request_guard, + concurrent_requests, + }) +} + +#[allow(clippy::too_many_arguments)] +pub(super) async fn maybe_get_cached_get_object_flow_result( + manager: &ConcurrencyManager, + bucket: &str, + key: &str, + cache_key: &str, + version_id_for_event: String, + part_number: Option, + rs: Option<&HTTPRangeSpec>, + request_start: std::time::Instant, +) -> Option { + if !manager.is_cache_enabled() || part_number.is_some() || rs.is_some() { + return None; + } + + let cached = manager.get_cached_object(cache_key).await?; + let cache_serve_duration = request_start.elapsed(); + let metric_contract = ObjectIoGetObjectDataPlaneMetricContract::cache_served(); + + debug!("Serving object from response cache: {} (latency: {:?})", cache_key, cache_serve_duration); + + if metric_contract.record_cache_served_metric { + rustfs_io_metrics::record_get_object_cache_served(cache_serve_duration.as_secs_f64(), cached.body.len()); + } + rustfs_io_metrics::record_io_path_selected("get", metric_contract.io_path); + rustfs_io_metrics::record_io_copy_mode("get", metric_contract.copy_mode, cached.body.len()); + + manager.record_transfer(cached.content_length as u64, Duration::from_micros(1)); + + rustfs_io_metrics::record_get_object(request_start.elapsed().as_millis() as f64, cached.content_length, true); + + Some(object_io_build_cached_get_object_flow_result_from_source( + bucket, + key, + cached.as_ref(), + version_id_for_event, + )) +} + +pub(super) struct GetObjectBodyAdapterOutput { + pub(super) body: Option, + pub(super) body_plan: ObjectIoGetObjectBodyPlan, + pub(super) cache_writeback: Option, +} + +pub(super) fn spawn_get_object_cache_writeback( + cache_key: &str, + writeback: GetObjectCacheWriteback, + metric_contract: ObjectIoGetObjectDataPlaneMetricContract, +) { + debug_assert_eq!( + metric_contract.request_source, + rustfs_object_io::get::GetObjectDataPlaneRequestSource::Disk + ); + debug_assert!(!metric_contract.record_cache_served_metric); + debug_assert!(metric_contract.record_cache_writeback_metric); + + let cached_response = CachedGetObject::from_get_object_cache_writeback(writeback); + + let cache_key_clone = cache_key.to_string(); + crate::storage::request_context::spawn_traced(async move { + let manager = get_concurrency_manager(); + manager.put_cached_object(cache_key_clone.clone(), cached_response).await; + debug!("Object cached successfully with metadata: {}", cache_key_clone); + }); + + if metric_contract.record_cache_writeback_metric { + rustfs_io_metrics::record_object_cache_writeback(); + } +} + +pub(super) async fn build_get_object_body_adapter( + final_stream: R, + info: &ObjectInfo, + cache_key: &str, + response_content_length: i64, + optimal_buffer_size: usize, + cache_eligibility: rustfs_concurrency::GetObjectCacheEligibility, +) -> S3Result +where + R: AsyncRead + Send + Sync + Unpin + 'static, +{ + let body_plan = object_io_plan_get_object_body(cache_eligibility, rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD); + + match body_plan { + ObjectIoGetObjectBodyPlan::CacheWriteback => { + debug!( + "Reading object into memory for caching: key={} size={}", + cache_key, response_content_length + ); + } + ObjectIoGetObjectBodyPlan::BufferSeekable => { + debug!( + "Reading small object into memory for seek support: key={} size={}", + cache_key, response_content_length + ); + } + ObjectIoGetObjectBodyPlan::Stream if cache_eligibility.encryption_applied => { + info!( + "Encrypted object: Using unlimited stream for decryption with buffer size {}", + optimal_buffer_size + ); + } + _ => {} + } + + let materialized = + object_io_materialize_get_object_body(final_stream, info, body_plan, response_content_length, optimal_buffer_size) + .await + .map_err(|err| match err { + ObjectIoMaterializeGetObjectBodyError::CacheRead(err) => { + error!("Failed to read object into memory for caching: {}", err); + ApiError::from(StorageError::other(format!("Failed to read object for caching: {err}"))) + } + ObjectIoMaterializeGetObjectBodyError::EncryptedRead(err) => { + error!("Failed to read decrypted object into memory: {}", err); + ApiError::from(StorageError::other(format!("Failed to read decrypted object: {err}"))) + } + })?; + + Ok(GetObjectBodyAdapterOutput { + body: materialized.body, + body_plan: materialized.plan, + cache_writeback: materialized.cache_writeback.map(|writeback| { + object_io_finalize_get_object_cache_writeback( + info, + writeback, + filter_object_metadata(&info.user_defined).unwrap_or_default(), + ) + }), + }) +} + +pub(super) fn finalize_get_object_completion( + cache_key: &str, + wrapper: &RequestTimeoutWrapper, + timeout_config: &TimeoutConfig, + total_duration: Duration, + response_content_length: i64, + optimal_buffer_size: usize, + metric_contract: ObjectIoGetObjectDataPlaneMetricContract, +) { + rustfs_io_metrics::record_get_object_completion(total_duration.as_secs_f64(), response_content_length, optimal_buffer_size); + + rustfs_io_metrics::record_get_object(total_duration.as_millis() as f64, response_content_length, false); + rustfs_io_metrics::record_io_copy_mode("get", metric_contract.copy_mode, response_content_length.max(0) as usize); + + if wrapper.is_timeout() { + warn!( + "GetObject request exceeded timeout: key={} duration={:?} timeout={:?}", + cache_key, + wrapper.elapsed(), + timeout_config.get_object_timeout + ); + rustfs_io_metrics::record_get_object_timeout(None, Some(wrapper.elapsed().as_secs_f64())); + } + + debug!( + "GetObject completed: key={} size={} duration={:?} buffer={}", + cache_key, response_content_length, total_duration, optimal_buffer_size + ); +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn finalize_get_object_strategy_runtime( + base_buffer_size: usize, + manager: &ConcurrencyManager, + bucket: &str, + key: &str, + info: &ObjectInfo, + rs: Option<&HTTPRangeSpec>, + response_content_length: i64, + permit_wait_duration: Duration, + queue_utilization: f64, + queue_status: &concurrency::IoQueueStatus, + concurrent_requests: usize, +) -> (concurrency::IoStrategy, usize) { + let strategy_layout = object_io_plan_get_object_strategy_layout( + rs, + response_content_length, + 0, + get_buffer_size_opt_in(response_content_length), + ); + + if let Some(range_spec) = rs + && range_spec.start >= 0 + { + manager.record_access(range_spec.start as u64, response_content_length as u64); + } + + if response_content_length > 0 { + manager.record_transfer(response_content_length as u64, permit_wait_duration); + } + + let io_strategy = manager.calculate_io_strategy_with_context( + info.size, + base_buffer_size, + permit_wait_duration, + strategy_layout.is_sequential_hint, + ); + + debug!( + wait_ms = permit_wait_duration.as_millis() as u64, + load_level = ?io_strategy.load_level, + buffer_size = io_strategy.buffer_size, + buffer_multiplier = io_strategy.buffer_multiplier, + readahead = io_strategy.enable_readahead, + cache_wb = io_strategy.cache_writeback_enabled, + storage_media = ?io_strategy.storage_media, + access_pattern = ?io_strategy.access_pattern, + bandwidth_tier = ?io_strategy.bandwidth_tier, + concurrent_requests = io_strategy.concurrent_requests, + file_size = info.size, + is_sequential = strategy_layout.is_sequential_hint, + "Enhanced multi-factor I/O strategy calculated" + ); + + let io_priority = manager.get_io_priority(response_content_length); + + if manager.is_priority_scheduling_enabled() { + debug!( + bucket = %bucket, + key = %key, + priority = %io_priority, + request_size = response_content_length, + "I/O priority assigned (based on actual request size)" + ); + + rustfs_io_metrics::record_io_priority_assignment(io_priority.as_str()); + } + + rustfs_io_metrics::record_get_object_io_state( + permit_wait_duration.as_secs_f64(), + queue_utilization, + queue_status.permits_in_use, + queue_status.total_permits.saturating_sub(queue_status.permits_in_use), + io_strategy.load_level.as_str(), + io_strategy.buffer_multiplier, + ); + rustfs_io_metrics::record_io_priority_assignment(io_priority.as_str()); + + let strategy_layout = object_io_plan_get_object_strategy_layout( + rs, + response_content_length, + io_strategy.buffer_size, + get_buffer_size_opt_in(response_content_length), + ); + + debug!( + actual_request_size = response_content_length, + priority = %io_priority.as_str(), + "I/O priority finalized with actual request size" + ); + + debug!( + "GetObject buffer sizing: file_size={}, base={}, optimal={}, concurrent_requests={}, io_strategy={:?}", + response_content_length, + get_buffer_size_opt_in(response_content_length), + strategy_layout.optimal_buffer_size, + concurrent_requests, + io_strategy.load_level + ); + + (io_strategy, strategy_layout.optimal_buffer_size) +} + +pub(super) fn prepare_put_object_request_context(req: &S3Request) -> PutObjectRequestContext { + PutObjectRequestContext { + headers: req.headers.clone(), + trailing_headers: req.trailing_headers.clone(), + uri_query: req.uri.query().map(str::to_string), + is_post_object: req.extensions.get::().is_some(), + method: req.method.clone(), + uri: req.uri.clone(), + extensions: req.extensions.clone(), + credentials: req.credentials.clone(), + region: req.region.clone(), + service: req.service.clone(), + } +} + +pub(super) fn put_object_execution_context(req: &S3Request) -> (EventName, QuotaOperation, &'static str) { + if req.extensions.get::().is_some() { + (EventName::ObjectCreatedPost, QuotaOperation::PostObject, "POST") + } else { + (EventName::ObjectCreatedPut, QuotaOperation::PutObject, "PUT") + } +} + +pub(super) fn new_operation_helper( + req: &S3Request, + event_name: EventName, + operation: S3Operation, + suppress_event: bool, +) -> OperationHelper { + let helper = OperationHelper::new(req, event_name, operation); + if suppress_event { helper.suppress_event() } else { helper } +} + +pub(super) fn bind_helper_object( + helper: OperationHelper, + object_info: ObjectInfo, + version_id: Option, +) -> OperationHelper { + let helper = helper.object(object_info); + if let Some(version_id) = version_id { + helper.version_id(version_id) + } else { + helper + } +} + +pub(super) async fn complete_get_flow_result( + helper: OperationHelper, + request_context: &GetObjectRequestContext, + flow_result: GetObjectFlowResult, +) -> S3Result> { + match flow_result.response_mode { + GetObjectResponseMode::Plain => { + let helper = bind_helper_object(helper, flow_result.event_info, Some(flow_result.version_id_for_event)); + let result = Ok(S3Response::new(flow_result.output)); + let _ = helper.complete(&result); + result + } + GetObjectResponseMode::CorsWrapped => { + let helper = helper + .object(flow_result.event_info) + .version_id(flow_result.version_id_for_event); + let response = wrap_response_with_cors( + &request_context.bucket, + &request_context.method, + &request_context.headers, + flow_result.output, + ) + .await; + let result = Ok(response); + let _ = helper.complete(&result); + result + } + } +} + +pub(super) fn complete_put_response(helper: OperationHelper, output: PutObjectOutput) -> S3Result> { + let result = Ok(S3Response::new(output)); + let _ = helper.complete(&result); + result +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn spawn_put_extract_notification( + notify: Arc, + request_context: Option, + bucket: String, + req_params: HashMap, + version_id: String, + host: String, + port: u16, + user_agent: String, + obj_info: ObjectInfo, + output: PutObjectOutput, +) { + let event_args = rustfs_notify::EventArgs { + event_name: EventName::ObjectCreatedPut, + bucket_name: bucket, + object: obj_info, + req_params, + resp_elements: extract_resp_elements(&S3Response::new(output)), + version_id, + host, + port, + user_agent, + }; + + crate::storage::helper::spawn_background_with_context(request_context, async move { + notify.notify(event_args).await; + }); +} + +pub(super) async fn get_validated_store_adapter(bucket: &str) -> S3Result> { + get_validated_store(bucket).await +} + +pub(super) async fn bucket_prefix_versioning_enabled(bucket: &str, key: &str) -> bool { + BucketVersioningSys::prefix_enabled(bucket, key).await +} + +pub(super) async fn authorize_extract_put_target( + request_context: &PutObjectRequestContext, + bucket: &str, + object: &str, +) -> S3Result<()> { + let mut auth_req = S3Request { + input: PutObjectInput::default(), + method: request_context.method.clone(), + uri: request_context.uri.clone(), + headers: request_context.headers.clone(), + extensions: request_context.extensions.clone(), + credentials: request_context.credentials.clone(), + region: request_context.region.clone(), + service: request_context.service.clone(), + trailing_headers: request_context.trailing_headers.clone(), + }; + { + let req_info = req_info_mut(&mut auth_req)?; + req_info.bucket = Some(bucket.to_string()); + req_info.object = Some(object.to_string()); + req_info.version_id = None; + } + authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectAction)).await +} diff --git a/rustfs/src/app/object_usecase/get_object_flow.rs b/rustfs/src/app/object_usecase/get_object_flow.rs new file mode 100644 index 000000000..92934e4fa --- /dev/null +++ b/rustfs/src/app/object_usecase/get_object_flow.rs @@ -0,0 +1,280 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::DeadlockRequestGuard; +use super::app_adapters::{ + bucket_prefix_versioning_enabled, build_get_object_body_adapter, finalize_get_object_completion, + finalize_get_object_strategy_runtime, maybe_get_cached_get_object_flow_result, spawn_get_object_cache_writeback, +}; +use super::get_object_zero_copy::{GetObjectPreparedRead, prepare_get_object_read_execution}; +use super::types::GetObjectRequestContext; +use crate::error::ApiError; +use crate::storage::concurrency::{self, ConcurrencyManager, GetObjectGuard}; +use crate::storage::options::filter_object_metadata; +use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig}; +use rustfs_ecstore::store_api::{HTTPRangeSpec, ObjectInfo}; +use rustfs_object_io::get::{ + GetObjectBodyPlan as ObjectIoGetObjectBodyPlan, GetObjectBodySource, + GetObjectDataPlaneMetricContract as ObjectIoGetObjectDataPlaneMetricContract, GetObjectFlowResult, GetObjectOutputContext, + GetObjectReadSetup, build_chunk_blob as object_io_build_chunk_blob, + build_cors_wrapped_get_object_flow_result as object_io_build_cors_wrapped_get_object_flow_result, + build_get_object_checksums as object_io_build_get_object_checksums, + build_get_object_output_context as object_io_build_get_object_output_context, + chunk_body_data_plane_labels as object_io_chunk_body_data_plane_labels, +}; +use s3s::S3Result; +use s3s::dto::{ContentType, SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, ServerSideEncryption, Timestamp}; +use std::time::Duration; + +pub(super) struct GetObjectBootstrap { + pub(super) timeout_config: TimeoutConfig, + pub(super) wrapper: RequestTimeoutWrapper, + pub(super) request_start: std::time::Instant, + pub(super) request_guard: GetObjectGuard, + pub(super) _deadlock_request_guard: DeadlockRequestGuard, + pub(super) concurrent_requests: usize, +} + +#[derive(Clone, Copy)] +pub(super) struct GetObjectFlowRuntime<'a> { + pub(super) manager: &'a ConcurrencyManager, + pub(super) bootstrap: &'a GetObjectBootstrap, + pub(super) base_buffer_size: usize, +} + +#[allow(clippy::too_many_arguments)] +pub(super) async fn build_get_object_output_context( + request_context: &GetObjectRequestContext, + cache_key: &str, + manager: &ConcurrencyManager, + bucket: &str, + key: &str, + info: ObjectInfo, + event_info: ObjectInfo, + body_source: GetObjectBodySource, + rs: Option, + content_type: Option, + last_modified: Option, + response_content_length: i64, + content_range: Option, + server_side_encryption: Option, + sse_customer_algorithm: Option, + sse_customer_key_md5: Option, + ssekms_key_id: Option, + encryption_applied: bool, + permit_wait_duration: Duration, + queue_utilization: f64, + queue_status: &concurrency::IoQueueStatus, + concurrent_requests: usize, + base_buffer_size: usize, + part_number: Option, + versioned: bool, +) -> S3Result<(GetObjectOutputContext, ObjectIoGetObjectDataPlaneMetricContract)> { + let (io_strategy, optimal_buffer_size) = finalize_get_object_strategy_runtime( + base_buffer_size, + manager, + bucket, + key, + &info, + rs.as_ref(), + response_content_length, + permit_wait_duration, + queue_utilization, + queue_status, + concurrent_requests, + ); + + let (body, metric_contract) = match body_source { + GetObjectBodySource::Reader(final_stream) => { + let cache_eligibility = manager.get_object_cache_eligibility( + io_strategy.cache_writeback_enabled, + part_number.is_some(), + rs.is_some(), + encryption_applied, + response_content_length, + ); + let adapter_output = build_get_object_body_adapter( + final_stream, + &info, + cache_key, + response_content_length, + optimal_buffer_size, + cache_eligibility, + ) + .await?; + let metric_contract = ObjectIoGetObjectDataPlaneMetricContract::disk( + rustfs_io_metrics::IoPath::Legacy, + rustfs_io_metrics::CopyMode::SingleCopy, + adapter_output.body_plan, + ); + if let Some(writeback) = adapter_output.cache_writeback { + spawn_get_object_cache_writeback(cache_key, writeback, metric_contract); + } + + (adapter_output.body, metric_contract) + } + GetObjectBodySource::Chunk { + stream: chunk_stream, + path, + copy_mode, + } => { + let (io_path, copy_mode) = object_io_chunk_body_data_plane_labels(path, copy_mode); + ( + object_io_build_chunk_blob(chunk_stream), + ObjectIoGetObjectDataPlaneMetricContract::disk(io_path, copy_mode, ObjectIoGetObjectBodyPlan::Stream), + ) + } + }; + + let checksums = object_io_build_get_object_checksums(&info, &request_context.headers, part_number, rs.as_ref()) + .map_err(ApiError::from)?; + let filtered_metadata = filter_object_metadata(&info.user_defined); + + Ok(( + object_io_build_get_object_output_context( + body, + info, + event_info, + content_type, + last_modified, + response_content_length, + content_range, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + &checksums, + filtered_metadata, + versioned, + optimal_buffer_size, + Some(metric_contract.copy_mode), + ), + metric_contract, + )) +} + +pub(super) async fn run_get_object_flow( + request_context: GetObjectRequestContext, + runtime: GetObjectFlowRuntime<'_>, +) -> S3Result { + let GetObjectFlowRuntime { + manager, + bootstrap, + base_buffer_size, + } = runtime; + let timeout_config = &bootstrap.timeout_config; + let wrapper = &bootstrap.wrapper; + let request_start = bootstrap.request_start; + let concurrent_requests = bootstrap.concurrent_requests; + let bucket = request_context.bucket.clone(); + let key = request_context.key.clone(); + let cache_key = request_context.cache_key.clone(); + let version_id_for_event = request_context.version_id_for_event.clone(); + let part_number = request_context.part_number; + let rs = request_context.rs.clone(); + let opts = request_context.opts.clone(); + + if let Some(cached_result) = maybe_get_cached_get_object_flow_result( + manager, + &bucket, + &key, + &cache_key, + version_id_for_event.clone(), + part_number, + rs.as_ref(), + request_start, + ) + .await + { + return Ok(cached_result); + } + + let prepared_read = prepare_get_object_read_execution( + &request_context, + manager, + wrapper, + timeout_config, + &bucket, + &key, + rs, + &opts, + part_number, + ) + .await?; + let GetObjectPreparedRead { io_planning, read_setup } = prepared_read; + let permit_wait_duration = io_planning.permit_wait_duration; + let queue_status = io_planning.queue_status; + let queue_utilization = io_planning.queue_utilization; + + let GetObjectReadSetup { + info, + event_info, + body_source, + rs, + content_type, + last_modified, + response_content_length, + content_range, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + encryption_applied, + } = read_setup; + + let versioned = bucket_prefix_versioning_enabled(&bucket, &key).await; + let (output_context, metric_contract) = build_get_object_output_context( + &request_context, + &cache_key, + manager, + &bucket, + &key, + info, + event_info, + body_source, + rs, + content_type, + last_modified, + response_content_length, + content_range, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + encryption_applied, + permit_wait_duration, + queue_utilization, + &queue_status, + concurrent_requests, + base_buffer_size, + part_number, + versioned, + ) + .await?; + let response_content_length = output_context.response_content_length; + let optimal_buffer_size = output_context.optimal_buffer_size; + + let total_duration = request_start.elapsed(); + finalize_get_object_completion( + &cache_key, + wrapper, + timeout_config, + total_duration, + response_content_length, + optimal_buffer_size, + metric_contract, + ); + + Ok(object_io_build_cors_wrapped_get_object_flow_result(output_context, version_id_for_event)) +} diff --git a/rustfs/src/app/object_usecase/get_object_zero_copy.rs b/rustfs/src/app/object_usecase/get_object_zero_copy.rs new file mode 100644 index 000000000..0df64b82c --- /dev/null +++ b/rustfs/src/app/object_usecase/get_object_zero_copy.rs @@ -0,0 +1,338 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::app_adapters::get_validated_store_adapter; +use super::types::GetObjectRequestContext; +use crate::error::ApiError; +use crate::storage::concurrency::{self, ConcurrencyManager}; +use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig}; +use crate::storage::{ + DecryptionRequest, check_preconditions, sse_decryption, validate_sse_headers_for_read, validate_ssec_for_read, +}; +use http::HeaderMap; +use rustfs_concurrency::GetObjectQueueSnapshot; +use rustfs_ecstore::store_api::{HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectOptions}; +use rustfs_object_io::get::{ + ChunkReadDecision, ChunkReadPlanError, GetObjectEncryptionState as ObjectIoGetObjectEncryptionState, GetObjectReadSetup, + build_reader_read_setup as object_io_build_reader_read_setup, + finalize_chunk_read_setup as object_io_finalize_chunk_read_setup, + get_object_chunk_fast_path_guard as object_io_get_object_chunk_fast_path_guard, plan_chunk_read as object_io_plan_chunk_read, + plan_legacy_read as object_io_plan_legacy_read, +}; +use rustfs_rio::{Reader, WarpReader}; +use s3s::{S3Error, S3ErrorCode, S3Result, s3_error}; +use std::time::Duration; +use tracing::{debug, warn}; + +pub(super) struct GetObjectIoPlanning<'a> { + pub(super) _disk_permit: tokio::sync::SemaphorePermit<'a>, + pub(super) permit_wait_duration: Duration, + pub(super) queue_status: concurrency::IoQueueStatus, + pub(super) queue_utilization: f64, +} + +pub(super) struct GetObjectPreparedRead<'a> { + pub(super) io_planning: GetObjectIoPlanning<'a>, + pub(super) read_setup: GetObjectReadSetup, +} + +pub(super) async fn acquire_get_object_io_planning<'a>( + manager: &'a ConcurrencyManager, + wrapper: &RequestTimeoutWrapper, + timeout_config: &TimeoutConfig, + bucket: &str, + key: &str, +) -> S3Result> { + let permit_wait_start = std::time::Instant::now(); + let disk_permit = manager + .acquire_disk_read_permit() + .await + .map_err(|_| s3_error!(InternalError, "disk read semaphore closed"))?; + let permit_wait_duration = permit_wait_start.elapsed(); + + if wrapper.is_timeout() { + warn!( + bucket = %bucket, + key = %key, + wait_ms = permit_wait_duration.as_millis(), + timeout_secs = timeout_config.get_object_timeout.as_secs(), + elapsed_ms = wrapper.elapsed().as_millis(), + "GetObject request timed out while waiting for disk permit" + ); + + rustfs_io_metrics::record_get_object_timeout(Some("disk_permit"), Some(wrapper.elapsed().as_secs_f64())); + return Err(s3_error!(InternalError, "Request timeout while waiting for disk permit")); + } + + let queue_status = manager.io_queue_status(); + let queue_snapshot = GetObjectQueueSnapshot::from_available_permits( + queue_status.total_permits, + queue_status.total_permits.saturating_sub(queue_status.permits_in_use), + ); + let queue_utilization = queue_snapshot.utilization_percent(); + + if queue_snapshot.is_congested(80.0) { + warn!( + bucket = %bucket, + key = %key, + queue_utilization = format!("{:.1}%", queue_utilization), + permits_in_use = queue_status.permits_in_use, + total_permits = queue_status.total_permits, + "I/O queue congestion detected" + ); + + rustfs_io_metrics::record_io_queue_congestion(); + } + + if wrapper.is_timeout() { + warn!( + bucket = %bucket, + key = %key, + timeout_secs = timeout_config.get_object_timeout.as_secs(), + elapsed_ms = wrapper.elapsed().as_millis(), + "GetObject request timed out before reading object" + ); + rustfs_io_metrics::record_get_object_timeout(Some("before_read"), Some(wrapper.elapsed().as_secs_f64())); + return Err(s3_error!(InternalError, "Request timeout before reading object")); + } + + Ok(GetObjectIoPlanning { + _disk_permit: disk_permit, + permit_wait_duration, + queue_status, + queue_utilization, + }) +} + +#[allow(clippy::too_many_arguments)] +pub(super) async fn prepare_get_object_read( + request_context: &GetObjectRequestContext, + store: &rustfs_ecstore::store::ECStore, + manager: &ConcurrencyManager, + bucket: &str, + key: &str, + rs: Option, + h: HeaderMap, + opts: &ObjectOptions, + part_number: Option, + read_start: std::time::Instant, +) -> S3Result { + let reader = store + .get_object_reader(bucket, key, rs.clone(), h, opts) + .await + .map_err(ApiError::from)?; + + let info = reader.object_info; + + let read_duration = read_start.elapsed(); + rustfs_io_metrics::record_io_path_selected("get", rustfs_io_metrics::IoPath::Legacy); + + manager.record_disk_operation(info.size as u64, read_duration, true).await; + + check_preconditions(&request_context.headers, &info)?; + + debug!(object_size = info.size, part_count = info.parts.len(), "GET object metadata snapshot"); + for part in &info.parts { + debug!( + part_number = part.number, + part_size = part.size, + part_actual_size = part.actual_size, + "GET object part details" + ); + } + + let event_info = info.clone(); + validate_sse_headers_for_read(&info.user_defined, &request_context.headers)?; + validate_ssec_for_read( + &info.user_defined, + request_context.sse_customer_key.as_ref(), + request_context.sse_customer_key_md5.as_ref(), + )?; + let read_plan = object_io_plan_legacy_read(&info, rs, part_number).map_err(ApiError::from)?; + + debug!( + "GET object metadata check: parts={}, provided_sse_key={:?}", + info.parts.len(), + request_context.sse_customer_key.is_some() + ); + + let decryption_request = DecryptionRequest { + bucket, + key, + metadata: &info.user_defined, + sse_customer_key: request_context.sse_customer_key.as_ref(), + sse_customer_key_md5: request_context.sse_customer_key_md5.as_ref(), + part_number: None, + parts: &info.parts, + etag: info.etag.as_deref(), + }; + + let encrypted_stream = reader.stream; + + let (encryption_state, final_stream) = match sse_decryption(decryption_request).await? { + Some(material) => { + let server_side_encryption = Some(material.server_side_encryption.clone()); + let sse_customer_algorithm = Some(material.algorithm.clone()); + let sse_customer_key_md5 = material.customer_key_md5.clone(); + let ssekms_key_id = material.kms_key_id.clone(); + let (decrypted_stream, plaintext_size) = material + .wrap_reader(encrypted_stream, read_plan.response_content_length) + .await + .map_err(ApiError::from)?; + + ( + ObjectIoGetObjectEncryptionState { + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + encryption_applied: true, + response_content_length_override: Some(plaintext_size), + }, + decrypted_stream, + ) + } + None => ( + ObjectIoGetObjectEncryptionState::default(), + Box::new(WarpReader::new(encrypted_stream)) as Box, + ), + }; + + Ok(object_io_build_reader_read_setup( + info, + event_info, + final_stream, + read_plan, + encryption_state, + )) +} + +#[allow(clippy::too_many_arguments)] +pub(super) async fn prepare_get_object_read_execution<'a>( + request_context: &GetObjectRequestContext, + manager: &'a ConcurrencyManager, + wrapper: &RequestTimeoutWrapper, + timeout_config: &TimeoutConfig, + bucket: &str, + key: &str, + rs: Option, + opts: &ObjectOptions, + part_number: Option, +) -> S3Result> { + let h = HeaderMap::new(); + let io_planning = acquire_get_object_io_planning(manager, wrapper, timeout_config, bucket, key).await?; + let store = get_validated_store_adapter(bucket).await?; + + let read_start = std::time::Instant::now(); + let read_setup = match object_io_get_object_chunk_fast_path_guard( + request_context.sse_customer_key.is_some(), + request_context.sse_customer_key_md5.is_some(), + ) { + Ok(()) => match prepare_get_object_chunk_read( + request_context, + &store, + manager, + bucket, + key, + rs.clone(), + part_number, + opts, + read_start, + ) + .await? + { + Some(read_setup) => read_setup, + None => { + prepare_get_object_read(request_context, &store, manager, bucket, key, rs, h, opts, part_number, read_start) + .await? + } + }, + Err(fallback) => { + rustfs_io_metrics::record_io_fallback(fallback.stage, fallback.reason); + prepare_get_object_read(request_context, &store, manager, bucket, key, rs, h, opts, part_number, read_start).await? + } + }; + + Ok(GetObjectPreparedRead { io_planning, read_setup }) +} + +#[allow(clippy::too_many_arguments)] +pub(super) async fn prepare_get_object_chunk_read( + request_context: &GetObjectRequestContext, + store: &rustfs_ecstore::store::ECStore, + manager: &ConcurrencyManager, + bucket: &str, + key: &str, + mut rs: Option, + part_number: Option, + opts: &ObjectOptions, + read_start: std::time::Instant, +) -> S3Result> { + let info = store.get_object_info(bucket, key, opts).await.map_err(ApiError::from)?; + + validate_sse_headers_for_read(&info.user_defined, &request_context.headers)?; + validate_ssec_for_read( + &info.user_defined, + request_context.sse_customer_key.as_ref(), + request_context.sse_customer_key_md5.as_ref(), + )?; + check_preconditions(&request_context.headers, &info)?; + + let encrypted_object = info.user_defined.contains_key("x-rustfs-encryption-key") + || info + .user_defined + .contains_key("x-amz-server-side-encryption-customer-algorithm"); + if encrypted_object { + rustfs_io_metrics::record_io_fallback( + rustfs_io_metrics::IoStage::ReadSetup, + rustfs_io_metrics::FallbackReason::EncryptionEnabled, + ); + return Ok(None); + } + + let plan = match object_io_plan_chunk_read(&info, opts.version_id.is_none(), rs.clone(), part_number) { + Ok(ChunkReadDecision::Eligible(plan)) => plan, + Ok(ChunkReadDecision::Fallback(fallback)) => { + rustfs_io_metrics::record_io_fallback(fallback.stage, fallback.reason); + return Ok(None); + } + Err(ChunkReadPlanError::NoSuchKey) => return Err(S3Error::new(S3ErrorCode::NoSuchKey)), + Err(ChunkReadPlanError::MethodNotAllowed) => return Err(S3Error::new(S3ErrorCode::MethodNotAllowed)), + Err(ChunkReadPlanError::Io(err)) => return Err(ApiError::from(err).into()), + }; + rs = plan.rs.clone(); + + let read_duration = read_start.elapsed(); + manager.record_disk_operation(info.size as u64, read_duration, true).await; + let event_info = info.clone(); + + let chunk_result = match store + .get_object_chunks(bucket, key, rs.clone(), HeaderMap::new(), opts) + .await + .map_err(ApiError::from) + { + Ok(result) => result, + Err(_err) => { + rustfs_io_metrics::record_io_fallback( + rustfs_io_metrics::IoStage::HttpBridge, + rustfs_io_metrics::FallbackReason::ChunkBridgeUnavailable, + ); + return Ok(None); + } + }; + let setup_result = object_io_finalize_chunk_read_setup(info, event_info, chunk_result, plan); + rustfs_io_metrics::record_io_path_selected("get", setup_result.io_path); + + Ok(Some(setup_result.read_setup)) +} diff --git a/rustfs/src/app/object_usecase/put_object_extract.rs b/rustfs/src/app/object_usecase/put_object_extract.rs new file mode 100644 index 000000000..0f8071b50 --- /dev/null +++ b/rustfs/src/app/object_usecase/put_object_extract.rs @@ -0,0 +1,499 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::app::context::NotifyInterface; +use rustfs_object_io::put::{ + apply_extract_entry_pax_extensions, apply_trailing_checksums, is_sse_kms_requested, map_extract_archive_error, + normalize_extract_entry_key, resolve_put_object_extract_options, +}; + +impl DefaultObjectUsecase { + pub(super) async fn run_put_object_extract_flow( + input: PutObjectInput, + request_context: PutObjectRequestContext, + notify: Arc, + resolved_size: i64, + ) -> S3Result { + if is_sse_kms_requested(&input, &request_context.headers) { + return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for extract uploads")); + } + + let PutObjectInput { + body, + bucket, + key, + version_id, + cache_control, + content_disposition, + content_encoding, + content_length: _content_length, + content_language, + content_type, + content_md5, + expires, + object_lock_legal_hold_status, + object_lock_mode, + object_lock_retain_until_date, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key, + sse_customer_key_md5, + ssekms_key_id, + storage_class, + tagging, + website_redirect_location, + .. + } = input; + + let event_version_id = version_id; + let (h_algo, h_key, h_md5) = extract_ssec_params_from_headers(&request_context.headers)?; + let sse_customer_algorithm = sse_customer_algorithm.or(h_algo); + let sse_customer_key = sse_customer_key.or(h_key); + let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5); + + let original_sse = server_side_encryption.or(extract_server_side_encryption_from_headers(&request_context.headers)?); + let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + let mut effective_sse = original_sse.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + config.rules.first().and_then(|rule| { + rule.apply_server_side_encryption_by_default + .as_ref() + .map(|sse| match sse.sse_algorithm.as_str() { + "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), + _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + }) + }) + }) + }); + let mut effective_kms_key_id = ssekms_key_id.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + config.rules.first().and_then(|rule| { + rule.apply_server_side_encryption_by_default + .as_ref() + .and_then(|sse| sse.kms_master_key_id.clone()) + }) + }) + }); + if effective_sse + .as_ref() + .is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) + { + return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for extract uploads")); + } + validate_sse_headers_for_write( + effective_sse.as_ref(), + effective_kms_key_id.as_ref(), + sse_customer_algorithm.as_ref(), + sse_customer_key.as_ref(), + sse_customer_key_md5.as_ref(), + true, + )?; + let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; + + let size = resolved_size; + validate_object_key(&key, "PUT")?; + + let buffer_size = get_buffer_size_opt_in(size); + let body = tokio::io::BufReader::with_capacity( + buffer_size, + StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), + ); + + let Some(ext) = Path::new(&key).extension().and_then(|s| s.to_str()) else { + return Err(s3_error!(InvalidArgument, "key extension not found")); + }; + + let ext = ext.to_owned(); + + let md5hex = if let Some(base64_md5) = content_md5 { + let md5 = base64_simd::STANDARD + .decode_to_vec(base64_md5.as_bytes()) + .map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?; + Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower)) + } else { + None + }; + + let sha256hex = get_content_sha256_with_query(&request_context.headers, request_context.uri_query.as_deref()); + let actual_size = size; + + let mut archive_reader = + HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?; + + if let Err(err) = + archive_reader.add_checksum_from_s3s(&request_context.headers, request_context.trailing_headers.clone(), false) + { + return Err(ApiError::from(err).into()); + } + + let archive_etag = Arc::new(Mutex::new(None)); + let decoder = CompressionFormat::from_extension(&ext) + .get_decoder(ExtractArchiveEtagReader::new(archive_reader, archive_etag.clone())) + .map_err(|e| { + error!("get_decoder err {:?}", e); + s3_error!(InvalidArgument, "get_decoder err") + })?; + + let mut ar = Archive::new(decoder); + let mut entries = ar.entries().map_err(|e| { + error!("get entries err {:?}", e); + s3_error!(InvalidArgument, "get entries err") + })?; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + let extract_options = resolve_put_object_extract_options(&request_context.headers); + let version_id = match event_version_id { + Some(v) => v.to_string(), + None => String::new(), + }; + + let req_params = extract_params_header(&request_context.headers); + let host = get_request_host(&request_context.headers); + let port = get_request_port(&request_context.headers); + let user_agent = get_request_user_agent(&request_context.headers); + let tracing_context = request_context + .extensions + .get::() + .cloned(); + + while let Some(entry) = entries.next().await { + let mut f = match entry { + Ok(f) => f, + Err(e) => { + if extract_options.ignore_errors { + warn!("Skipping archive entry because read failed and ignore-errors is enabled: {e}"); + continue; + } + error!("Failed to read archive entry: {}", e); + return Err(s3_error!(InvalidArgument, "Failed to read archive entry: {:?}", e)); + } + }; + + let fpath = match f.path() { + Ok(path) => path, + Err(e) => { + if extract_options.ignore_errors { + warn!("Skipping archive entry because path decode failed and ignore-errors is enabled: {e}"); + continue; + } + return Err(s3_error!(InvalidArgument, "Failed to decode archive entry path")); + } + }; + + let is_dir = f.header().entry_type().is_dir(); + let fpath = normalize_extract_entry_key(&fpath.to_string_lossy(), extract_options.prefix.as_deref(), is_dir); + + authorize_extract_put_target(&request_context, &bucket, &fpath).await?; + + let mut size = f.header().size().unwrap_or_default() as i64; + let archive_entry_mod_time = f + .header() + .mtime() + .ok() + .and_then(|modified_at_secs| OffsetDateTime::from_unix_timestamp(modified_at_secs as i64).ok()); + let mut metadata = HashMap::new(); + apply_put_request_metadata( + &mut metadata, + &request_context.headers, + &fpath, + cache_control.clone(), + content_disposition.clone(), + content_encoding.clone(), + content_language.clone(), + content_type.clone(), + expires.clone(), + website_redirect_location.clone(), + tagging.clone(), + storage_class.clone(), + )?; + let mut opts = put_opts(&bucket, &fpath, None, &request_context.headers, metadata.clone()) + .await + .map_err(ApiError::from)?; + apply_extract_entry_pax_extensions(&mut f, &mut metadata, &mut opts).await?; + if archive_entry_mod_time.is_some() { + opts.mod_time = archive_entry_mod_time; + } + + debug!("Extracting file: {}, size: {} bytes", fpath, size); + + if is_dir { + if extract_options.ignore_dirs { + debug!("Skipping directory entry during archive extract: {}", fpath); + continue; + } + size = 0; + } + + let actual_size = size; + let should_compress = !is_dir && is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64; + + let mut hrd = if is_dir { + HashReader::from_stream(std::io::Cursor::new(Vec::new()), size, actual_size, None, None, false) + .map_err(ApiError::from)? + } else if should_compress { + insert_str(&mut metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string()); + insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); + + let hrd = HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)?; + size = HashReader::SIZE_PRESERVE_LAYER; + HashReader::from_reader( + CompressReader::new(hrd, CompressionAlgorithm::default()), + size, + actual_size, + None, + None, + false, + ) + .map_err(ApiError::from)? + } else { + HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)? + }; + apply_put_request_object_lock_opts( + &bucket, + object_lock_legal_hold_status.clone(), + object_lock_mode.clone(), + object_lock_retain_until_date.clone(), + &mut opts, + ) + .await?; + if let Some(material) = sse_encryption(EncryptionRequest { + bucket: &bucket, + key: &fpath, + server_side_encryption: effective_sse.clone(), + ssekms_key_id: effective_kms_key_id.clone(), + sse_customer_algorithm: sse_customer_algorithm.clone(), + sse_customer_key: sse_customer_key.clone(), + sse_customer_key_md5: sse_customer_key_md5.clone(), + content_size: actual_size, + part_number: None, + part_key: None, + part_nonce: None, + }) + .await? + { + effective_sse = Some(material.server_side_encryption.clone()); + effective_kms_key_id = material.kms_key_id.clone(); + + let encrypted_reader = material.wrap_reader(hrd); + hrd = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) + .map_err(ApiError::from)?; + + let encryption_metadata = material.metadata; + metadata.extend(encryption_metadata.clone()); + opts.user_defined.extend(encryption_metadata); + } + opts.user_defined.extend(metadata); + let mut reader = rustfs_ecstore::store_api::ChunkNativePutData::new(hrd); + + let obj_info = match store.put_object(&bucket, &fpath, &mut reader, &opts).await { + Ok(info) => info, + Err(e) => { + if extract_options.ignore_errors { + warn!("Skipping archive entry because object write failed and ignore-errors is enabled: {e}"); + continue; + } + return Err(ApiError::from(e).into()); + } + }; + + let manager = get_concurrency_manager(); + let fpath_clone = fpath.clone(); + let bucket_clone = bucket.clone(); + crate::storage::request_context::spawn_traced(async move { + manager.invalidate_cache_versioned(&bucket_clone, &fpath_clone, None).await; + }); + + let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)); + + let output = PutObjectOutput { + e_tag, + ..Default::default() + }; + + spawn_put_extract_notification( + notify.clone(), + tracing_context.clone(), + bucket.clone(), + req_params.clone(), + version_id.clone(), + host.clone(), + port, + user_agent.clone(), + obj_info.clone(), + output, + ); + } + + let mut checksums = PutObjectChecksums { + crc32: input.checksum_crc32, + crc32c: input.checksum_crc32c, + sha1: input.checksum_sha1, + sha256: input.checksum_sha256, + crc64nvme: input.checksum_crc64nvme, + }; + apply_trailing_checksums( + input.checksum_algorithm.as_ref().map(|a| a.as_str()), + &request_context.trailing_headers, + &mut checksums, + ); + + drop(entries); + let mut decoder = match ar.into_inner() { + Ok(decoder) => decoder, + Err(_) => return Err(s3_error!(InvalidArgument, "Failed to finalize archive reader")), + }; + tokio::io::copy(&mut decoder, &mut tokio::io::sink()) + .await + .map_err(map_extract_archive_error)?; + let archive_etag = archive_etag + .lock() + .ok() + .and_then(|etag| etag.clone()) + .map(|etag| to_s3s_etag(&etag)); + + let output = PutObjectOutput { + e_tag: archive_etag, + checksum_crc32: checksums.crc32, + checksum_crc32c: checksums.crc32c, + checksum_sha1: checksums.sha1, + checksum_sha256: checksums.sha256, + checksum_crc64nvme: checksums.crc64nvme, + ..Default::default() + }; + Ok(output) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use http::{Extensions, HeaderMap, HeaderValue, Method, Uri}; + use rustfs_utils::http::headers::{AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_SNOWBALL_EXTRACT}; + + fn build_request(input: T, method: Method) -> S3Request { + S3Request { + input, + method, + uri: Uri::from_static("/"), + headers: HeaderMap::new(), + extensions: Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + } + } + + #[tokio::test] + async fn execute_put_object_rejects_post_object_sse_kms_from_input() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .server_side_encryption(Some(ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS))) + .build() + .unwrap(); + + let mut req = build_request(input, Method::POST); + req.extensions.insert(PostObjectRequestMarker); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + + #[tokio::test] + async fn execute_put_object_rejects_extract_sse_kms() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("archive.tar".to_string()) + .server_side_encryption(Some(ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS))) + .build() + .unwrap(); + + let mut req = build_request(input, Method::PUT); + req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + + #[tokio::test] + async fn execute_put_object_extract_rejects_invalid_storage_class() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("archive.tar".to_string()) + .storage_class(Some(StorageClass::from_static("INVALID"))) + .build() + .unwrap(); + + let mut req = build_request(input, Method::PUT); + req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass); + } + + #[tokio::test] + async fn execute_put_object_rejects_post_object_sse_kms_from_headers() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .build() + .unwrap(); + + let mut req = build_request(input, Method::POST); + req.extensions.insert(PostObjectRequestMarker); + req.headers + .insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("aws:kms")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + + #[tokio::test] + async fn execute_put_object_rejects_post_object_sse_kms_key_id_header() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .build() + .unwrap(); + + let mut req = build_request(input, Method::POST); + req.extensions.insert(PostObjectRequestMarker); + req.headers + .insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, HeaderValue::from_static("test-kms-key-id")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } +} diff --git a/rustfs/src/app/object_usecase/put_object_flow.rs b/rustfs/src/app/object_usecase/put_object_flow.rs new file mode 100644 index 000000000..c64d97e73 --- /dev/null +++ b/rustfs/src/app/object_usecase/put_object_flow.rs @@ -0,0 +1,868 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use bytes::Buf; +use futures::{Stream, StreamExt}; +use rustfs_ecstore::config::GLOBAL_STORAGE_CLASS; +use rustfs_io_core::{BytesPool, PooledBuffer}; +use rustfs_object_io::put::{ + PutObjectChecksums, PutObjectIngressKind, PutObjectLegacyHashStagePlan, PutObjectLegacyHashValues, PutObjectTransformStage, + apply_trailing_checksums, build_put_object_ingress_source, build_put_object_legacy_hash_stage, + build_put_object_plain_hash_stage, plan_put_object_body_with_transforms, resolve_put_transformed_fallback_reason, +}; +use rustfs_rio::{BlockReadable, BoxReadBlockFuture, EtagResolvable, HashReaderDetector, TryGetIndex}; +use rustfs_utils::http::headers::AMZ_TRAILER; + +const DEFAULT_SMALL_PUT_EAGER_MAX_BYTES: i64 = 1024 * 1024; +const ENV_RUSTFS_PUT_SMALL_EAGER_MAX_BYTES: &str = "RUSTFS_PUT_SMALL_EAGER_MAX_BYTES"; +const ENV_RUSTFS_PUT_FORCE_DISABLE_SMALL_EAGER: &str = "RUSTFS_PUT_FORCE_DISABLE_SMALL_EAGER"; +const SLOW_PUT_PHASE_DEBUG_THRESHOLD_MS: u64 = 100; +const SLOW_PUT_PHASE_WARN_THRESHOLD_MS: u64 = 1_000; +const SLOW_PUT_PHASE_ERROR_THRESHOLD_MS: u64 = 5_000; + +fn resolved_checksum_bytes(checksums: &PutObjectChecksums) -> Option { + [ + (rustfs_rio::ChecksumType::CRC32, checksums.crc32.as_deref()), + (rustfs_rio::ChecksumType::CRC32C, checksums.crc32c.as_deref()), + (rustfs_rio::ChecksumType::SHA1, checksums.sha1.as_deref()), + (rustfs_rio::ChecksumType::SHA256, checksums.sha256.as_deref()), + (rustfs_rio::ChecksumType::CRC64_NVME, checksums.crc64nvme.as_deref()), + ] + .into_iter() + .find_map(|(checksum_type, value)| { + value.and_then(|value| rustfs_rio::Checksum::new_with_type(checksum_type, value).map(|checksum| checksum.to_bytes(&[]))) + }) +} + +fn clamp_small_put_eager_max_bytes(inline_object_limit_bytes: Option) -> i64 { + inline_object_limit_bytes + .unwrap_or(DEFAULT_SMALL_PUT_EAGER_MAX_BYTES as usize) + .min(DEFAULT_SMALL_PUT_EAGER_MAX_BYTES as usize) as i64 +} + +fn env_flag_enabled(name: &str) -> bool { + rustfs_utils::get_env_bool(name, false) +} + +fn env_non_negative_i64(name: &str) -> Option { + rustfs_utils::get_env_opt_i64(name).filter(|value| *value >= 0) +} + +fn topology_aware_small_put_eager_max_bytes(store: &rustfs_ecstore::store::ECStore, versioned: bool) -> i64 { + let Some(first_pool) = store.pools.first() else { + return DEFAULT_SMALL_PUT_EAGER_MAX_BYTES; + }; + + let data_shards = first_pool + .set_drive_count + .saturating_sub(first_pool.default_parity_count) + .max(1); + + let inline_object_limit = GLOBAL_STORAGE_CLASS + .get() + .map(|config| config.inline_object_limit_bytes(data_shards, versioned)); + + clamp_small_put_eager_max_bytes(inline_object_limit) +} + +fn resolved_small_put_eager_max_bytes(default_max_bytes: i64) -> i64 { + if env_flag_enabled(ENV_RUSTFS_PUT_FORCE_DISABLE_SMALL_EAGER) { + return 0; + } + + env_non_negative_i64(ENV_RUSTFS_PUT_SMALL_EAGER_MAX_BYTES) + .map(|value| value.min(DEFAULT_SMALL_PUT_EAGER_MAX_BYTES).min(default_max_bytes)) + .unwrap_or(default_max_bytes) +} + +fn should_use_small_put_eager_path(size: i64, eager_max_bytes: i64, compression_enabled: bool, encryption_enabled: bool) -> bool { + size > 0 && size <= eager_max_bytes && !compression_enabled && !encryption_enabled +} + +fn request_uses_trailing_checksum(headers: &HeaderMap, trailing_headers: &Option) -> bool { + trailing_headers.is_some() + || headers.contains_key(AMZ_TRAILER) + || matches!( + rustfs_rio::get_content_checksum(headers), + Ok(Some(checksum)) if checksum.checksum_type.trailing() + ) +} + +fn put_path_label(small_eager: bool, reduced_copy: bool, compressed: bool) -> &'static str { + if small_eager { + "small_eager" + } else if compressed { + "compressed" + } else if reduced_copy { + "reduced_copy" + } else { + "legacy_plain" + } +} + +#[allow(clippy::too_many_arguments)] +fn log_put_flow_phase( + bucket: &str, + key: &str, + phase: &str, + elapsed: std::time::Duration, + object_size: i64, + small_eager: bool, + reduced_copy: bool, + compressed: bool, + encrypted: bool, +) { + let duration_ms = elapsed.as_millis() as u64; + if duration_ms < SLOW_PUT_PHASE_DEBUG_THRESHOLD_MS { + return; + } + + let put_path = put_path_label(small_eager, reduced_copy, compressed); + if duration_ms >= SLOW_PUT_PHASE_ERROR_THRESHOLD_MS { + error!( + phase, + duration_ms, object_size, put_path, compressed, encrypted, bucket, key, "Small PUT phase is critically slow" + ); + } else if duration_ms >= SLOW_PUT_PHASE_WARN_THRESHOLD_MS { + warn!( + phase, + duration_ms, object_size, put_path, compressed, encrypted, bucket, key, "Small PUT phase is slow" + ); + } else { + debug!( + phase, + duration_ms, object_size, put_path, compressed, encrypted, bucket, key, "Small PUT phase exceeded debug threshold" + ); + } +} + +struct PooledBufferReader { + buffer: PooledBuffer, + position: usize, +} + +impl PooledBufferReader { + fn new(buffer: PooledBuffer) -> Self { + Self { buffer, position: 0 } + } +} + +impl tokio::io::AsyncRead for PooledBufferReader { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let remaining = &self.buffer[self.position..]; + if remaining.is_empty() { + return std::task::Poll::Ready(Ok(())); + } + + let to_copy = remaining.len().min(buf.remaining()); + buf.put_slice(&remaining[..to_copy]); + self.position += to_copy; + std::task::Poll::Ready(Ok(())) + } +} + +impl BlockReadable for PooledBufferReader { + fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> { + Box::pin(async move { + let remaining = &self.buffer[self.position..]; + if remaining.is_empty() { + return Ok(0); + } + + let to_copy = remaining.len().min(buf.len()); + buf[..to_copy].copy_from_slice(&remaining[..to_copy]); + self.position += to_copy; + Ok(to_copy) + }) + } +} + +impl EtagResolvable for PooledBufferReader {} + +impl HashReaderDetector for PooledBufferReader {} + +impl TryGetIndex for PooledBufferReader {} + +async fn read_small_put_body_eager(body: S, size: i64, pool: std::sync::Arc) -> S3Result +where + S: Stream>, + B: Buf, + E: std::fmt::Display, +{ + let expected_len = usize::try_from(size).map_err(|_| s3_error!(InvalidRequest, "Object size overflow"))?; + let mut data = pool.acquire_buffer(expected_len).await; + let mut body = Box::pin(body); + + while let Some(result) = body.next().await { + let mut chunk = result.map_err(|err| S3Error::with_message(S3ErrorCode::IncompleteBody, err.to_string()))?; + let chunk_len = chunk.remaining(); + if chunk_len == 0 { + continue; + } + + let new_len = data + .len() + .checked_add(chunk_len) + .ok_or_else(|| s3_error!(InvalidRequest, "Object size overflow"))?; + if new_len > expected_len { + return Err(s3_error!(IncompleteBody)); + } + + let start = data.len(); + data.resize(new_len, 0); + chunk.copy_to_slice(&mut data[start..new_len]); + + if data.len() == expected_len { + return Ok(data); + } + } + + if data.len() != expected_len { + return Err(s3_error!(IncompleteBody)); + } + + Ok(data) +} + +async fn build_small_put_eager_hash_stage( + body: S, + size: i64, + pool: std::sync::Arc, + hash_values: PutObjectLegacyHashValues, + headers: &HeaderMap, + trailing_headers: Option, +) -> S3Result +where + S: Stream>, + B: Buf, + E: std::fmt::Display, +{ + let data = read_small_put_body_eager(body, size, pool).await?; + build_put_object_legacy_hash_stage( + Box::new(PooledBufferReader::new(data)), + hash_values, + PutObjectLegacyHashStagePlan { + size, + actual_size: size, + apply_s3_checksum: true, + ignore_s3_checksum_value: false, + }, + headers, + trailing_headers, + ) + .map_err(ApiError::from) + .map_err(Into::into) +} + +impl DefaultObjectUsecase { + pub(super) async fn run_put_object_flow( + input: PutObjectInput, + request_context: PutObjectRequestContext, + request_method_name: &'static str, + resolved_size: i64, + ) -> S3Result { + let start_time = std::time::Instant::now(); + + let PutObjectInput { + body, + bucket, + cache_control, + key, + content_length: _content_length, + content_disposition, + content_encoding, + content_language, + content_type, + expires, + tagging, + metadata, + version_id, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key, + sse_customer_key_md5, + ssekms_key_id, + content_md5, + object_lock_legal_hold_status, + object_lock_mode, + object_lock_retain_until_date, + storage_class, + website_redirect_location, + .. + } = input; + + let (h_algo, h_key, h_md5) = extract_ssec_params_from_headers(&request_context.headers)?; + let sse_customer_algorithm = sse_customer_algorithm.or(h_algo); + let sse_customer_key = sse_customer_key.or(h_key); + let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5); + + let server_side_encryption = + server_side_encryption.or(extract_server_side_encryption_from_headers(&request_context.headers)?); + + validate_object_key(&key, request_method_name)?; + + let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; + + let mut size = resolved_size; + let mut transform_stage = PutObjectTransformStage::default(); + let mut plain_reduced_copy_stage = false; + let mut small_object_eager_stage = false; + let bytes_pool = get_concurrency_manager().bytes_pool(); + + let store = get_validated_store_adapter(&bucket).await?; + + let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + + let mut effective_sse = server_side_encryption.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + config.rules.first().and_then(|rule| { + rule.apply_server_side_encryption_by_default + .as_ref() + .map(|sse| match sse.sse_algorithm.as_str() { + "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), + _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + }) + }) + }) + }); + + let mut effective_kms_key_id = ssekms_key_id.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + config.rules.first().and_then(|rule| { + rule.apply_server_side_encryption_by_default + .as_ref() + .and_then(|sse| sse.kms_master_key_id.clone()) + }) + }) + }); + + validate_sse_headers_for_write( + effective_sse.as_ref(), + effective_kms_key_id.as_ref(), + sse_customer_algorithm.as_ref(), + sse_customer_key.as_ref(), + sse_customer_key_md5.as_ref(), + true, + )?; + + let encryption_enabled_for_put = effective_sse.is_some() + || effective_kms_key_id.is_some() + || sse_customer_algorithm.is_some() + || sse_customer_key.is_some() + || sse_customer_key_md5.is_some(); + + let body_plan = plan_put_object_body_with_transforms( + size, + &request_context.headers, + &key, + get_buffer_size_opt_in(size), + encryption_enabled_for_put, + ); + if body_plan.ingress.kind == PutObjectIngressKind::ReducedCopyCandidate { + rustfs_io_metrics::record_put_object_attempted_fast_path(size); + debug!( + encryption_enabled = encryption_enabled_for_put, + compressed = body_plan.should_compress(), + "Zero-copy write enabled for {} byte object (bucket={}, key={})", + size, + bucket, + key + ); + } else if let Some(reason) = resolve_put_transformed_fallback_reason( + body_plan.ingress.kind, + body_plan.should_compress(), + encryption_enabled_for_put, + ) { + rustfs_io_metrics::record_io_fallback(rustfs_io_metrics::IoStage::PutTransform, reason); + rustfs_io_metrics::record_put_fallback(size, reason); + } + + let mut metadata = metadata.unwrap_or_default(); + apply_put_request_metadata( + &mut metadata, + &request_context.headers, + &key, + cache_control, + content_disposition, + content_encoding, + content_language, + content_type, + expires, + website_redirect_location, + tagging, + storage_class.clone(), + )?; + + let mut opts: ObjectOptions = put_opts(&bucket, &key, version_id.clone(), &request_context.headers, metadata.clone()) + .await + .map_err(ApiError::from)?; + apply_put_request_object_lock_opts( + &bucket, + object_lock_legal_hold_status, + object_lock_mode, + object_lock_retain_until_date, + &mut opts, + ) + .await?; + let eager_max_bytes = + resolved_small_put_eager_max_bytes(topology_aware_small_put_eager_max_bytes(&store, opts.versioned)); + let can_use_small_put_eager = + !request_uses_trailing_checksum(&request_context.headers, &request_context.trailing_headers); + + let current_opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &request_context.headers) + .await + .map_err(ApiError::from)?; + match store.get_object_info(&bucket, &key, ¤t_opts).await { + Ok(existing_obj_info) => validate_existing_object_lock_for_write(&existing_obj_info)?, + Err(err) => { + if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { + return Err(ApiError::from(err).into()); + } + } + } + + let actual_size = size; + let mut hash_values = PutObjectLegacyHashValues { + md5hex: if let Some(base64_md5) = content_md5 { + let md5 = base64_simd::STANDARD + .decode_to_vec(base64_md5.as_bytes()) + .map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?; + Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower)) + } else { + None + }, + sha256hex: get_content_sha256_with_query(&request_context.headers, request_context.uri_query.as_deref()), + }; + + let reader_stage_start = std::time::Instant::now(); + let stage = if can_use_small_put_eager + && should_use_small_put_eager_path(size, eager_max_bytes, body_plan.should_compress(), encryption_enabled_for_put) + { + small_object_eager_stage = true; + debug!( + "Plain PUT is using the eager small-object path (bucket={}, key={}, size={}, eager_max={})", + bucket, key, size, eager_max_bytes + ); + build_small_put_eager_hash_stage( + body, + size, + bytes_pool.clone(), + hash_values, + &request_context.headers, + request_context.trailing_headers.clone(), + ) + .await? + } else if body_plan.should_compress() { + transform_stage.mark_compression(); + let algorithm = CompressionAlgorithm::default(); + insert_str(&mut metadata, SUFFIX_COMPRESSION, algorithm.to_string()); + insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); + + let ingress_source = build_put_object_ingress_source(body, body_plan); + let stage = build_put_object_plain_hash_stage( + ingress_source, + std::mem::take(&mut hash_values), + PutObjectLegacyHashStagePlan { + size, + actual_size: size, + apply_s3_checksum: true, + ignore_s3_checksum_value: false, + }, + &request_context.headers, + request_context.trailing_headers.clone(), + ) + .map_err(ApiError::from)?; + + if stage.ingress_kind == PutObjectIngressKind::ReducedCopyCandidate { + plain_reduced_copy_stage = true; + } + opts.want_checksum = stage.want_checksum; + insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, algorithm.to_string()); + insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string()); + + let reader: Box = Box::new(CompressReader::new(stage.reader, algorithm)); + size = HashReader::SIZE_PRESERVE_LAYER; + hash_values.clear_for_transformed_body(); + build_put_object_legacy_hash_stage( + reader, + hash_values, + PutObjectLegacyHashStagePlan { + size, + actual_size, + apply_s3_checksum: size >= 0, + ignore_s3_checksum_value: false, + }, + &request_context.headers, + request_context.trailing_headers.clone(), + ) + .map_err(ApiError::from)? + } else { + let ingress_source = build_put_object_ingress_source(body, body_plan); + let stage = build_put_object_plain_hash_stage( + ingress_source, + hash_values, + PutObjectLegacyHashStagePlan { + size, + actual_size, + apply_s3_checksum: size >= 0, + ignore_s3_checksum_value: false, + }, + &request_context.headers, + request_context.trailing_headers.clone(), + ) + .map_err(ApiError::from)?; + + if stage.ingress_kind == PutObjectIngressKind::ReducedCopyCandidate { + plain_reduced_copy_stage = true; + debug!( + "Plain PUT is using the reduced-copy Reader + BlockReadable hash path (bucket={}, key={})", + bucket, key + ); + } + + stage + }; + log_put_flow_phase( + &bucket, + &key, + "build_hash_stage", + reader_stage_start.elapsed(), + actual_size, + small_object_eager_stage, + plain_reduced_copy_stage, + transform_stage.compression_applied(), + false, + ); + let mut reader = stage.reader; + if stage.want_checksum.is_some() { + opts.want_checksum = stage.want_checksum; + } + + let encryption_request = EncryptionRequest { + bucket: &bucket, + key: &key, + server_side_encryption: effective_sse.clone(), + ssekms_key_id: effective_kms_key_id.clone(), + sse_customer_algorithm: sse_customer_algorithm.clone(), + sse_customer_key, + sse_customer_key_md5: sse_customer_key_md5.clone(), + content_size: actual_size, + part_number: None, + part_key: None, + part_nonce: None, + }; + + if let Some(material) = sse_encryption(encryption_request).await? { + transform_stage.mark_encryption(); + effective_sse = Some(material.server_side_encryption.clone()); + effective_kms_key_id = material.kms_key_id.clone(); + + let encrypted_reader = material.wrap_reader(reader); + reader = HashReader::new(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) + .map_err(ApiError::from)?; + + let encryption_metadata = material.metadata; + metadata.extend(encryption_metadata.clone()); + opts.user_defined.extend(encryption_metadata); + } + + let mut reader = ChunkNativePutData::new(reader); + + let mt2 = metadata.clone(); + opts.user_defined.extend(metadata); + + let repoptions = + get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts.clone()); + + let dsc = must_replicate(&bucket, &key, repoptions).await; + + if dsc.replicate_any() { + insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); + insert_str( + &mut opts.user_defined, + SUFFIX_REPLICATION_STATUS, + dsc.pending_status().unwrap_or_default(), + ); + } + + let store_put_start = std::time::Instant::now(); + let obj_info = store + .put_object(&bucket, &key, &mut reader, &opts) + .await + .map_err(ApiError::from)?; + log_put_flow_phase( + &bucket, + &key, + "store_put_object", + store_put_start.elapsed(), + actual_size, + small_object_eager_stage, + plain_reduced_copy_stage, + transform_stage.compression_applied(), + transform_stage.encryption_applied(), + ); + + maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; + + rustfs_ecstore::data_usage::increment_bucket_usage_memory(&bucket, obj_info.size as u64).await; + + let raw_version = obj_info.version_id.map(|v| v.to_string()); + + Self::spawn_cache_invalidation(bucket.clone(), key.clone(), raw_version.clone()); + + let put_version = if bucket_prefix_versioning_enabled(&bucket, &key).await { + raw_version.clone() + } else { + None + }; + + let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)); + + let repoptions = + get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts); + + let dsc = must_replicate(&bucket, &key, repoptions).await; + let expiration = resolve_put_object_expiration(&bucket, &obj_info).await; + + if dsc.replicate_any() { + schedule_replication(obj_info.clone(), store.clone(), dsc, ReplicationType::Object).await; + } + + let mut checksums = PutObjectChecksums { + crc32: input.checksum_crc32, + crc32c: input.checksum_crc32c, + sha1: input.checksum_sha1, + sha256: input.checksum_sha256, + crc64nvme: input.checksum_crc64nvme, + }; + apply_trailing_checksums( + input.checksum_algorithm.as_ref().map(|a| a.as_str()), + &request_context.trailing_headers, + &mut checksums, + ); + checksums.merge_from_map(&reader.content_crc()); + if let Some(checksum_bytes) = resolved_checksum_bytes(&checksums) + && obj_info + .checksum + .as_ref() + .is_none_or(|stored| rustfs_rio::read_checksums(stored.as_ref(), 0).0.is_empty()) + { + let checksum_update_opts = ObjectOptions { + version_id: raw_version.clone(), + resolved_checksum: Some(checksum_bytes), + ..Default::default() + }; + let _ = store + .put_object_metadata(&bucket, &key, &checksum_update_opts) + .await + .map_err(ApiError::from)?; + } + + let output = PutObjectOutput { + e_tag, + server_side_encryption: effective_sse, + sse_customer_algorithm: sse_customer_algorithm.clone(), + sse_customer_key_md5: sse_customer_key_md5.clone(), + ssekms_key_id: effective_kms_key_id, + expiration, + checksum_crc32: checksums.crc32, + checksum_crc32c: checksums.crc32c, + checksum_sha1: checksums.sha1, + checksum_sha256: checksums.sha256, + checksum_crc64nvme: checksums.crc64nvme, + version_id: put_version, + ..Default::default() + }; + + let manager = get_capacity_manager(); + manager.record_write_operation().await; + + { + let duration_ms = start_time.elapsed().as_millis() as f64; + let fast_path_selected = plain_reduced_copy_stage || small_object_eager_stage; + rustfs_io_metrics::record_put_object(duration_ms, size, fast_path_selected); + let io_path = if fast_path_selected { + rustfs_io_metrics::IoPath::Fast + } else { + rustfs_io_metrics::IoPath::Legacy + }; + rustfs_io_metrics::record_io_path_selected("put", io_path); + rustfs_io_metrics::record_put_path_selected(actual_size, io_path); + let effective_copy_mode = transform_stage.effective_copy_mode(); + rustfs_io_metrics::record_io_copy_mode("put", effective_copy_mode, actual_size.max(0) as usize); + rustfs_io_metrics::record_put_copy_mode(actual_size, effective_copy_mode); + if let Some(transform_kind) = transform_stage.metric_kind() { + rustfs_io_metrics::record_put_transform_selected(transform_kind, io_path, actual_size.max(0) as usize); + } + } + + Ok(PutObjectFlowResult { + output, + helper_object: obj_info, + helper_version_id: raw_version, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use futures::{StreamExt, stream}; + use rustfs_io_core::BytesPool; + use serial_test::serial; + use std::sync::Arc; + use tokio::time::{Duration, timeout}; + + #[test] + fn small_put_eager_path_only_targets_plain_small_objects() { + assert!(should_use_small_put_eager_path( + 64 * 1024, + DEFAULT_SMALL_PUT_EAGER_MAX_BYTES, + false, + false + )); + assert!(should_use_small_put_eager_path( + DEFAULT_SMALL_PUT_EAGER_MAX_BYTES, + DEFAULT_SMALL_PUT_EAGER_MAX_BYTES, + false, + false + )); + assert!(!should_use_small_put_eager_path( + DEFAULT_SMALL_PUT_EAGER_MAX_BYTES + 1, + DEFAULT_SMALL_PUT_EAGER_MAX_BYTES, + false, + false + )); + assert!(!should_use_small_put_eager_path( + 64 * 1024, + DEFAULT_SMALL_PUT_EAGER_MAX_BYTES, + true, + false + )); + assert!(!should_use_small_put_eager_path( + 64 * 1024, + DEFAULT_SMALL_PUT_EAGER_MAX_BYTES, + false, + true + )); + assert!(!should_use_small_put_eager_path(0, DEFAULT_SMALL_PUT_EAGER_MAX_BYTES, false, false)); + } + + #[test] + fn clamp_small_put_eager_max_bytes_caps_inline_budget() { + assert_eq!( + clamp_small_put_eager_max_bytes(Some(rustfs_object_io::put::PUT_REDUCED_COPY_MIN_SIZE_BYTES as usize * 2)), + DEFAULT_SMALL_PUT_EAGER_MAX_BYTES + ); + assert_eq!(clamp_small_put_eager_max_bytes(Some(128 * 1024)), 128 * 1024); + assert_eq!(clamp_small_put_eager_max_bytes(None), DEFAULT_SMALL_PUT_EAGER_MAX_BYTES); + } + + #[test] + #[serial] + fn resolved_small_put_eager_max_bytes_honors_disable_env() { + temp_env::with_var(ENV_RUSTFS_PUT_FORCE_DISABLE_SMALL_EAGER, Some("true"), || { + assert_eq!(resolved_small_put_eager_max_bytes(256 * 1024), 0); + }); + } + + #[test] + #[serial] + fn resolved_small_put_eager_max_bytes_narrows_default_budget() { + temp_env::with_var(ENV_RUSTFS_PUT_SMALL_EAGER_MAX_BYTES, Some("4096"), || { + assert_eq!(resolved_small_put_eager_max_bytes(256 * 1024), 4096); + }); + } + + #[test] + #[serial] + fn resolved_small_put_eager_max_bytes_ignores_invalid_override() { + temp_env::with_var(ENV_RUSTFS_PUT_SMALL_EAGER_MAX_BYTES, Some("invalid"), || { + assert_eq!(resolved_small_put_eager_max_bytes(256 * 1024), 256 * 1024); + }); + } + + #[tokio::test] + async fn read_small_put_body_eager_requires_exact_content_length() { + let body = stream::iter(vec![ + Ok::(Bytes::from_static(b"abc")), + Ok::(Bytes::from_static(b"def")), + ]); + + let pool = Arc::new(BytesPool::new_tiered()); + let data = read_small_put_body_eager(body, 6, pool) + .await + .expect("eager read should succeed"); + assert_eq!(data.as_ref(), b"abcdef"); + } + + #[tokio::test] + async fn read_small_put_body_eager_rejects_length_mismatch() { + let body = stream::iter(vec![Ok::(Bytes::from_static(b"abc"))]); + let pool = Arc::new(BytesPool::new_tiered()); + + let err = read_small_put_body_eager(body, 4, pool) + .await + .expect_err("short eager read should fail"); + assert_eq!(err.code(), &S3ErrorCode::IncompleteBody); + } + + #[tokio::test] + async fn read_small_put_body_eager_rejects_overlong_body() { + let body = stream::iter(vec![ + Ok::(Bytes::from_static(b"abc")), + Ok::(Bytes::from_static(b"def")), + ]); + let pool = Arc::new(BytesPool::new_tiered()); + + let err = read_small_put_body_eager(body, 5, pool) + .await + .expect_err("overlong eager read should fail"); + assert_eq!(err.code(), &S3ErrorCode::IncompleteBody); + } + + #[tokio::test] + async fn read_small_put_body_eager_returns_buffer_to_pool_after_drop() { + let body = stream::iter(vec![Ok::(Bytes::from_static(b"abc"))]); + let pool = Arc::new(BytesPool::new_tiered()); + + let data = read_small_put_body_eager(body, 3, pool.clone()) + .await + .expect("pooled eager read should succeed"); + assert_eq!(pool.available_buffers(), 0); + + drop(data); + assert_eq!(pool.available_buffers(), 1); + } + + #[tokio::test] + async fn read_small_put_body_eager_returns_after_expected_bytes_without_waiting_for_eof() { + let body = stream::once(async { Ok::(Bytes::from_static(b"abc")) }).chain(stream::pending()); + let pool = Arc::new(BytesPool::new_tiered()); + + let data = timeout(Duration::from_millis(50), read_small_put_body_eager(body, 3, pool)) + .await + .expect("eager read should not wait for stream termination") + .expect("eager read should succeed once content-length bytes are read"); + + assert_eq!(data.as_ref(), b"abc"); + } +} diff --git a/rustfs/src/app/object_usecase/types.rs b/rustfs/src/app/object_usecase/types.rs new file mode 100644 index 000000000..63444d8d9 --- /dev/null +++ b/rustfs/src/app/object_usecase/types.rs @@ -0,0 +1,52 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +#[derive(Clone)] +pub(super) struct GetObjectRequestContext { + pub(super) bucket: String, + pub(super) key: String, + pub(super) cache_key: String, + pub(super) version_id_for_event: String, + pub(super) part_number: Option, + pub(super) rs: Option, + pub(super) opts: ObjectOptions, + pub(super) headers: HeaderMap, + pub(super) method: hyper::Method, + pub(super) sse_customer_key: Option, + pub(super) sse_customer_key_md5: Option, +} + +pub(super) type PutObjectChecksums = rustfs_object_io::put::PutObjectChecksums; + +#[derive(Clone)] +pub(super) struct PutObjectRequestContext { + pub(super) headers: HeaderMap, + pub(super) trailing_headers: Option, + pub(super) uri_query: Option, + pub(super) is_post_object: bool, + pub(super) method: hyper::Method, + pub(super) uri: hyper::Uri, + pub(super) extensions: http::Extensions, + pub(super) credentials: Option, + pub(super) region: Option, + pub(super) service: Option, +} + +pub(super) struct PutObjectFlowResult { + pub(super) output: PutObjectOutput, + pub(super) helper_object: ObjectInfo, + pub(super) helper_version_id: Option, +} diff --git a/rustfs/src/app/object_usecase/zero_copy_tests.rs b/rustfs/src/app/object_usecase/zero_copy_tests.rs new file mode 100644 index 000000000..59c3dba1c --- /dev/null +++ b/rustfs/src/app/object_usecase/zero_copy_tests.rs @@ -0,0 +1,1222 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use futures::StreamExt; +use http::{Extensions, HeaderMap, Method, Uri}; +use rustfs_ecstore::store_api::GetObjectChunkPath; +use rustfs_ecstore::{ + bucket::metadata_sys, + disk::endpoint::Endpoint, + endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, + store::ECStore, + store_api::{ + BucketOperations, ChunkNativePutData, CompletePart, MakeBucketOptions, MultipartOperations, ObjectIO, ObjectOptions, + }, +}; +use rustfs_object_io::get::{GetObjectBodySource, GetObjectReadSetup}; +use serial_test::serial; +use std::{ + path::PathBuf, + sync::{Arc, Once, OnceLock}, +}; +use tokio::fs; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +static DIRECT_CHUNK_TEST_ENV: OnceLock<(Vec, Arc)> = OnceLock::new(); +static DIRECT_CHUNK_MULTI_DISK_TEST_ENV: OnceLock<(Vec, Arc)> = OnceLock::new(); +static DIRECT_CHUNK_TEST_INIT: Once = Once::new(); + +fn init_direct_chunk_test_tracing() { + DIRECT_CHUNK_TEST_INIT.call_once(|| {}); +} + +async fn setup_direct_chunk_test_env() -> (Vec, Arc) { + init_direct_chunk_test_tracing(); + + if let Some((paths, store)) = DIRECT_CHUNK_TEST_ENV.get() { + return (paths.clone(), store.clone()); + } + + let test_base_dir = format!("/tmp/rustfs_app_chunk_direct_test_{}", Uuid::new_v4()); + let temp_dir = PathBuf::from(&test_base_dir); + if temp_dir.exists() { + fs::remove_dir_all(&temp_dir).await.ok(); + } + fs::create_dir_all(&temp_dir).await.unwrap(); + + let disk_path = temp_dir.join("disk1"); + fs::create_dir_all(&disk_path).await.unwrap(); + + let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap(); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); + + let pool_endpoints = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 1, + endpoints: Endpoints::from(vec![endpoint]), + cmd_line: "test".to_string(), + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }; + + let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); + + rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap(); + + let server_addr: std::net::SocketAddr = "127.0.0.1:9013".parse().unwrap(); + let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) + .await + .unwrap(); + + let buckets_list = ecstore + .list_bucket(&rustfs_ecstore::store_api::BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + .unwrap(); + let buckets = buckets_list.into_iter().map(|v| v.name).collect(); + metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await; + + let _ = DIRECT_CHUNK_TEST_ENV.set((vec![disk_path], ecstore.clone())); + + (DIRECT_CHUNK_TEST_ENV.get().unwrap().0.clone(), ecstore) +} + +async fn setup_direct_chunk_multi_disk_test_env() -> (Vec, Arc) { + init_direct_chunk_test_tracing(); + + if let Some((paths, store)) = DIRECT_CHUNK_MULTI_DISK_TEST_ENV.get() { + return (paths.clone(), store.clone()); + } + + let test_base_dir = format!("/tmp/rustfs_app_chunk_direct_multi_test_{}", Uuid::new_v4()); + let temp_dir = PathBuf::from(&test_base_dir); + if temp_dir.exists() { + fs::remove_dir_all(&temp_dir).await.ok(); + } + fs::create_dir_all(&temp_dir).await.unwrap(); + + let disk_paths = vec![ + temp_dir.join("disk1"), + temp_dir.join("disk2"), + temp_dir.join("disk3"), + temp_dir.join("disk4"), + ]; + for disk_path in &disk_paths { + fs::create_dir_all(disk_path).await.unwrap(); + } + + let mut endpoints = Vec::new(); + for (i, disk_path) in disk_paths.iter().enumerate() { + let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap(); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(i); + endpoints.push(endpoint); + } + + let pool_endpoints = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 4, + endpoints: Endpoints::from(endpoints), + cmd_line: "test".to_string(), + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }; + + let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); + + rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap(); + + let server_addr: std::net::SocketAddr = "127.0.0.1:9014".parse().unwrap(); + let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) + .await + .unwrap(); + + let buckets_list = ecstore + .list_bucket(&rustfs_ecstore::store_api::BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + .unwrap(); + let buckets = buckets_list.into_iter().map(|v| v.name).collect(); + metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await; + + let _ = DIRECT_CHUNK_MULTI_DISK_TEST_ENV.set((disk_paths.clone(), ecstore.clone())); + + (DIRECT_CHUNK_MULTI_DISK_TEST_ENV.get().unwrap().0.clone(), ecstore) +} + +async fn create_direct_chunk_test_bucket(ecstore: &Arc, bucket_name: &str) { + (**ecstore) + .make_bucket( + bucket_name, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .unwrap(); +} + +async fn create_direct_chunk_test_multipart_object( + ecstore: &Arc, + bucket: &str, + key: &str, + parts: Vec>, +) -> Vec> { + let upload = ecstore + .new_multipart_upload(bucket, key, &ObjectOptions::default()) + .await + .unwrap(); + + let mut completed_parts = Vec::new(); + for (idx, part) in parts.iter().enumerate() { + let mut reader = ChunkNativePutData::from_vec(part.clone()); + let part_info = ecstore + .put_object_part(bucket, key, &upload.upload_id, idx + 1, &mut reader, &ObjectOptions::default()) + .await + .unwrap(); + completed_parts.push(CompletePart { + part_num: idx + 1, + etag: part_info.etag, + ..Default::default() + }); + } + + ecstore + .clone() + .complete_multipart_upload(bucket, key, &upload.upload_id, completed_parts, &ObjectOptions::default()) + .await + .unwrap(); + + parts +} + +fn find_part_file(root: &std::path::Path, part_name: &str) -> Option { + let entries = std::fs::read_dir(root).ok()?; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + if let Some(found) = find_part_file(&path, part_name) { + return Some(found); + } + continue; + } + + if path.file_name().and_then(|name| name.to_str()) == Some(part_name) { + return Some(path); + } + } + + None +} + +fn find_part_files(root: &std::path::Path, part_name: &str, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(root) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + find_part_files(&path, part_name, out); + continue; + } + + if path.file_name().and_then(|name| name.to_str()) == Some(part_name) { + out.push(path); + } + } +} + +async fn remove_part_files(root: &std::path::Path, part_name: &str) -> Vec<(PathBuf, Vec)> { + let mut paths = Vec::new(); + find_part_files(root, part_name, &mut paths); + + let mut removed = Vec::with_capacity(paths.len()); + for path in paths { + let content = fs::read(&path).await.expect("read part file before removal"); + fs::remove_file(&path).await.expect("remove part file"); + removed.push((path, content)); + } + + removed +} + +async fn restore_part_files(files: Vec<(PathBuf, Vec)>) { + for (path, content) in files { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).await.expect("ensure parent for part restore"); + } + fs::write(&path, content).await.expect("restore part file"); + } +} + +async fn select_reconstructed_chunk_read( + disk_paths: &[PathBuf], + bucket: &str, + key: &str, + missing_part_name: &str, + ecstore: &Arc, + manager: &ConcurrencyManager, + request_context: &GetObjectRequestContext, +) -> GetObjectReadSetup { + for disk_path in disk_paths { + let object_root = disk_path.join(bucket).join(key); + let removed_parts = remove_part_files(&object_root, missing_part_name).await; + if removed_parts.is_empty() { + continue; + } + + let candidate = get_object_zero_copy::prepare_get_object_chunk_read( + request_context, + ecstore, + manager, + &request_context.bucket, + &request_context.key, + request_context.rs.clone(), + request_context.part_number, + &request_context.opts, + std::time::Instant::now(), + ) + .await + .unwrap() + .expect("expected chunk fast path"); + + let is_reconstructed = matches!( + &candidate.body_source, + GetObjectBodySource::Chunk { + path: GetObjectChunkPath::Direct, + copy_mode: rustfs_io_metrics::CopyMode::Reconstructed, + .. + } + ); + if is_reconstructed { + return candidate; + } + + restore_part_files(removed_parts).await; + } + + panic!("expected reconstructed chunk path after removing one disk shard copy of {missing_part_name}"); +} + +fn build_request(input: T, method: Method) -> S3Request { + S3Request { + input, + method, + uri: Uri::from_static("/"), + headers: HeaderMap::new(), + extensions: Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + } +} + +#[tokio::test] +async fn execute_get_object_rejects_zero_part_number() { + let input = GetObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .part_number(Some(0)) + .build() + .unwrap(); + + let req = build_request(input, Method::GET); + let usecase = DefaultObjectUsecase::without_context(); + + let err = usecase.execute_get_object(req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn prepare_get_object_chunk_read_marks_direct_path_for_single_disk_store() { + let (_disk_paths, ecstore) = setup_direct_chunk_test_env().await; + let bucket = format!("direct-chunk-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/object.bin"; + let payload = vec![1u8; 128 * 1024]; + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + + let mut reader = ChunkNativePutData::from_vec(payload.clone()); + ecstore + .put_object(&bucket, key, &mut reader, &ObjectOptions::default()) + .await + .unwrap(); + + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + + let read_setup = get_object_zero_copy::prepare_get_object_chunk_read( + &request_context, + &ecstore, + manager, + &request_context.bucket, + &request_context.key, + request_context.rs.clone(), + request_context.part_number, + &request_context.opts, + std::time::Instant::now(), + ) + .await + .unwrap() + .expect("expected chunk fast path"); + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + assert_eq!(copy_mode, rustfs_io_metrics::CopyMode::TrueZeroCopy); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some(payload.len() as i64)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected, payload); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn prepare_get_object_chunk_read_falls_back_to_legacy_when_chunk_bridge_fails() { + let (disk_paths, ecstore) = setup_direct_chunk_test_env().await; + let bucket = format!("direct-chunk-fallback-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/fallback.bin"; + let payload = vec![3u8; 128 * 1024]; + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + + let mut reader = ChunkNativePutData::from_vec(payload); + ecstore + .put_object(&bucket, key, &mut reader, &ObjectOptions::default()) + .await + .unwrap(); + + let object_root = disk_paths[0].join(&bucket).join("test").join("fallback.bin"); + let missing_part = find_part_file(&object_root, "part.1").expect("part file on disk"); + fs::remove_file(&missing_part).await.expect("remove single-disk part file"); + + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + + let read_setup = get_object_zero_copy::prepare_get_object_chunk_read( + &request_context, + &ecstore, + manager, + &request_context.bucket, + &request_context.key, + request_context.rs.clone(), + request_context.part_number, + &request_context.opts, + std::time::Instant::now(), + ) + .await + .unwrap(); + + assert!(read_setup.is_none(), "chunk bridge failure should fall back to legacy reader"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_range_marks_direct_path_for_single_disk_store() { + let (_disk_paths, ecstore) = setup_direct_chunk_test_env().await; + let bucket = format!("direct-range-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/range.bin"; + let payload = vec![2u8; 128 * 1024]; + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + + let mut reader = ChunkNativePutData::from_vec(payload.clone()); + ecstore + .put_object(&bucket, key, &mut reader, &ObjectOptions::default()) + .await + .unwrap(); + + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .range(Some(Range::Int { + first: 0, + last: Some(63 * 1024), + })) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + + let read_setup = get_object_zero_copy::prepare_get_object_chunk_read( + &request_context, + &ecstore, + manager, + &request_context.bucket, + &request_context.key, + request_context.rs.clone(), + request_context.part_number, + &request_context.opts, + std::time::Instant::now(), + ) + .await + .unwrap() + .expect("expected chunk fast path"); + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path") + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some(63 * 1024 + 1)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected, payload[..(63 * 1024 + 1)].to_vec()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_range_marks_direct_path_for_multi_disk_store_without_rebuild() { + let (_disk_paths, ecstore) = setup_direct_chunk_multi_disk_test_env().await; + let bucket = format!("direct-multi-range-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/multi-range.bin"; + let payload_len = 3 * 1024 * 1024 + 137; + let payload: Vec = (0..payload_len).map(|idx| (idx % 251) as u8).collect(); + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + + let mut reader = ChunkNativePutData::from_vec(payload.clone()); + let put_info = ecstore + .put_object(&bucket, key, &mut reader, &ObjectOptions::default()) + .await + .unwrap(); + assert!( + put_info.data_blocks > 1, + "expected multi-data-shard object, got {}+{}", + put_info.data_blocks, + put_info.parity_blocks + ); + + let range_start = 123_457_u64; + let range_end = 2 * 1024 * 1024 + 33_333_u64; + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .range(Some(Range::Int { + first: range_start, + last: Some(range_end), + })) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + + let read_setup = get_object_zero_copy::prepare_get_object_chunk_read( + &request_context, + &ecstore, + manager, + &request_context.bucket, + &request_context.key, + request_context.rs.clone(), + request_context.part_number, + &request_context.opts, + std::time::Instant::now(), + ) + .await + .unwrap() + .expect("expected chunk fast path"); + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + #[cfg(unix)] + assert_eq!( + copy_mode, + rustfs_io_metrics::CopyMode::TrueZeroCopy, + "multi-data-shard direct path should preserve shard mmap slices without assembly copies" + ); + #[cfg(not(unix))] + assert_eq!( + copy_mode, + rustfs_io_metrics::CopyMode::SharedBytes, + "multi-data-shard direct path should avoid assembly copies even without mmap" + ); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some((range_end - range_start + 1) as i64)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected, payload[range_start as usize..=range_end as usize].to_vec()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_range_marks_reconstructed_path_for_multi_disk_store_with_missing_shard() { + let (disk_paths, ecstore) = setup_direct_chunk_multi_disk_test_env().await; + let bucket = format!("direct-multi-reconstructed-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/multi-reconstructed.bin"; + let payload_len = 3 * 1024 * 1024 + 137; + let payload: Vec = (0..payload_len).map(|idx| (idx % 251) as u8).collect(); + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + + let mut reader = ChunkNativePutData::from_vec(payload.clone()); + let put_info = ecstore + .put_object(&bucket, key, &mut reader, &ObjectOptions::default()) + .await + .unwrap(); + assert!(put_info.data_blocks > 1, "expected multi-data-shard object"); + + let object_root = disk_paths[0].join(&bucket).join("test").join("multi-reconstructed.bin"); + let missing_part = find_part_file(&object_root, "part.1").expect("part file on first disk"); + fs::remove_file(&missing_part).await.expect("remove first shard part"); + + let range_start = 123_457_u64; + let range_end = 2 * 1024 * 1024 + 33_333_u64; + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .range(Some(Range::Int { + first: range_start, + last: Some(range_end), + })) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + + let read_setup = get_object_zero_copy::prepare_get_object_chunk_read( + &request_context, + &ecstore, + manager, + &request_context.bucket, + &request_context.key, + request_context.rs.clone(), + request_context.part_number, + &request_context.opts, + std::time::Instant::now(), + ) + .await + .unwrap() + .expect("expected chunk fast path"); + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + assert_eq!( + copy_mode, + rustfs_io_metrics::CopyMode::Reconstructed, + "missing data shard should trigger reconstructed chunk path" + ); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some((range_end - range_start + 1) as i64)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected, payload[range_start as usize..=range_end as usize].to_vec()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_part_number_marks_direct_path_for_single_disk_store() { + let (_disk_paths, ecstore) = setup_direct_chunk_test_env().await; + let bucket = format!("direct-part-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/multipart.bin"; + let part_one = vec![11u8; 5 * 1024 * 1024]; + let part_two = vec![22u8; 5 * 1024 * 1024]; + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + let parts = create_direct_chunk_test_multipart_object(&ecstore, &bucket, key, vec![part_one.clone(), part_two.clone()]).await; + + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .part_number(Some(1)) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + + let read_setup = get_object_zero_copy::prepare_get_object_chunk_read( + &request_context, + &ecstore, + manager, + &request_context.bucket, + &request_context.key, + request_context.rs.clone(), + request_context.part_number, + &request_context.opts, + std::time::Instant::now(), + ) + .await + .unwrap() + .expect("expected chunk fast path"); + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + assert_eq!(copy_mode, rustfs_io_metrics::CopyMode::TrueZeroCopy); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some(parts[0].len() as i64)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected, parts[0]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_whole_multipart_marks_direct_path_for_single_disk_store() { + let (_disk_paths, ecstore) = setup_direct_chunk_test_env().await; + let bucket = format!("direct-whole-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/multipart-whole.bin"; + let part_one = vec![11u8; 5 * 1024 * 1024]; + let part_two = vec![22u8; 5 * 1024 * 1024 + 321]; + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + let parts = create_direct_chunk_test_multipart_object(&ecstore, &bucket, key, vec![part_one.clone(), part_two.clone()]).await; + + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + + let read_setup = get_object_zero_copy::prepare_get_object_chunk_read( + &request_context, + &ecstore, + manager, + &request_context.bucket, + &request_context.key, + request_context.rs.clone(), + request_context.part_number, + &request_context.opts, + std::time::Instant::now(), + ) + .await + .unwrap() + .expect("expected chunk fast path"); + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + assert_eq!(copy_mode, rustfs_io_metrics::CopyMode::TrueZeroCopy); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some(parts.iter().map(|part| part.len() as i64).sum())); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + + let mut expected = Vec::with_capacity(parts.iter().map(Vec::len).sum()); + for part in &parts { + expected.extend_from_slice(part); + } + assert_eq!(collected, expected); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_part_number_marks_reconstructed_path_for_multi_disk_store_with_missing_shard() { + let (disk_paths, ecstore) = setup_direct_chunk_multi_disk_test_env().await; + let bucket = format!("direct-part-reconstructed-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/multipart-reconstructed.bin"; + let part_one: Vec = (0..(5 * 1024 * 1024)).map(|idx| (idx % 251) as u8).collect(); + let part_two: Vec = (0..(5 * 1024 * 1024 + 137)).map(|idx| ((idx + 11) % 251) as u8).collect(); + let part_three: Vec = (0..(1024 * 1024 + 77)).map(|idx| ((idx + 29) % 251) as u8).collect(); + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + let parts = + create_direct_chunk_test_multipart_object(&ecstore, &bucket, key, vec![part_one.clone(), part_two.clone(), part_three]) + .await; + + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .part_number(Some(1)) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + let read_setup = + select_reconstructed_chunk_read(&disk_paths, &bucket, key, "part.1", &ecstore, manager, &request_context).await; + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + assert_eq!( + copy_mode, + rustfs_io_metrics::CopyMode::Reconstructed, + "missing data shard in multipart part should trigger reconstructed chunk path" + ); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some(parts[0].len() as i64)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected, parts[0]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_part_number_marks_reconstructed_path_for_second_multipart_part_with_missing_shard() { + let (disk_paths, ecstore) = setup_direct_chunk_multi_disk_test_env().await; + let bucket = format!("direct-part2-reconstructed-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/multipart-reconstructed-second-part.bin"; + let part_one: Vec = (0..(5 * 1024 * 1024)).map(|idx| (idx % 251) as u8).collect(); + let part_two: Vec = (0..(5 * 1024 * 1024 + 137)).map(|idx| ((idx + 11) % 251) as u8).collect(); + let part_three: Vec = (0..(1024 * 1024 + 77)).map(|idx| ((idx + 29) % 251) as u8).collect(); + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + let parts = + create_direct_chunk_test_multipart_object(&ecstore, &bucket, key, vec![part_one, part_two.clone(), part_three]).await; + + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .part_number(Some(2)) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + let read_setup = + select_reconstructed_chunk_read(&disk_paths, &bucket, key, "part.2", &ecstore, manager, &request_context).await; + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + assert_eq!( + copy_mode, + rustfs_io_metrics::CopyMode::Reconstructed, + "missing data shard in multipart part 2 should trigger reconstructed chunk path" + ); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some(parts[1].len() as i64)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected, parts[1]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_part_number_marks_reconstructed_path_for_final_multipart_part_with_missing_shard() { + let (disk_paths, ecstore) = setup_direct_chunk_multi_disk_test_env().await; + let bucket = format!("direct-part3-reconstructed-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/multipart-reconstructed-final-part.bin"; + let part_one: Vec = (0..(5 * 1024 * 1024)).map(|idx| (idx % 251) as u8).collect(); + let part_two: Vec = (0..(5 * 1024 * 1024 + 137)).map(|idx| ((idx + 11) % 251) as u8).collect(); + let part_three: Vec = (0..(1024 * 1024 + 77)).map(|idx| ((idx + 29) % 251) as u8).collect(); + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + let parts = + create_direct_chunk_test_multipart_object(&ecstore, &bucket, key, vec![part_one, part_two, part_three.clone()]).await; + + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .part_number(Some(3)) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + let read_setup = + select_reconstructed_chunk_read(&disk_paths, &bucket, key, "part.3", &ecstore, manager, &request_context).await; + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + assert_eq!( + copy_mode, + rustfs_io_metrics::CopyMode::Reconstructed, + "missing data shard in the final multipart part should trigger reconstructed chunk path" + ); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some(parts[2].len() as i64)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected, part_three); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_whole_multipart_marks_reconstructed_path_for_missing_middle_part_shard() { + let (disk_paths, ecstore) = setup_direct_chunk_multi_disk_test_env().await; + let bucket = format!("direct-whole-multipart-reconstructed-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/multipart-whole-reconstructed.bin"; + let part_one: Vec = (0..(5 * 1024 * 1024)).map(|idx| (idx % 251) as u8).collect(); + let part_two: Vec = (0..(5 * 1024 * 1024 + 257)).map(|idx| ((idx + 17) % 251) as u8).collect(); + let part_three: Vec = (0..(1024 * 1024 + 211)).map(|idx| ((idx + 33) % 251) as u8).collect(); + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + let parts = create_direct_chunk_test_multipart_object(&ecstore, &bucket, key, vec![part_one, part_two, part_three]).await; + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + let read_setup = + select_reconstructed_chunk_read(&disk_paths, &bucket, key, "part.2", &ecstore, manager, &request_context).await; + + let mut direct_stream = match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, stream } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + assert_eq!( + copy_mode, + rustfs_io_metrics::CopyMode::Reconstructed, + "whole multipart GET should keep the reconstructed chunk fast path when a middle part is missing a shard" + ); + stream + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + }; + + let mut expected = Vec::with_capacity(parts.iter().map(Vec::len).sum()); + for part in &parts { + expected.extend_from_slice(part); + } + + let mut direct_collected = Vec::new(); + while let Some(chunk) = direct_stream.next().await { + direct_collected.extend_from_slice(chunk.unwrap().as_bytes().as_ref()); + } + assert_eq!( + direct_collected.len(), + expected.len(), + "prepared chunk stream reconstructed whole multipart length mismatch" + ); + let direct_first_diff = direct_collected + .iter() + .zip(expected.iter()) + .position(|(left, right)| left != right); + assert_eq!( + direct_first_diff, None, + "prepared chunk stream reconstructed whole multipart first diff at {:?}", + direct_first_diff + ); + assert_eq!( + direct_collected, expected, + "prepared chunk stream should cover the whole reconstructed multipart object" + ); + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some(expected.len() as i64)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected.len(), expected.len(), "whole multipart reconstructed GET length mismatch"); + let first_diff = collected.iter().zip(expected.iter()).position(|(left, right)| left != right); + assert_eq!(first_diff, None, "whole multipart reconstructed GET first diff at {:?}", first_diff); + assert_eq!(collected, expected); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_multipart_range_marks_reconstructed_path_for_missing_shard_part() { + let (disk_paths, ecstore) = setup_direct_chunk_multi_disk_test_env().await; + let bucket = format!("direct-multipart-range-reconstructed-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/multipart-range-reconstructed.bin"; + let part_one: Vec = (0..(5 * 1024 * 1024)).map(|idx| (idx % 251) as u8).collect(); + let part_two: Vec = (0..(5 * 1024 * 1024 + 257)).map(|idx| ((idx + 17) % 251) as u8).collect(); + let part_three: Vec = (0..(1024 * 1024 + 211)).map(|idx| ((idx + 33) % 251) as u8).collect(); + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + let parts = + create_direct_chunk_test_multipart_object(&ecstore, &bucket, key, vec![part_one.clone(), part_two.clone(), part_three]) + .await; + + let range_start = 1024_u64; + let range_end = 64 * 1024 + 2048; + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .range(Some(Range::Int { + first: range_start, + last: Some(range_end), + })) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + let read_setup = + select_reconstructed_chunk_read(&disk_paths, &bucket, key, "part.1", &ecstore, manager, &request_context).await; + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + assert_eq!( + copy_mode, + rustfs_io_metrics::CopyMode::Reconstructed, + "multipart partial range should stay on reconstructed fast path when the covered part has a missing shard" + ); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let mut expected = Vec::with_capacity(parts.iter().map(Vec::len).sum()); + for part in &parts { + expected.extend_from_slice(part); + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some((range_end - range_start + 1) as i64)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected, expected[range_start as usize..=range_end as usize].to_vec()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_cross_part_multipart_range_marks_reconstructed_path_for_missing_next_part_shard() { + let (disk_paths, ecstore) = setup_direct_chunk_multi_disk_test_env().await; + let bucket = format!("direct-multipart-cross-range-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/multipart-cross-range-reconstructed.bin"; + let part_one: Vec = (0..(5 * 1024 * 1024)).map(|idx| (idx % 251) as u8).collect(); + let part_two: Vec = (0..(5 * 1024 * 1024 + 257)).map(|idx| ((idx + 17) % 251) as u8).collect(); + let part_three: Vec = (0..(1024 * 1024 + 211)).map(|idx| ((idx + 33) % 251) as u8).collect(); + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + let parts = create_direct_chunk_test_multipart_object(&ecstore, &bucket, key, vec![part_one, part_two, part_three]).await; + + let part_one_len = parts[0].len() as u64; + let range_start = part_one_len - 32 * 1024; + let range_end = part_one_len + 96 * 1024; + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .range(Some(Range::Int { + first: range_start, + last: Some(range_end), + })) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + let read_setup = + select_reconstructed_chunk_read(&disk_paths, &bucket, key, "part.2", &ecstore, manager, &request_context).await; + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + assert_eq!( + copy_mode, + rustfs_io_metrics::CopyMode::Reconstructed, + "cross-part multipart range should keep the reconstructed chunk fast path when the next part has a missing shard" + ); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let mut expected = Vec::with_capacity(parts.iter().map(Vec::len).sum()); + for part in &parts { + expected.extend_from_slice(part); + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some((range_end - range_start + 1) as i64)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected, expected[range_start as usize..=range_end as usize].to_vec()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn execute_get_object_cross_part_multipart_range_marks_reconstructed_path_for_missing_final_part_shard() { + let (disk_paths, ecstore) = setup_direct_chunk_multi_disk_test_env().await; + let bucket = format!("direct-multipart-final-cross-range-{}", &Uuid::new_v4().simple().to_string()[..8]); + let key = "test/multipart-final-cross-range-reconstructed.bin"; + let part_one: Vec = (0..(5 * 1024 * 1024)).map(|idx| (idx % 251) as u8).collect(); + let part_two: Vec = (0..(5 * 1024 * 1024 + 257)).map(|idx| ((idx + 17) % 251) as u8).collect(); + let part_three: Vec = (0..(1024 * 1024 + 211)).map(|idx| ((idx + 33) % 251) as u8).collect(); + + create_direct_chunk_test_bucket(&ecstore, &bucket).await; + let parts = create_direct_chunk_test_multipart_object(&ecstore, &bucket, key, vec![part_one, part_two, part_three]).await; + + let part_two_end = (parts[0].len() + parts[1].len()) as u64; + let range_start = part_two_end - 32 * 1024; + let range_end = part_two_end + 96 * 1024; + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .range(Some(Range::Int { + first: range_start, + last: Some(range_end), + })) + .build() + .unwrap(); + let req = build_request(input, Method::GET); + + let request_context = prepare_get_object_request_context(&req).await.unwrap(); + let manager = get_concurrency_manager(); + let read_setup = + select_reconstructed_chunk_read(&disk_paths, &bucket, key, "part.3", &ecstore, manager, &request_context).await; + + match read_setup.body_source { + GetObjectBodySource::Chunk { path, copy_mode, .. } => { + assert!(matches!(path, GetObjectChunkPath::Direct), "expected direct chunk path"); + assert_eq!( + copy_mode, + rustfs_io_metrics::CopyMode::Reconstructed, + "cross-part multipart range into the final part should keep the reconstructed chunk fast path" + ); + } + GetObjectBodySource::Reader(_) => panic!("expected chunk body source"), + } + + let mut expected = Vec::with_capacity(parts.iter().map(Vec::len).sum()); + for part in &parts { + expected.extend_from_slice(part); + } + + let usecase = DefaultObjectUsecase::without_context(); + let response = usecase.execute_get_object(req).await.unwrap(); + assert_eq!(response.output.content_length, Some((range_end - range_start + 1) as i64)); + let mut body = response.output.body.expect("expected body"); + let mut collected = Vec::new(); + while let Some(chunk) = body.next().await { + collected.extend_from_slice(&chunk.unwrap()); + } + assert_eq!(collected, expected[range_start as usize..=range_end as usize].to_vec()); +} diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 14a2d8352..95b40dfb7 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -259,6 +259,9 @@ impl From for ApiError { fn from(err: std::io::Error) -> Self { // Check if the error is a ChecksumMismatch (BadDigest) if let Some(inner) = err.get_ref() { + if let Some(storage_error) = inner.downcast_ref::() { + return storage_error.clone().into(); + } if inner.downcast_ref::().is_some() { return ApiError { code: S3ErrorCode::BadDigest, @@ -552,4 +555,15 @@ mod tests { // This is expected because ApiError is not a typical Error implementation assert!(error.source().is_none()); } + + #[test] + fn test_api_error_from_io_error_unwraps_invalid_range_storage_error() { + let io_error = std::io::Error::from(StorageError::InvalidRangeSpec("range invalid".to_string())); + + let api_error: ApiError = io_error.into(); + + assert_eq!(api_error.code, S3ErrorCode::InvalidRange); + assert_eq!(api_error.message, ApiError::error_code_to_message(&S3ErrorCode::InvalidRange)); + assert!(api_error.source.is_some()); + } } diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 58a5b7af6..90b940e86 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -426,7 +426,7 @@ pub async fn start_http_server( } } }; - + #[allow(unused)] let socket_ref = SockRef::from(&socket); // ── POST-ACCEPT SOCKET SYSCALLS ── diff --git a/rustfs/src/storage/concurrency/object_cache.rs b/rustfs/src/storage/concurrency/object_cache.rs index b8f715e3e..676773f72 100644 --- a/rustfs/src/storage/concurrency/object_cache.rs +++ b/rustfs/src/storage/concurrency/object_cache.rs @@ -32,6 +32,7 @@ use hashbrown::HashMap; use moka::future::Cache; use rustfs_config::MI_B; +use rustfs_object_io::get::GetObjectCacheWriteback; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; @@ -1063,6 +1064,13 @@ pub struct CachedGetObject { pub replication_status: Option, /// User-defined metadata (x-amz-meta-*) pub user_metadata: std::collections::HashMap, + /// Additional checksum metadata persisted with cached GET responses + pub checksum_crc32: Option, + pub checksum_crc32c: Option, + pub checksum_sha1: Option, + pub checksum_sha256: Option, + pub checksum_crc64nvme: Option, + pub checksum_type: Option, /// When this object was cached (for internal use, automatically set) #[allow(dead_code)] cached_at: Option, @@ -1089,6 +1097,12 @@ impl Default for CachedGetObject { tag_count: None, replication_status: None, user_metadata: std::collections::HashMap::new(), + checksum_crc32: None, + checksum_crc32c: None, + checksum_sha1: None, + checksum_sha256: None, + checksum_crc64nvme: None, + checksum_type: None, cached_at: None, access_count: Arc::new(AtomicU64::new(0)), } @@ -1109,6 +1123,35 @@ impl CachedGetObject { } } + /// Consume a GET cache writeback payload into the cache-owned representation. + pub fn from_get_object_cache_writeback(writeback: GetObjectCacheWriteback) -> Self { + Self { + body: writeback.body, + content_length: writeback.content_length, + content_type: writeback.content_type, + e_tag: writeback.e_tag, + last_modified: writeback.last_modified, + expires: writeback.expires, + cache_control: writeback.cache_control, + content_disposition: writeback.content_disposition, + content_encoding: writeback.content_encoding, + content_language: writeback.content_language, + storage_class: writeback.storage_class, + version_id: writeback.version_id, + delete_marker: writeback.delete_marker, + user_metadata: writeback.user_metadata, + checksum_crc32: writeback.checksum_crc32, + checksum_crc32c: writeback.checksum_crc32c, + checksum_sha1: writeback.checksum_sha1, + checksum_sha256: writeback.checksum_sha256, + checksum_crc64nvme: writeback.checksum_crc64nvme, + checksum_type: writeback.checksum_type, + cached_at: Some(Instant::now()), + access_count: Arc::new(AtomicU64::new(0)), + ..Default::default() + } + } + /// Builder method to set content_type pub fn with_content_type(mut self, content_type: String) -> Self { self.content_type = Some(content_type); @@ -1881,6 +1924,57 @@ mod cached_object_tests { assert_eq!(obj.user_metadata.get("x-amz-meta-custom"), Some(&"value".to_string())); } + #[test] + fn test_cached_get_object_from_get_object_cache_writeback() { + let body = Arc::new(Bytes::from("test data")); + let obj = CachedGetObject::from_get_object_cache_writeback(GetObjectCacheWriteback { + body: Arc::clone(&body), + content_length: 9, + content_type: Some("text/plain".to_string()), + content_encoding: Some("gzip".to_string()), + cache_control: Some("max-age=3600".to_string()), + content_disposition: Some("attachment".to_string()), + content_language: Some("en-US".to_string()), + expires: Some("2024-12-31T23:59:59Z".to_string()), + storage_class: Some("STANDARD".to_string()), + version_id: Some("null".to_string()), + delete_marker: false, + user_metadata: { + let mut metadata = std::collections::HashMap::new(); + metadata.insert("custom-key".to_string(), "value".to_string()); + metadata + }, + e_tag: Some("\"abc123\"".to_string()), + last_modified: Some("2024-01-01T12:00:00Z".to_string()), + checksum_crc32: Some("crc32".to_string()), + checksum_crc32c: None, + checksum_sha1: None, + checksum_sha256: None, + checksum_crc64nvme: None, + checksum_type: Some(s3s::dto::ChecksumType::from_static(s3s::dto::ChecksumType::FULL_OBJECT)), + }); + + assert_eq!(obj.content_length, 9); + assert_eq!(obj.content_type.as_deref(), Some("text/plain")); + assert_eq!(obj.content_encoding.as_deref(), Some("gzip")); + assert_eq!(obj.cache_control.as_deref(), Some("max-age=3600")); + assert_eq!(obj.content_disposition.as_deref(), Some("attachment")); + assert_eq!(obj.content_language.as_deref(), Some("en-US")); + assert_eq!(obj.expires.as_deref(), Some("2024-12-31T23:59:59Z")); + assert_eq!(obj.storage_class.as_deref(), Some("STANDARD")); + assert_eq!(obj.version_id.as_deref(), Some("null")); + assert!(!obj.delete_marker); + assert_eq!(obj.user_metadata.get("custom-key").map(String::as_str), Some("value")); + assert_eq!(obj.e_tag.as_deref(), Some("\"abc123\"")); + assert_eq!(obj.last_modified.as_deref(), Some("2024-01-01T12:00:00Z")); + assert_eq!(obj.checksum_crc32.as_deref(), Some("crc32")); + assert_eq!( + obj.checksum_type, + Some(s3s::dto::ChecksumType::from_static(s3s::dto::ChecksumType::FULL_OBJECT)) + ); + assert!(Arc::ptr_eq(&obj.body, &body)); + } + #[test] fn test_cached_get_object_size() { let obj = CachedGetObject::new(Bytes::from("test"), 4); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 28679492f..d488a0209 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -29,7 +29,6 @@ use rustfs_ecstore::{ use rustfs_s3_common::{S3Operation, record_s3_op}; use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error}; use std::fmt::Debug; -use tokio::io::{AsyncRead, AsyncSeek}; use tracing::{debug, error, instrument, warn}; use uuid::Uuid; @@ -44,44 +43,6 @@ pub(crate) struct ListObjectUnorderedQuery { pub(crate) allow_unordered: Option, } -pub(crate) struct InMemoryAsyncReader { - cursor: std::io::Cursor>, -} - -impl InMemoryAsyncReader { - pub(crate) fn new(data: Vec) -> Self { - Self { - cursor: std::io::Cursor::new(data), - } - } -} - -impl AsyncRead for InMemoryAsyncReader { - fn poll_read( - mut self: std::pin::Pin<&mut Self>, - _cx: &mut std::task::Context<'_>, - buf: &mut tokio::io::ReadBuf<'_>, - ) -> std::task::Poll> { - let unfilled = buf.initialize_unfilled(); - let bytes_read = std::io::Read::read(&mut self.cursor, unfilled)?; - buf.advance(bytes_read); - std::task::Poll::Ready(Ok(())) - } -} - -impl AsyncSeek for InMemoryAsyncReader { - fn start_seek(mut self: std::pin::Pin<&mut Self>, position: std::io::SeekFrom) -> std::io::Result<()> { - // std::io::Cursor natively supports negative SeekCurrent offsets - // It will automatically handle validation and return an error if the final position would be negative - std::io::Seek::seek(&mut self.cursor, position)?; - Ok(()) - } - - fn poll_complete(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll> { - std::task::Poll::Ready(Ok(self.cursor.position())) - } -} - impl FS { pub fn new() -> Self { rustfs_s3_common::init_s3_metrics(); diff --git a/scripts/bench-small-put-local.sh b/scripts/bench-small-put-local.sh new file mode 100755 index 000000000..eb10b4a70 --- /dev/null +++ b/scripts/bench-small-put-local.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +RUSTFS_BIN="${ROOT_DIR}/target/debug/rustfs" +WARP_BIN="${WARP_BIN:-warp}" +MC_BIN="${MC_BIN:-mc}" + +PORT="${RUSTFS_BENCH_PORT:-9900}" +CONSOLE_PORT="${RUSTFS_BENCH_CONSOLE_PORT:-9901}" +HOST="127.0.0.1:${PORT}" +ACCESS_KEY="${RUSTFS_BENCH_ACCESS_KEY:-rustfsadmin}" +SECRET_KEY="${RUSTFS_BENCH_SECRET_KEY:-rustfsadmin}" +BENCH_ROOT="${ROOT_DIR}/target/benchmarks" +RUN_ID="${RUSTFS_BENCH_RUN_ID:-small-put-$(date +%Y%m%d-%H%M%S)}" +RUN_DIR="${BENCH_ROOT}/${RUN_ID}" +VOLUME_DIR="${RUN_DIR}/volumes" +LOG_DIR="${RUN_DIR}/logs" +MC_CONFIG_DIR="${RUN_DIR}/mc-config" +RESULTS_MD="${RUN_DIR}/RESULTS.md" +SERVER_LOG="${LOG_DIR}/rustfs.log" +WARP_BUCKET="${RUSTFS_BENCH_BUCKET:-warp-small-put-benchmark}" +WARP_DURATION="${RUSTFS_BENCH_DURATION:-20s}" +WARP_CONCURRENCY="${RUSTFS_BENCH_CONCURRENCY:-32}" + +mkdir -p "${VOLUME_DIR}" "${LOG_DIR}" "${MC_CONFIG_DIR}" +mkdir -p "${VOLUME_DIR}"/disk{1..4} + +# Verify mc (MinIO Client) is available +if ! command -v "${MC_BIN}" >/dev/null 2>&1; then + echo "Error: 'mc' (MinIO Client) not found. MC_BIN=${MC_BIN}" >&2 + echo "" >&2 + echo "Install mc with one of the following:" >&2 + echo " macOS (Homebrew): brew install minio/stable/mc" >&2 + echo " Go: go install github.com/minio/mc@latest" >&2 + echo " Linux (amd64): curl -sSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2 + echo " Linux (arm64): curl -sSL https://dl.min.io/client/mc/release/linux-arm64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2 + echo "" >&2 + echo "Or set MC_BIN=/path/to/mc before running this script." >&2 + exit 1 +fi + +# Verify warp is available +if ! command -v "${WARP_BIN}" >/dev/null 2>&1; then + echo "Error: 'warp' benchmark tool not found. WARP_BIN=${WARP_BIN}" >&2 + echo "" >&2 + echo "Install warp with one of the following:" >&2 + echo " Go: go install github.com/minio/warp@latest" >&2 + echo " Linux (amd64): curl -sSL https://github.com/minio/warp/releases/latest/download/warp_Linux_x86_64.tar.gz | tar -xz -C /usr/local/bin warp" >&2 + echo " macOS (amd64): curl -sSL https://github.com/minio/warp/releases/latest/download/warp_Darwin_x86_64.tar.gz | tar -xz -C /usr/local/bin warp" >&2 + echo " macOS (arm64): curl -sSL https://github.com/minio/warp/releases/latest/download/warp_Darwin_arm64.tar.gz | tar -xz -C /usr/local/bin warp" >&2 + echo "" >&2 + echo "Or set WARP_BIN=/path/to/warp before running this script." >&2 + exit 1 +fi + +SIZES=( + "4KiB" + "16KiB" + "64KiB" + "256KiB" + "1MiB" +) + +cleanup() { + if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" >/dev/null 2>&1; then + kill "${SERVER_PID}" >/dev/null 2>&1 || true + wait "${SERVER_PID}" >/dev/null 2>&1 || true + fi +} + +trap cleanup EXIT + +echo "Starting local RustFS benchmark server..." +( + cd "${ROOT_DIR}" + export RUST_LOG="${RUST_LOG:-warn}" + export RUSTFS_VOLUMES="${VOLUME_DIR}/disk{1...4}" + export RUSTFS_ADDRESS=":${PORT}" + export RUSTFS_CONSOLE_ENABLE=false + export RUSTFS_CONSOLE_ADDRESS=":${CONSOLE_PORT}" + export RUSTFS_ROOT_USER="${ACCESS_KEY}" + export RUSTFS_ROOT_PASSWORD="${SECRET_KEY}" + export RUSTFS_OBJECT_CACHE_ENABLE=false + "${RUSTFS_BIN}" server >"${SERVER_LOG}" 2>&1 +) & +SERVER_PID=$! + +echo "Waiting for RustFS server to become ready..." +for _ in $(seq 1 60); do + if curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 || curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then + break + fi + sleep 1 +done + +if ! curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 && ! curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then + echo "RustFS did not become ready in time. See ${SERVER_LOG}" >&2 + exit 1 +fi + +echo "Preparing mc alias..." +MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" alias set bench "http://${HOST}" "${ACCESS_KEY}" "${SECRET_KEY}" >/dev/null +MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" mb --ignore-existing "bench/${WARP_BUCKET}" >/dev/null + +{ + echo "# Small PUT Benchmark Results" + echo + echo "- Run ID: \`${RUN_ID}\`" + echo "- Host: \`${HOST}\`" + echo "- Duration per size: \`${WARP_DURATION}\`" + echo "- Concurrency: \`${WARP_CONCURRENCY}\`" + echo "- Bucket: \`${WARP_BUCKET}\`" + echo +} >"${RESULTS_MD}" + +for size in "${SIZES[@]}"; do + echo + echo "Running warp PUT benchmark for size ${size}..." + size_slug="$(echo "${size}" | tr '[:upper:]' '[:lower:]')" + OUT_FILE="${LOG_DIR}/warp-put-${size}.log" + BENCHDATA_FILE="${RUN_DIR}/warp-put-${size}.csv.zst" + "${WARP_BIN}" put \ + --no-color \ + --host "${HOST}" \ + --access-key "${ACCESS_KEY}" \ + --secret-key "${SECRET_KEY}" \ + --bucket "${WARP_BUCKET}" \ + --obj.size "${size}" \ + --duration "${WARP_DURATION}" \ + --concurrent "${WARP_CONCURRENCY}" \ + --prefix "${RUN_ID}/${size_slug}" \ + --disable-multipart \ + --noclear \ + --benchdata "${BENCHDATA_FILE}" \ + --insecure \ + >"${OUT_FILE}" 2>&1 + + { + echo "## ${size}" + echo + echo '```text' + tail -n 20 "${OUT_FILE}" + echo '```' + echo + } >>"${RESULTS_MD}" +done + +echo +echo "Benchmark completed." +echo "Results: ${RESULTS_MD}" +echo "Server log: ${SERVER_LOG}" diff --git a/scripts/bench-small-put-mc-local.sh b/scripts/bench-small-put-mc-local.sh new file mode 100755 index 000000000..d7f716a76 --- /dev/null +++ b/scripts/bench-small-put-mc-local.sh @@ -0,0 +1,300 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +RUSTFS_BIN="${RUSTFS_BIN:-${ROOT_DIR}/target/debug/rustfs}" +MC_BIN="${MC_BIN:-mc}" + +PORT="${RUSTFS_BENCH_PORT:-9910}" +CONSOLE_PORT="${RUSTFS_BENCH_CONSOLE_PORT:-9911}" +HOST="127.0.0.1:${PORT}" +ACCESS_KEY="${RUSTFS_BENCH_ACCESS_KEY:-rustfsadmin}" +SECRET_KEY="${RUSTFS_BENCH_SECRET_KEY:-rustfsadmin}" + +OPS_PER_SIZE="${RUSTFS_BENCH_OPS_PER_SIZE:-0}" +BENCH_SECONDS="${RUSTFS_BENCH_SECONDS:-10}" +CONCURRENCY="${RUSTFS_BENCH_CONCURRENCY:-32}" +PUT_TIMEOUT_SECS="${RUSTFS_BENCH_PUT_TIMEOUT_SECS:-15}" +RUN_ID="${RUSTFS_BENCH_RUN_ID:-small-put-mc-$(date +%Y%m%d-%H%M%S)}" +BENCH_ROOT="${ROOT_DIR}/target/benchmarks" +RUN_DIR="${BENCH_ROOT}/${RUN_ID}" +VOLUME_DIR="${RUN_DIR}/volumes" +LOG_DIR="${RUN_DIR}/logs" +FILE_DIR="${RUN_DIR}/files" +MC_CONFIG_DIR="${RUN_DIR}/mc-config" +RESULTS_MD="${RUN_DIR}/RESULTS.md" +SERVER_LOG="${LOG_DIR}/rustfs.log" +BUCKET="${RUSTFS_BENCH_BUCKET:-small-put-benchmark}" + +# Verify mc (MinIO Client) is available +if ! command -v "${MC_BIN}" >/dev/null 2>&1; then + echo "Error: 'mc' (MinIO Client) not found. MC_BIN=${MC_BIN}" >&2 + echo "" >&2 + echo "Install mc with one of the following:" >&2 + echo " macOS (Homebrew): brew install minio/stable/mc" >&2 + echo " Go: go install github.com/minio/mc@latest" >&2 + echo " Linux (amd64): curl -sSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2 + echo " Linux (arm64): curl -sSL https://dl.min.io/client/mc/release/linux-arm64/mc -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc" >&2 + echo "" >&2 + echo "Or set MC_BIN=/path/to/mc before running this script." >&2 + exit 1 +fi + +mkdir -p "${VOLUME_DIR}" "${LOG_DIR}" "${FILE_DIR}" "${MC_CONFIG_DIR}" +mkdir -p "${VOLUME_DIR}"/disk{1..4} + +SIZES=( + "4KiB" + "16KiB" + "64KiB" + "256KiB" + "1MiB" +) + +size_to_bytes() { + case "$1" in + 4KiB) echo 4096 ;; + 16KiB) echo 16384 ;; + 64KiB) echo 65536 ;; + 256KiB) echo 262144 ;; + 1MiB) echo 1048576 ;; + *) + echo "unsupported size: $1" >&2 + return 1 + ;; + esac +} + +now_secs() { + perl -MTime::HiRes=time -e 'printf "%.6f\n", time' +} + +prepare_sample_file() { + local size_label="$1" + local bytes="$2" + local file_path="${FILE_DIR}/${size_label}.bin" + if [[ ! -f "${file_path}" ]]; then + dd if=/dev/zero of="${file_path}" bs="${bytes}" count=1 status=none + fi + echo "${file_path}" +} + +run_mc_put_with_timeout() { + local sample_file="$1" + local key="$2" + perl -e ' + use strict; + use warnings; + + my $timeout = shift @ARGV; + my $pid = fork(); + die "fork failed: $!" unless defined $pid; + + if ($pid == 0) { + exec @ARGV; + die "exec failed: $!"; + } + + my $timed_out = 0; + local $SIG{ALRM} = sub { + $timed_out = 1; + kill "TERM", $pid; + select undef, undef, undef, 0.2; + kill "KILL", $pid; + }; + + alarm($timeout); + my $waited = waitpid($pid, 0); + alarm(0); + + exit(1) if $timed_out; + exit(($waited == $pid && $? == 0) ? 0 : 1); + ' "${PUT_TIMEOUT_SECS}" env MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" put --quiet --disable-multipart "${sample_file}" "${key}" >/dev/null 2>&1 +} + +export -f run_mc_put_with_timeout + +cleanup() { + if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" >/dev/null 2>&1; then + kill "${SERVER_PID}" >/dev/null 2>&1 || true + wait "${SERVER_PID}" >/dev/null 2>&1 || true + fi +} + +trap cleanup EXIT + +echo "Starting local RustFS benchmark server..." +( + cd "${ROOT_DIR}" + export RUST_LOG="${RUST_LOG:-warn}" + export RUSTFS_VOLUMES="${VOLUME_DIR}/disk{1...4}" + export RUSTFS_ADDRESS=":${PORT}" + export RUSTFS_CONSOLE_ENABLE=false + export RUSTFS_CONSOLE_ADDRESS=":${CONSOLE_PORT}" + export RUSTFS_ROOT_USER="${ACCESS_KEY}" + export RUSTFS_ROOT_PASSWORD="${SECRET_KEY}" + export RUSTFS_OBJECT_CACHE_ENABLE=false + "${RUSTFS_BIN}" server >"${SERVER_LOG}" 2>&1 +) & +SERVER_PID=$! + +echo "Waiting for RustFS server to become ready..." +for _ in $(seq 1 60); do + if curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 || curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then + break + fi + sleep 1 +done + +if ! curl -fsS "http://${HOST}/health/ready" >/dev/null 2>&1 && ! curl -fsS "http://${HOST}/minio/health/ready" >/dev/null 2>&1; then + echo "RustFS did not become ready in time. See ${SERVER_LOG}" >&2 + exit 1 +fi + +echo "Preparing mc alias..." +MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" alias set bench "http://${HOST}" "${ACCESS_KEY}" "${SECRET_KEY}" >/dev/null +MC_CONFIG_DIR="${MC_CONFIG_DIR}" "${MC_BIN}" mb --ignore-existing "bench/${BUCKET}" >/dev/null + +{ + echo "# Small PUT Benchmark Results" + echo + echo "- Run ID: \`${RUN_ID}\`" + echo "- Host: \`${HOST}\`" + echo "- Bucket: \`${BUCKET}\`" + if [[ "${OPS_PER_SIZE}" -gt 0 ]]; then + echo "- Ops per size: \`${OPS_PER_SIZE}\`" + else + echo "- Duration per size: \`${BENCH_SECONDS}s\`" + fi + echo "- Concurrency: \`${CONCURRENCY}\`" + echo "- Per request timeout: \`${PUT_TIMEOUT_SECS}s\`" + echo +} >"${RESULTS_MD}" + +for size_label in "${SIZES[@]}"; do + bytes="$(size_to_bytes "${size_label}")" + sample_file="$(prepare_sample_file "${size_label}" "${bytes}")" + raw_json="${RUN_DIR}/${size_label}.jsonl" + prefix="$(echo "${size_label}" | tr '[:upper:]' '[:lower:]')" + worker_dir="${RUN_DIR}/${size_label}-workers" + mkdir -p "${worker_dir}" + + echo "Running mc PUT benchmark for size ${size_label}..." + batch_start="$(now_secs)" + if [[ "${OPS_PER_SIZE}" -gt 0 ]]; then + seq 1 "${OPS_PER_SIZE}" | xargs -n1 -P "${CONCURRENCY}" -I{} \ + env MC_CONFIG_DIR="${MC_CONFIG_DIR}" MC_BIN="${MC_BIN}" PUT_TIMEOUT_SECS="${PUT_TIMEOUT_SECS}" SAMPLE_FILE="${sample_file}" BUCKET="${BUCKET}" PREFIX="${prefix}" /bin/bash -lc ' + idx="$1" + start=$(perl -MTime::HiRes=time -e '"'"'printf "%.6f\n", time'"'"') + if run_mc_put_with_timeout "${SAMPLE_FILE}" "bench/${BUCKET}/${PREFIX}/obj-${idx}.bin"; then + finish=$(perl -MTime::HiRes=time -e '"'"'printf "%.6f\n", time'"'"') + perl -e '"'"' + use strict; + use warnings; + my ($idx, $start, $finish) = @ARGV; + my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000); + print "{\"idx\":${idx},\"ok\":true,\"duration_ms\":${duration_ms}}\n"; + '"'"' -- "${idx}" "${start}" "${finish}" + else + finish=$(perl -MTime::HiRes=time -e '"'"'printf "%.6f\n", time'"'"') + perl -e '"'"' + use strict; + use warnings; + my ($idx, $start, $finish) = @ARGV; + my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000); + print "{\"idx\":${idx},\"ok\":false,\"duration_ms\":${duration_ms}}\n"; + '"'"' -- "${idx}" "${start}" "${finish}" + fi + ' _ {} >"${raw_json}" + else + bench_deadline="$(perl -e 'use strict; use warnings; my ($start, $secs) = @ARGV; printf "%.6f", $start + $secs;' -- "${batch_start}" "${BENCH_SECONDS}")" + pids=() + for worker in $(seq 1 "${CONCURRENCY}"); do + ( + out_file="${worker_dir}/worker-${worker}.jsonl" + idx=0 + while true; do + now="$(now_secs)" + if perl -e 'use strict; use warnings; exit(($ARGV[0] >= $ARGV[1]) ? 0 : 1)' -- "${now}" "${bench_deadline}"; then + break + fi + start="${now}" + key="bench/${BUCKET}/${prefix}/worker-${worker}/obj-${idx}.bin" + if run_mc_put_with_timeout "${sample_file}" "${key}"; then + finish="$(now_secs)" + perl -e ' + use strict; + use warnings; + my ($idx, $start, $finish) = @ARGV; + my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000); + print "{\"idx\":${idx},\"ok\":true,\"duration_ms\":${duration_ms}}\n"; + ' -- "${idx}" "${start}" "${finish}" >>"${out_file}" + else + finish="$(now_secs)" + perl -e ' + use strict; + use warnings; + my ($idx, $start, $finish) = @ARGV; + my $duration_ms = sprintf("%.3f", ($finish - $start) * 1000); + print "{\"idx\":${idx},\"ok\":false,\"duration_ms\":${duration_ms}}\n"; + ' -- "${idx}" "${start}" "${finish}" >>"${out_file}" + fi + idx=$((idx + 1)) + done + ) & + pids+=($!) + done + for pid in "${pids[@]}"; do + wait "${pid}" + done + cat "${worker_dir}"/worker-*.jsonl >"${raw_json}" + fi + batch_finish="$(now_secs)" + + jq_summary="$(jq -s ' + def pct(p): + if length == 0 then null + else (sort_by(.duration_ms) | .[((length - 1) * p | floor)].duration_ms) + end; + { + total: length, + succeeded: (map(select(.ok == true)) | length), + failed: (map(select(.ok == false)) | length), + avg_ms: (if length == 0 then null else (map(.duration_ms) | add / length) end), + p50_ms: pct(0.50), + p90_ms: pct(0.90), + p99_ms: pct(0.99) + } + ' "${raw_json}")" + + success_count="$(echo "${jq_summary}" | jq -r '.succeeded')" + avg_ms="$(echo "${jq_summary}" | jq -r '.avg_ms')" + p50_ms="$(echo "${jq_summary}" | jq -r '.p50_ms')" + p90_ms="$(echo "${jq_summary}" | jq -r '.p90_ms')" + p99_ms="$(echo "${jq_summary}" | jq -r '.p99_ms')" + failed_count="$(echo "${jq_summary}" | jq -r '.failed')" + + wall_secs="$(perl -e 'use strict; use warnings; my ($start, $finish) = @ARGV; printf "%.6f", ($finish - $start);' -- "${batch_start}" "${batch_finish}")" + mib_per_sec="$(perl -e 'use strict; use warnings; my ($bytes, $count, $secs) = @ARGV; printf "%.3f", (($bytes * $count) / (1024 * 1024)) / $secs;' -- "${bytes}" "${success_count}" "${wall_secs}")" + obj_per_sec="$(perl -e 'use strict; use warnings; my ($count, $secs) = @ARGV; printf "%.3f", $count / $secs;' -- "${success_count}" "${wall_secs}")" + + { + echo "## ${size_label}" + echo + echo "- Successful PUTs: \`${success_count}\`" + echo "- Failed PUTs: \`${failed_count}\`" + echo "- Wall time: \`${wall_secs}s\`" + echo "- Throughput: \`${mib_per_sec} MiB/s\`" + echo "- Object rate: \`${obj_per_sec} obj/s\`" + echo "- Avg latency: \`${avg_ms} ms\`" + echo "- p50 latency: \`${p50_ms} ms\`" + echo "- p90 latency: \`${p90_ms} ms\`" + echo "- p99 latency: \`${p99_ms} ms\`" + echo + } >>"${RESULTS_MD}" +done + +echo +echo "Benchmark completed." +echo "Results: ${RESULTS_MD}" +echo "Server log: ${SERVER_LOG}" diff --git a/scripts/run.sh b/scripts/run.sh index 36c4f788c..4d3e94dce 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -38,7 +38,8 @@ mkdir -p ./target/volume/test{1..4} if [ -z "$RUST_LOG" ]; then export RUST_BACKTRACE=1 - export RUST_LOG="info,rustfs=debug,rustfs_ecstore=info,s3s=debug,rustfs_iam=info,rustfs_notify=info" +# export RUST_LOG="info,rustfs=debug,rustfs_ecstore=info,s3s=debug,rustfs_iam=info,rustfs_notify=info" + export RUST_LOG="error" fi # export RUSTFS_ERASURE_SET_DRIVE_COUNT=5 @@ -71,7 +72,7 @@ export RUSTFS_OBS_SERVICE_NAME=rustfs # Service name export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version export RUSTFS_OBS_ENVIRONMENT=production # Environment name development, staging, production export RUSTFS_OBS_LOGGER_LEVEL=info # Log level, supports trace, debug, info, warn, error -export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging +export RUSTFS_OBS_LOG_STDOUT_ENABLED=false # Whether to enable local stdout logging export RUSTFS_OBS_LOG_DIRECTORY="$current_dir/deploy/logs" # Log directory export RUSTFS_OBS_LOG_ROTATION_TIME="minutely" # Log rotation time unit, can be "minutely", "hourly", "daily" export RUSTFS_OBS_LOG_KEEP_FILES=10 # Number of log files to keep