diff --git a/Cargo.lock b/Cargo.lock index 2f27b0cd1..a65584942 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -893,9 +893,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -904,14 +904,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -1715,9 +1716,9 @@ dependencies = [ [[package]] name = "camino" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ce8d3bd5823c7504d3f579f13e7b2f3da252fcb938c594d5680ee508bf846f" +checksum = "5f2d30e4173c4026932d51d31d6b0613b1fd3014bf3f9f8943d4ba139c437ba0" dependencies = [ "serde_core", ] @@ -5001,9 +5002,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ "ctutils", "subtle", @@ -5332,9 +5333,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" +checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0" dependencies = [ "bitflags 2.13.0", "cfg-if", @@ -5447,9 +5448,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.29" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46" +checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" dependencies = [ "defmt", "jiff-static", @@ -5463,9 +5464,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.29" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f" +checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" dependencies = [ "proc-macro2", "quote", @@ -5789,18 +5790,18 @@ dependencies = [ [[package]] name = "liblzma" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899" +checksum = "45aec2360b3933207e27908049d8e4df4e476b58180afb1e56b2a4fb72efe4ba" dependencies = [ "liblzma-sys", ] [[package]] name = "liblzma-sys" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6" +checksum = "a046c7f353ba30f810545151e04f63545833803f5b86ee3ddf1517247fe560a5" dependencies = [ "cc", "libc", @@ -7567,11 +7568,12 @@ dependencies = [ [[package]] name = "pkcs5" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279a91971a1d8eb1260a30938eae3be9cb67b472dffecb222fbbbe2fd2dc1453" +checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca" dependencies = [ "aes 0.9.1", + "aes-gcm", "cbc 0.2.1", "der 0.8.0", "pbkdf2 0.13.0", @@ -7598,7 +7600,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ "der 0.8.0", - "pkcs5 0.8.0", + "pkcs5 0.8.1", "rand_core 0.10.1", "spki 0.8.0", ] @@ -8227,6 +8229,16 @@ name = "quick-xml" version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "encoding_rs", "memchr", @@ -8287,7 +8299,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -8418,9 +8430,9 @@ dependencies = [ [[package]] name = "rapidhash" -version = "4.4.1" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59" +checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37" dependencies = [ "rustversion", ] @@ -8901,7 +8913,7 @@ dependencies = [ "pageant", "pbkdf2 0.13.0", "pkcs1 0.8.0-rc.4", - "pkcs5 0.8.0", + "pkcs5 0.8.1", "pkcs8 0.11.0", "polyval", "rand 0.10.1", @@ -9330,7 +9342,7 @@ dependencies = [ "path-absolutize", "pin-project-lite", "proptest", - "quick-xml 0.40.1", + "quick-xml 0.41.0", "rand 0.10.1", "ratelimit", "rcgen", @@ -9682,7 +9694,7 @@ dependencies = [ "hashbrown 0.17.1", "metrics", "percent-encoding", - "quick-xml 0.40.1", + "quick-xml 0.41.0", "rayon", "rustc-hash", "rustfs-config", @@ -9821,7 +9833,7 @@ dependencies = [ "md5", "percent-encoding", "proptest", - "quick-xml 0.40.1", + "quick-xml 0.41.0", "regex", "russh", "russh-sftp", @@ -10318,9 +10330,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", @@ -11353,9 +11365,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "suppaftp" -version = "9.0.0" +version = "10.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b367d8e84825dc6e858d2044f8cfff5e80f63b2aa0634c6382fc9a94b6926b22" +checksum = "81b8286a5639930b8fc0400f8a5111439ffb57e5fba6e8276a8bce857ec47b18" dependencies = [ "async-trait", "chrono", @@ -11655,9 +11667,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.51" +version = "0.3.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +checksum = "0e48db7b415311b615f910b3dcaa4557bcd4bf1982379c95c223fd8c2a20e210" dependencies = [ "deranged", "js-sys", @@ -11678,9 +11690,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.30" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", @@ -12781,7 +12793,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -12790,7 +12802,16 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -12808,14 +12829,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -12833,48 +12871,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "1.0.3" diff --git a/Cargo.toml b/Cargo.toml index 9023ca61a..eb893830c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -168,7 +168,7 @@ byteorder = "1.5.0" flatbuffers = "25.12.19" form_urlencoded = "1.2.2" prost = "0.14.4" -quick-xml = "0.40.1" +quick-xml = "0.41.0" rmp = { version = "0.8.15" } rmp-serde = { version = "1.3.1" } serde = { version = "1.0.228", features = ["derive"] } @@ -192,7 +192,7 @@ pbkdf2 = "0.13.0" rsa = { version = "=0.10.0-rc.18" } rustls = { version = "0.23.41", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] } rustls-native-certs = "0.8" -rustls-pki-types = "1.14.1" +rustls-pki-types = "1.15.0" sha1 = "0.11.0" sha2 = "0.11.0" subtle = "2.6" @@ -201,8 +201,8 @@ zeroize = { version = "1.9.0", features = ["derive"] } # Time and Date chrono = { version = "0.4.45", features = ["serde"] } humantime = "2.3.0" -jiff = { version = "0.2.29", features = ["serde"] } -time = { version = "0.3.51", features = ["std", "parsing", "formatting", "macros", "serde"] } +jiff = { version = "0.2.31", features = ["serde"] } +time = { version = "0.3.52", features = ["std", "parsing", "formatting", "macros", "serde"] } # Database deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] } @@ -211,7 +211,7 @@ tokio-postgres-rustls = "0.14.0" # Utilities and Tools anyhow = "1.0.103" -arc-swap = "1.9.1" +arc-swap = "1.9.2" astral-tokio-tar = "0.6.3" atoi = "3.1.0" atomic_enum = "0.3.0" @@ -323,7 +323,7 @@ pyroscope = { version = "=2.0.5", features = ["backend-pprof-rs", "backend-jemal # FTP and SFTP libunftp = { version = "0.23.0", features = ["experimental"] } unftp-core = "0.1.0" -suppaftp = { version = "9.0.0", features = ["tokio", "tokio-rustls-aws-lc-rs"] } +suppaftp = { version = "10.0.0", features = ["tokio", "tokio-rustls-aws-lc-rs"] } rcgen = "0.14.8" russh = { version = "0.61.2", features = ["serde"] } russh-sftp = "2.3.0" @@ -338,7 +338,7 @@ tikv-jemallocator = { version = "0.6", features = ["profiling", "stats", "unpref # Used to control and obtain statistics for jemalloc at runtime tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats", "profiling"] } # Snapshot testing for output format regression detection -insta = { version = "1.41", features = ["yaml", "json"] } +insta = { version = "1.48", features = ["yaml", "json"] } # Used to generate pprof-compatible memory profiling data and support symbolization and flame graphs jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] } # Used to generate CPU performance analysis data and flame diagrams diff --git a/crates/ecstore/src/client/object_api_utils.rs b/crates/ecstore/src/client/object_api_utils.rs index 7b930e741..944610198 100644 --- a/crates/ecstore/src/client/object_api_utils.rs +++ b/crates/ecstore/src/client/object_api_utils.rs @@ -149,6 +149,7 @@ pub fn new_getobjectreader<'a>( let r = GetObjectReader { object_info: oi.clone(), stream: Box::new(input_reader), + buffered_body: None, }; r //}) diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 025096a2d..28bb1c22f 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -1536,6 +1536,7 @@ mod tests { _guard: guard, }), object_info: self.object_info(bucket, object), + buffered_body: None, }) } diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index 4b6aa0789..58b29ca12 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -1149,6 +1149,7 @@ mod tests { let rd = GetObjectReader { stream: Box::new(Cursor::new(raw_payload.clone())), object_info: object_info.clone(), + buffered_body: None, }; let mut data = data_movement_put_object_reader("bucket-a", &object_info, rd, "test_migration") diff --git a/crates/ecstore/src/diagnostics/get.rs b/crates/ecstore/src/diagnostics/get.rs index 253467345..1872e9bef 100644 --- a/crates/ecstore/src/diagnostics/get.rs +++ b/crates/ecstore/src/diagnostics/get.rs @@ -21,9 +21,13 @@ pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING: &str = "codec_streaming"; pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE: &str = "codec_streaming_legacy_engine"; pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE: &str = "codec_streaming_rustfs_engine"; pub(crate) const GET_OBJECT_PATH_EMPTY: &str = "empty"; +pub(crate) const GET_OBJECT_PATH_DIRECT_MEMORY: &str = "direct_memory"; pub(crate) const GET_OBJECT_PATH_INLINE_DIRECT: &str = "inline_direct"; pub(crate) const GET_OBJECT_PATH_LEGACY_DUPLEX: &str = "legacy_duplex"; pub(crate) const GET_OBJECT_PATH_REMOTE_TRANSITION: &str = "remote_transition"; +pub(crate) const GET_OBJECT_PATH_SET_DISK: &str = "set_disk"; +pub(crate) const GET_DIRECT_MEMORY_SUBPATH_DISK_DATA_BLOCKS: &str = "disk_data_blocks"; +pub(crate) const GET_DIRECT_MEMORY_SUBPATH_INLINE_BUFFERED: &str = "inline_buffered"; pub(crate) const GET_CODEC_STREAMING_DECISION_USE: &str = "use"; pub(crate) const GET_CODEC_STREAMING_DECISION_FALLBACK: &str = "fallback"; pub(crate) const GET_CODEC_STREAMING_REASON_NONE: &str = "none"; @@ -42,13 +46,39 @@ pub(crate) const GET_STAGE_FIRST_METADATA_RESPONSE: &str = "first_metadata_respo pub(crate) const GET_STAGE_FIRST_VALID_METADATA_RESPONSE: &str = "first_valid_metadata_response"; pub(crate) const GET_STAGE_FIRST_SHARD_READ: &str = "first_shard_read"; pub(crate) const GET_STAGE_FULL_BODY: &str = "full_body"; +pub(crate) const GET_STAGE_INLINE_PREPARE: &str = "inline_prepare"; +pub(crate) const GET_STAGE_LOCK_ACQUIRE: &str = "lock_acquire"; pub(crate) const GET_STAGE_METADATA: &str = "metadata"; +pub(crate) const GET_STAGE_METADATA_CACHE_LOOKUP: &str = "metadata_cache_lookup"; pub(crate) const GET_STAGE_METADATA_FANOUT: &str = "metadata_fanout"; +pub(crate) const GET_STAGE_METADATA_RESOLVE: &str = "metadata_resolve"; +pub(crate) const GET_STAGE_OBJECT_INFO: &str = "object_info"; pub(crate) const GET_STAGE_OUTPUT_LOCK_WAIT: &str = "output_lock_wait"; pub(crate) const GET_STAGE_OUTPUT_POLL: &str = "output_poll"; +pub(crate) const GET_STAGE_PATH_DECISION: &str = "path_decision"; pub(crate) const GET_STAGE_QUORUM_REACHED: &str = "quorum_reached"; pub(crate) const GET_STAGE_RANGE: &str = "range"; pub(crate) const GET_STAGE_READER_SETUP: &str = "reader_setup"; +pub(crate) const GET_STAGE_READER_SETUP_DROP_PENDING: &str = "reader_setup_drop_pending"; +pub(crate) const GET_STAGE_READER_SETUP_SCHEDULE: &str = "reader_setup_schedule"; +pub(crate) const GET_STAGE_READER_SETUP_WAIT_QUORUM: &str = "reader_setup_wait_quorum"; +pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK: &str = "reader_open_mmap_copy_fallback"; +pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS: &str = "reader_open_mmap_copy_success"; +pub(crate) const GET_STAGE_READER_OPEN_STREAM: &str = "reader_open_stream"; +pub(crate) const GET_STAGE_READER_MMAP_ACCESS_CHECK: &str = "reader_mmap_access_check"; +pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_TASK: &str = "reader_mmap_blocking_task"; +pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_WAIT: &str = "reader_mmap_blocking_wait"; +pub(crate) const GET_STAGE_READER_MMAP_COPY_BUFFER: &str = "reader_mmap_copy_buffer"; +pub(crate) const GET_STAGE_READER_MMAP_DIRECT_READ_COPY: &str = "reader_mmap_direct_read_copy"; +pub(crate) const GET_STAGE_READER_MMAP_FILE_OPEN: &str = "reader_mmap_file_open"; +pub(crate) const GET_STAGE_READER_MMAP_MAP: &str = "reader_mmap_map"; +pub(crate) const GET_STAGE_READER_MMAP_METADATA_LOOKUP: &str = "reader_mmap_metadata_lookup"; +pub(crate) const GET_STAGE_READER_MMAP_METADATA_VALIDATE: &str = "reader_mmap_metadata_validate"; +pub(crate) const GET_STAGE_READER_MMAP_PATH_RESOLVE: &str = "reader_mmap_path_resolve"; +pub(crate) const GET_STAGE_READER_STREAM_FIRST_READ: &str = "reader_stream_first_read"; +pub(crate) const GET_STAGE_READER_TASK_BITROT_READER_INIT: &str = "reader_task_bitrot_reader_init"; +pub(crate) const GET_STAGE_READER_TASK_FILE_OPEN: &str = "reader_task_file_open"; +pub(crate) const GET_STAGE_READER_TASK_READER_CONSTRUCTION: &str = "reader_task_reader_construction"; pub(crate) const GET_STAGE_RECONSTRUCT: &str = "reconstruct"; pub(crate) const GET_STAGE_RESPONSE_HANDOFF: &str = "response_handoff"; pub(crate) const GET_STAGE_SLOWEST_METADATA_RESPONSE: &str = "slowest_metadata_response"; @@ -88,6 +118,25 @@ pub(crate) const GET_METADATA_RESPONSE_NOT_FOUND: &str = "not_found"; pub(crate) const GET_METADATA_RESPONSE_TIMEOUT: &str = "timeout"; pub(crate) const GET_METADATA_RESPONSE_VALID: &str = "valid"; pub(crate) const GET_METADATA_RESPONSE_VERSION_NOT_FOUND: &str = "version_not_found"; +pub(crate) const GET_METADATA_CACHE_DECISION_HIT: &str = "hit"; +pub(crate) const GET_METADATA_CACHE_DECISION_MISS: &str = "miss"; +pub(crate) const GET_METADATA_CACHE_DECISION_REJECT: &str = "reject"; +pub(crate) const GET_METADATA_CACHE_DECISION_SKIP: &str = "skip"; +pub(crate) const GET_METADATA_CACHE_REASON_DATA_MOVEMENT: &str = "data_movement"; +pub(crate) const GET_METADATA_CACHE_REASON_DELETE_MARKER: &str = "delete_marker"; +pub(crate) const GET_METADATA_CACHE_REASON_DIST_ERASURE: &str = "dist_erasure"; +pub(crate) const GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS: &str = "incl_free_versions"; +pub(crate) const GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM: &str = "insufficient_cached_quorum"; +pub(crate) const GET_METADATA_CACHE_REASON_META_BUCKET: &str = "meta_bucket"; +pub(crate) const GET_METADATA_CACHE_REASON_NO_LOCK: &str = "no_lock"; +pub(crate) const GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED: &str = "not_found_or_expired"; +pub(crate) const GET_METADATA_CACHE_REASON_NOT_READ_DATA: &str = "not_read_data"; +pub(crate) const GET_METADATA_CACHE_REASON_PART_NUMBER: &str = "part_number"; +pub(crate) const GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ: &str = "raw_data_movement_read"; +pub(crate) const GET_METADATA_CACHE_REASON_USABLE: &str = "usable"; +pub(crate) const GET_METADATA_CACHE_REASON_VERSION_ID: &str = "version_id"; +pub(crate) const GET_METADATA_CACHE_REASON_VERSION_SUSPENDED: &str = "version_suspended"; +pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error"; @@ -258,6 +307,7 @@ mod tests { assert_eq!(GET_CODEC_STREAMING_OBJECT_CLASS_COMPRESSED, "compressed"); assert_eq!(GET_CODEC_STREAMING_OBJECT_CLASS_REMOTE, "remote"); assert_eq!(GET_CODEC_STREAMING_OBJECT_CLASS_MULTIPART, "multipart"); + assert_eq!(GET_OBJECT_PATH_SET_DISK, "set_disk"); assert_eq!(GET_READER_PREFETCH_DIRECT, "direct"); assert_eq!(GET_READER_PREFETCH_STORED, "stored"); assert_eq!(GET_READER_PREFETCH_EOF, "eof"); @@ -275,10 +325,16 @@ mod tests { assert_eq!(GET_STAGE_FIRST_VALID_METADATA_RESPONSE, "first_valid_metadata_response"); assert_eq!(GET_STAGE_FIRST_SHARD_READ, "first_shard_read"); assert_eq!(GET_STAGE_FULL_BODY, "full_body"); + assert_eq!(GET_STAGE_INLINE_PREPARE, "inline_prepare"); + assert_eq!(GET_STAGE_LOCK_ACQUIRE, "lock_acquire"); assert_eq!(GET_STAGE_METADATA, "metadata"); + assert_eq!(GET_STAGE_METADATA_CACHE_LOOKUP, "metadata_cache_lookup"); assert_eq!(GET_STAGE_METADATA_FANOUT, "metadata_fanout"); + assert_eq!(GET_STAGE_METADATA_RESOLVE, "metadata_resolve"); + assert_eq!(GET_STAGE_OBJECT_INFO, "object_info"); assert_eq!(GET_STAGE_OUTPUT_LOCK_WAIT, "output_lock_wait"); assert_eq!(GET_STAGE_OUTPUT_POLL, "output_poll"); + assert_eq!(GET_STAGE_PATH_DECISION, "path_decision"); assert_eq!(GET_STAGE_QUORUM_REACHED, "quorum_reached"); assert_eq!(GET_STAGE_RANGE, "range"); assert_eq!(GET_STAGE_READER_SETUP, "reader_setup"); @@ -308,6 +364,25 @@ mod tests { assert_eq!(GET_METADATA_RESPONSE_TIMEOUT, "timeout"); assert_eq!(GET_METADATA_RESPONSE_VALID, "valid"); assert_eq!(GET_METADATA_RESPONSE_VERSION_NOT_FOUND, "version_not_found"); + assert_eq!(GET_METADATA_CACHE_DECISION_HIT, "hit"); + assert_eq!(GET_METADATA_CACHE_DECISION_MISS, "miss"); + assert_eq!(GET_METADATA_CACHE_DECISION_REJECT, "reject"); + assert_eq!(GET_METADATA_CACHE_DECISION_SKIP, "skip"); + assert_eq!(GET_METADATA_CACHE_REASON_DATA_MOVEMENT, "data_movement"); + assert_eq!(GET_METADATA_CACHE_REASON_DELETE_MARKER, "delete_marker"); + assert_eq!(GET_METADATA_CACHE_REASON_DIST_ERASURE, "dist_erasure"); + assert_eq!(GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, "incl_free_versions"); + assert_eq!(GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM, "insufficient_cached_quorum"); + assert_eq!(GET_METADATA_CACHE_REASON_META_BUCKET, "meta_bucket"); + assert_eq!(GET_METADATA_CACHE_REASON_NO_LOCK, "no_lock"); + assert_eq!(GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, "not_found_or_expired"); + assert_eq!(GET_METADATA_CACHE_REASON_NOT_READ_DATA, "not_read_data"); + assert_eq!(GET_METADATA_CACHE_REASON_PART_NUMBER, "part_number"); + assert_eq!(GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, "raw_data_movement_read"); + assert_eq!(GET_METADATA_CACHE_REASON_USABLE, "usable"); + assert_eq!(GET_METADATA_CACHE_REASON_VERSION_ID, "version_id"); + assert_eq!(GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, "version_suspended"); + assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error"); diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index c99cafb73..8d179ffb9 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -14,8 +14,8 @@ use crate::disk::{ CheckPartsResp, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint, Error, - FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result, UpdateMetadataOpts, VolumeInfo, - WalkDirOptions, + FileInfoVersions, MmapCopyStageMetrics, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result, + UpdateMetadataOpts, VolumeInfo, WalkDirOptions, health_state::{ RuntimeDriveHealthState, classify_drive_recovery, get_drive_returning_probe_interval, get_drive_returning_success_threshold, get_drive_suspect_failure_threshold, record_drive_offline_duration, @@ -1369,6 +1369,25 @@ impl DiskAPI for LocalDiskWrapper { .await } + async fn read_file_mmap_copy_with_metrics( + &self, + volume: &str, + path: &str, + offset: usize, + length: usize, + metrics: Option, + ) -> Result { + self.track_disk_health( + || async { + self.disk + .read_file_mmap_copy_with_metrics(volume, path, offset, length, metrics) + .await + }, + get_max_timeout_duration(), + ) + .await + } + async fn append_file(&self, volume: &str, path: &str) -> Result { self.track_disk_health(|| async { self.disk.append_file(volume, path).await }, Duration::ZERO) .await diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 7911a1ea4..fff424ef8 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -18,14 +18,17 @@ use crate::disk::disk_store::get_object_disk_read_timeout; use crate::disk::{ BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, - FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, - ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, - UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int, + FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, + RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, + STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int, endpoint::Endpoint, error::{DiskError, Error, FileAccessDeniedWithContext, Result}, error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}, format::FormatV3, - fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename}, + fs::{ + O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, access_std, lstat, lstat_std, remove, remove_all_std, + remove_std, rename, + }, os, os::{check_path_length, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source}, }; @@ -83,6 +86,130 @@ const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts"; const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed"; const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed"; const EVENT_DISK_LOCAL_FORMAT_DECODE_FAILED: &str = "disk_local_format_decode_failed"; +const METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_mmap_page_faults_total"; +const METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_direct_read_page_faults_total"; + +#[inline(always)] +fn record_mmap_copy_stage(metrics: MmapCopyStageMetrics, stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + rustfs_io_metrics::record_get_object_stage_duration(metrics.path, stage, started_at.elapsed().as_secs_f64()); + } +} + +#[cfg(unix)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct MmapPageFaultCounts { + minor: libc::c_long, + major: libc::c_long, +} + +#[cfg(unix)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct MmapPageFaultDelta { + minor: u64, + major: u64, +} + +#[cfg(all(unix, any(target_os = "linux", target_os = "android")))] +fn mmap_rusage_who() -> libc::c_int { + libc::RUSAGE_THREAD +} + +#[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))] +fn mmap_rusage_who() -> libc::c_int { + libc::RUSAGE_SELF +} + +#[cfg(unix)] +// SAFETY: this allowance is limited to reading kernel-provided rusage data via +// libc; each unsafe operation below documents pointer validity and initialization. +#[allow(unsafe_code)] +fn read_mmap_page_fault_counts(enabled: bool) -> Option { + if !enabled { + return None; + } + + let mut usage = std::mem::MaybeUninit::::uninit(); + // SAFETY: `getrusage` writes to the provided `rusage` pointer when it + // returns 0. The pointer is valid for writes and initialized only on success. + let rc = unsafe { libc::getrusage(mmap_rusage_who(), usage.as_mut_ptr()) }; + if rc != 0 { + return None; + } + + // SAFETY: `getrusage` returned success, so `usage` has been initialized. + let usage = unsafe { usage.assume_init() }; + Some(MmapPageFaultCounts { + minor: usage.ru_minflt, + major: usage.ru_majflt, + }) +} + +#[cfg(unix)] +fn non_negative_fault_delta(before: libc::c_long, after: libc::c_long) -> u64 { + if after <= before { + return 0; + } + + u64::try_from(after - before).unwrap_or(u64::MAX) +} + +#[cfg(unix)] +fn mmap_page_fault_delta(before: Option, after: Option) -> MmapPageFaultDelta { + match (before, after) { + (Some(before), Some(after)) => MmapPageFaultDelta { + minor: non_negative_fault_delta(before.minor, after.minor), + major: non_negative_fault_delta(before.major, after.major), + }, + _ => MmapPageFaultDelta::default(), + } +} + +#[cfg(unix)] +fn record_mmap_page_fault_delta(path: &'static str, stage: &'static str, delta: MmapPageFaultDelta) { + if delta.minor > 0 { + counter!( + METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL, + "path" => path, + "stage" => stage, + "kind" => "minor", + ) + .increment(delta.minor); + } + + if delta.major > 0 { + counter!( + METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL, + "path" => path, + "stage" => stage, + "kind" => "major", + ) + .increment(delta.major); + } +} + +#[cfg(unix)] +fn record_direct_read_page_fault_delta(path: &'static str, stage: &'static str, delta: MmapPageFaultDelta) { + if delta.minor > 0 { + counter!( + METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL, + "path" => path, + "stage" => stage, + "kind" => "minor", + ) + .increment(delta.minor); + } + + if delta.major > 0 { + counter!( + METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL, + "path" => path, + "stage" => stage, + "kind" => "major", + ) + .increment(delta.major); + } +} /// Enable O_DIRECT for large sequential reads. /// When enabled, shard reads bypass the page cache using O_DIRECT flag. @@ -96,6 +223,17 @@ const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false; /// Default: 4MB. const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD"; const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024; +const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE"; +const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false; +const ENV_RUSTFS_OBJECT_MMAP_READ_METHOD: &str = "RUSTFS_OBJECT_MMAP_READ_METHOD"; +const RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY: &str = "mmap_copy"; +const RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY: &str = "direct_read_copy"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum LocalReadCopyMethod { + MmapCopy, + DirectReadCopy, +} /// Check if O_DIRECT reads are enabled. fn is_direct_io_read_enabled() -> bool { @@ -107,6 +245,36 @@ fn get_direct_io_read_threshold() -> usize { rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD) } +fn should_populate_mmap_read(length: usize) -> bool { + length > 0 && rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE) +} + +fn local_read_copy_method() -> LocalReadCopyMethod { + let method = rustfs_utils::get_env_str(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY); + match method.as_str() { + RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY => LocalReadCopyMethod::DirectReadCopy, + _ => LocalReadCopyMethod::MmapCopy, + } +} + +#[cfg(unix)] +#[allow(unsafe_code)] +fn mmap_page_size() -> Result { + static PAGE_SIZE: OnceLock> = OnceLock::new(); + + PAGE_SIZE + .get_or_init(|| { + // SAFETY: `sysconf(_SC_PAGESIZE)` has no pointer arguments and only + // queries process-global OS configuration. + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page_size <= 0 { + return None; + } + u64::try_from(page_size).ok() + }) + .ok_or_else(|| DiskError::other("failed to determine system page size")) +} + #[cfg(test)] static RENAME_DATA_FAIL_BEFORE_OLD_METADATA_BACKUP: std::sync::Mutex> = std::sync::Mutex::new(None); @@ -1070,62 +1238,21 @@ impl LocalDisk { // Get the absolute path of an object pub fn get_object_path(&self, bucket: &str, key: &str) -> Result { - // For high-frequency paths, use faster string concatenation - let cache_key = if key.is_empty() { - bucket.to_string() - } else { - path_join_buf(&[bucket, key]) - }; - - #[cfg(windows)] - let path = self.root.join(cache_key.replace('/', "\\")); - #[cfg(not(windows))] - let path = self.root.join(cache_key); - - self.check_valid_path(&path)?; - Ok(path) + local_disk_object_path(&self.root, bucket, key) } // Get the absolute path of a bucket pub fn get_bucket_path(&self, bucket: &str) -> Result { - #[cfg(windows)] - let bucket_path = self.root.join(bucket.replace('/', "\\")); - #[cfg(not(windows))] - let bucket_path = self.root.join(bucket); - - self.check_valid_path(&bucket_path)?; - Ok(bucket_path) + local_disk_bucket_path(&self.root, bucket) } // Check if a path is valid fn check_valid_path>(&self, path: P) -> Result<()> { - let path = normalize_path_components(path); - if !path.starts_with(&self.root) { - return Err(DiskError::InvalidPath); - } - - self.reject_symlink_components(&path) + check_local_disk_valid_path(&self.root, path) } fn reject_symlink_components(&self, path: &Path) -> Result<()> { - let relative = path.strip_prefix(&self.root).map_err(|_| DiskError::InvalidPath)?; - let mut current = self.root.clone(); - - for component in relative.components() { - current.push(component.as_os_str()); - - match lstat_std(¤t) { - Ok(metadata) => { - if metadata.file_type().is_symlink() { - return Err(DiskError::InvalidPath); - } - } - Err(err) if err.kind() == ErrorKind::NotFound => break, - Err(err) => return Err(to_file_error(err).into()), - } - } - - Ok(()) + reject_local_disk_symlink_components(&self.root, path) } // Batch path generation with single lock acquisition @@ -2085,6 +2212,62 @@ fn skip_access_checks(p: impl AsRef) -> bool { false } +fn local_disk_object_path(root: &Path, bucket: &str, key: &str) -> Result { + let cache_key = if key.is_empty() { + bucket.to_string() + } else { + path_join_buf(&[bucket, key]) + }; + + #[cfg(windows)] + let path = root.join(cache_key.replace('/', "\\")); + #[cfg(not(windows))] + let path = root.join(cache_key); + + check_local_disk_valid_path(root, &path)?; + Ok(path) +} + +fn local_disk_bucket_path(root: &Path, bucket: &str) -> Result { + #[cfg(windows)] + let bucket_path = root.join(bucket.replace('/', "\\")); + #[cfg(not(windows))] + let bucket_path = root.join(bucket); + + check_local_disk_valid_path(root, &bucket_path)?; + Ok(bucket_path) +} + +fn check_local_disk_valid_path(root: &Path, path: impl AsRef) -> Result<()> { + let path = normalize_path_components(path); + if !path.starts_with(root) { + return Err(DiskError::InvalidPath); + } + + reject_local_disk_symlink_components(root, &path) +} + +fn reject_local_disk_symlink_components(root: &Path, path: &Path) -> Result<()> { + let relative = path.strip_prefix(root).map_err(|_| DiskError::InvalidPath)?; + let mut current = root.to_path_buf(); + + for component in relative.components() { + current.push(component.as_os_str()); + + match lstat_std(¤t) { + Ok(metadata) => { + if metadata.file_type().is_symlink() { + return Err(DiskError::InvalidPath); + } + } + Err(err) if err.kind() == ErrorKind::NotFound => break, + Err(err) => return Err(to_file_error(err).into()), + } + } + + Ok(()) +} + // Lightweight path normalization without filesystem calls fn normalize_path_components(path: impl AsRef) -> PathBuf { let path = path.as_ref(); @@ -2784,105 +2967,303 @@ impl DiskAPI for LocalDisk { #[allow(unsafe_code)] #[tracing::instrument(level = "debug", skip(self))] async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result { - let volume_dir = self.get_bucket_path(volume)?; - if !skip_access_checks(volume) { - access(&volume_dir) - .await - .map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?; - } - - let file_path = self.get_object_path(volume, path)?; - check_path_length(file_path.to_string_lossy().as_ref())?; - - // Verify file exists and get metadata - let file_path_clone = file_path.clone(); - let meta = tokio::task::spawn_blocking(move || std::fs::metadata(&file_path_clone).map_err(DiskError::from)) + self.read_file_mmap_copy_with_metrics(volume, path, offset, length, None) .await - .map_err(DiskError::from)??; + } - let end_offset = offset.checked_add(length).ok_or(DiskError::FileCorrupt)?; - if meta.len() < end_offset as u64 { - error!( - event = EVENT_DISK_LOCAL_READ_VERSION_FALLBACK, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - volume, - path, - offset, - length, - actual_size = meta.len(), - reason = "read_file_mmap_copy_out_of_bounds", - "Disk local read fallback failed" - ); + /// File read using mmap-then-copy on Unix or efficient read on non-Unix. + // SAFETY: Unix unsafe calls in this function only query page size and mmap + // a read-only file region after bounds and alignment are validated. + #[allow(unsafe_code)] + #[tracing::instrument(level = "debug", skip(self))] + async fn read_file_mmap_copy_with_metrics( + &self, + volume: &str, + path: &str, + offset: usize, + length: usize, + metrics: Option, + ) -> Result { + let metrics = metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled()); + let metrics_enabled = metrics.is_some(); + + let metadata_validate_start = metrics_enabled.then(std::time::Instant::now); + let Some(end_offset) = offset.checked_add(length) else { + if let Some(metrics) = metrics { + record_mmap_copy_stage(metrics, metrics.metadata_validate_stage, metadata_validate_start); + } return Err(DiskError::FileCorrupt); - } + }; // Unix: use mmap to read the data (copies into Bytes for safe ownership) // Non-Unix: fall back to efficient read #[cfg(unix)] { use memmap2::MmapOptions; - use std::time::Instant; + use std::time::{Duration as StdDuration, Instant as StdInstant}; - let start = Instant::now(); - let file_path_clone = file_path.clone(); + struct MmapCopyReadResult { + bytes: Bytes, + access_check_duration: StdDuration, + path_resolve_duration: StdDuration, + metadata_lookup_duration: StdDuration, + metadata_validate_duration: StdDuration, + file_open_duration: StdDuration, + mmap_map_duration: StdDuration, + mmap_copy_duration: StdDuration, + direct_read_copy_duration: StdDuration, + mmap_map_fault_delta: MmapPageFaultDelta, + mmap_copy_fault_delta: MmapPageFaultDelta, + direct_read_copy_fault_delta: MmapPageFaultDelta, + blocking_task_duration: StdDuration, + } + + enum MmapCopyReadError { + Disk(DiskError), + OutOfBounds { actual_size: u64 }, + } + + impl From for MmapCopyReadError { + fn from(err: DiskError) -> Self { + Self::Disk(err) + } + } + + let start = StdInstant::now(); + let root = self.root.clone(); + let volume_owned = volume.to_owned(); + let path_owned = path.to_owned(); let should_reclaim_after_read = should_reclaim_file_cache_after_read(length); - let bytes = tokio::task::spawn_blocking(move || { - let file = std::fs::File::open(&file_path_clone).map_err(DiskError::from)?; + let should_populate_mmap_read = should_populate_mmap_read(length); + let read_copy_method = local_read_copy_method(); + let offset_u64 = u64::try_from(offset).map_err(|_| DiskError::FileCorrupt)?; + let end_offset_u64 = u64::try_from(end_offset).map_err(|_| DiskError::FileCorrupt)?; + let blocking_wait_start = metrics_enabled.then(std::time::Instant::now); + let read_result = tokio::task::spawn_blocking(move || { + let blocking_task_start = metrics_enabled.then(StdInstant::now); + + let access_check_start = metrics_enabled.then(StdInstant::now); + let volume_dir = local_disk_bucket_path(&root, &volume_owned)?; + if !skip_access_checks(&volume_owned) { + access_std(&volume_dir).map_err(|e| DiskError::from(to_access_error(e, DiskError::VolumeAccessDenied)))?; + } + let access_check_duration = access_check_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed()); + + let path_resolve_start = metrics_enabled.then(StdInstant::now); + let file_path = local_disk_object_path(&root, &volume_owned, &path_owned)?; + check_path_length(file_path.to_string_lossy().as_ref())?; + let path_resolve_duration = path_resolve_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed()); + + let file_open_start = metrics_enabled.then(StdInstant::now); + let mut file = std::fs::File::open(&file_path).map_err(DiskError::from)?; + let file_open_duration = file_open_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed()); + + let metadata_lookup_start = metrics_enabled.then(StdInstant::now); + let meta = file.metadata().map_err(DiskError::from)?; + let metadata_lookup_duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed()); + + let metadata_validate_start = metrics_enabled.then(StdInstant::now); + if meta.len() < end_offset_u64 { + return Err(MmapCopyReadError::OutOfBounds { actual_size: meta.len() }); + } + let metadata_validate_duration = + metadata_validate_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed()); #[cfg(target_os = "macos")] if should_reclaim_after_read { let _ = set_std_fd_nocache(&file); } - // mmap offsets on Unix must be page-size aligned. Align the - // mapping down to the nearest page boundary, then slice out the - // originally requested logical range. - // SAFETY: `sysconf(_SC_PAGESIZE)` has no pointer arguments and - // only queries process-global OS configuration. - let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; - if page_size <= 0 { - return Err(DiskError::other("failed to determine system page size")); - } - let page_size = page_size as u64; - let offset_u64 = offset as u64; - let aligned_offset = offset_u64 - (offset_u64 % page_size); - let logical_offset = (offset_u64 - aligned_offset) as usize; - let map_len = logical_offset - .checked_add(length) - .ok_or_else(|| DiskError::other("mmap length overflow"))?; + let mut mmap_map_duration = StdDuration::ZERO; + let mut mmap_copy_duration = StdDuration::ZERO; + let mut direct_read_copy_duration = StdDuration::ZERO; + let mut mmap_map_fault_delta = MmapPageFaultDelta::default(); + let mut mmap_copy_fault_delta = MmapPageFaultDelta::default(); + let mut direct_read_copy_fault_delta = MmapPageFaultDelta::default(); + let mut _reclaim_offset = offset_u64; + let mut _reclaim_len = length; - // SAFETY: The file is opened as read-only, and we're mapping a region - // that we've already verified exists and is within file bounds. The - // file offset passed to mmap is page-size aligned as required on Unix. - let mmap = - unsafe { MmapOptions::new().offset(aligned_offset).len(map_len).map(&file) }.map_err(DiskError::other)?; + let bytes = match read_copy_method { + LocalReadCopyMethod::MmapCopy => { + // mmap offsets on Unix must be page-size aligned. Align the + // mapping down to the nearest page boundary, then slice out the + // originally requested logical range. + let page_size = mmap_page_size()?; + let aligned_offset = offset_u64 - (offset_u64 % page_size); + let logical_offset = + usize::try_from(offset_u64 - aligned_offset).map_err(|_| DiskError::other("mmap offset overflow"))?; + let map_len = logical_offset + .checked_add(length) + .ok_or_else(|| DiskError::other("mmap length overflow"))?; + _reclaim_offset = aligned_offset; + _reclaim_len = map_len; - // Copy only the requested logical range into a Bytes buffer. This - // avoids undefined behavior from treating OS-managed mmap memory as - // allocator-managed Vec storage, at the cost of an extra copy. - let end = logical_offset - .checked_add(length) - .ok_or_else(|| DiskError::other("mmap slice length overflow"))?; - let bytes = Bytes::copy_from_slice(&mmap[logical_offset..end]); + // SAFETY: The file is opened as read-only, and we're mapping a region + // that we've already verified exists and is within file bounds. The + // file offset passed to mmap is page-size aligned as required on Unix. + let mmap_map_start = metrics_enabled.then(StdInstant::now); + let mmap_map_faults_before = read_mmap_page_fault_counts(metrics_enabled); + let mut mmap_options = MmapOptions::new(); + mmap_options.offset(aligned_offset).len(map_len); + if should_populate_mmap_read { + mmap_options.populate(); + } + let mmap = unsafe { mmap_options.map(&file) }.map_err(DiskError::other)?; + let mmap_map_faults_after = read_mmap_page_fault_counts(metrics_enabled); + mmap_map_duration = mmap_map_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed()); + mmap_map_fault_delta = mmap_page_fault_delta(mmap_map_faults_before, mmap_map_faults_after); + + // Copy only the requested logical range into a Bytes buffer. This + // avoids undefined behavior from treating OS-managed mmap memory as + // allocator-managed Vec storage, at the cost of an extra copy. + let end = logical_offset + .checked_add(length) + .ok_or_else(|| DiskError::other("mmap slice length overflow"))?; + let mmap_copy_start = metrics_enabled.then(StdInstant::now); + let mmap_copy_faults_before = read_mmap_page_fault_counts(metrics_enabled); + let bytes = Bytes::copy_from_slice(&mmap[logical_offset..end]); + let mmap_copy_faults_after = read_mmap_page_fault_counts(metrics_enabled); + mmap_copy_duration = mmap_copy_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed()); + mmap_copy_fault_delta = mmap_page_fault_delta(mmap_copy_faults_before, mmap_copy_faults_after); + bytes + } + LocalReadCopyMethod::DirectReadCopy => { + use std::io::{Read as _, Seek as _}; + + let direct_read_copy_start = metrics_enabled.then(StdInstant::now); + let direct_read_copy_faults_before = read_mmap_page_fault_counts(metrics_enabled); + file.seek(SeekFrom::Start(offset_u64)).map_err(DiskError::from)?; + let mut buffer = vec![0; length]; + file.read_exact(&mut buffer).map_err(DiskError::from)?; + let direct_read_copy_faults_after = read_mmap_page_fault_counts(metrics_enabled); + direct_read_copy_duration = + direct_read_copy_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed()); + direct_read_copy_fault_delta = + mmap_page_fault_delta(direct_read_copy_faults_before, direct_read_copy_faults_after); + Bytes::from(buffer) + } + }; #[cfg(target_os = "linux")] - if should_reclaim_after_read { + if should_reclaim_after_read && _reclaim_len > 0 { use core::num::NonZeroU64; use rustix::fs::{Advice, fadvise}; - let reclaim_len = - NonZeroU64::new(map_len as u64).ok_or_else(|| DiskError::other("mmap reclaim length overflow"))?; - fadvise(&file, aligned_offset, Some(reclaim_len), Advice::DontNeed) + let reclaim_len = NonZeroU64::new( + u64::try_from(_reclaim_len).map_err(|_| DiskError::other("read reclaim length overflow"))?, + ) + .ok_or_else(|| DiskError::other("read reclaim length overflow"))?; + fadvise(&file, _reclaim_offset, Some(reclaim_len), Advice::DontNeed) .map_err(std::io::Error::from) .map_err(DiskError::from)?; } - Ok::(bytes) + let blocking_task_duration = blocking_task_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed()); + + Ok::(MmapCopyReadResult { + bytes, + access_check_duration, + path_resolve_duration, + metadata_lookup_duration, + metadata_validate_duration, + file_open_duration, + mmap_map_duration, + mmap_copy_duration, + direct_read_copy_duration, + mmap_map_fault_delta, + mmap_copy_fault_delta, + direct_read_copy_fault_delta, + blocking_task_duration, + }) }) .await - .map_err(DiskError::from)??; + .map_err(DiskError::from) + .map_err(MmapCopyReadError::Disk) + .and_then(|result| result); + if let Some(metrics) = metrics { + record_mmap_copy_stage(metrics, metrics.blocking_wait_stage, blocking_wait_start); + } + let read_result = match read_result { + Ok(read_result) => read_result, + Err(MmapCopyReadError::Disk(err)) => return Err(err), + Err(MmapCopyReadError::OutOfBounds { actual_size }) => { + error!( + event = EVENT_DISK_LOCAL_READ_VERSION_FALLBACK, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + volume, + path, + offset, + length, + actual_size, + reason = "read_file_mmap_copy_out_of_bounds", + "Disk local read fallback failed" + ); + return Err(DiskError::FileCorrupt); + } + }; + if metrics_enabled && let Some(metrics) = metrics { + rustfs_io_metrics::record_get_object_stage_duration( + metrics.path, + metrics.blocking_task_stage, + read_result.blocking_task_duration.as_secs_f64(), + ); + rustfs_io_metrics::record_get_object_stage_duration( + metrics.path, + metrics.access_check_stage, + read_result.access_check_duration.as_secs_f64(), + ); + rustfs_io_metrics::record_get_object_stage_duration( + metrics.path, + metrics.path_resolve_stage, + read_result.path_resolve_duration.as_secs_f64(), + ); + rustfs_io_metrics::record_get_object_stage_duration( + metrics.path, + metrics.metadata_lookup_stage, + read_result.metadata_lookup_duration.as_secs_f64(), + ); + rustfs_io_metrics::record_get_object_stage_duration( + metrics.path, + metrics.metadata_validate_stage, + read_result.metadata_validate_duration.as_secs_f64(), + ); + rustfs_io_metrics::record_get_object_stage_duration( + metrics.path, + metrics.file_open_stage, + read_result.file_open_duration.as_secs_f64(), + ); + match read_copy_method { + LocalReadCopyMethod::MmapCopy => { + rustfs_io_metrics::record_get_object_stage_duration( + metrics.path, + metrics.mmap_map_stage, + read_result.mmap_map_duration.as_secs_f64(), + ); + rustfs_io_metrics::record_get_object_stage_duration( + metrics.path, + metrics.mmap_copy_stage, + read_result.mmap_copy_duration.as_secs_f64(), + ); + record_mmap_page_fault_delta(metrics.path, metrics.mmap_map_stage, read_result.mmap_map_fault_delta); + record_mmap_page_fault_delta(metrics.path, metrics.mmap_copy_stage, read_result.mmap_copy_fault_delta); + } + LocalReadCopyMethod::DirectReadCopy => { + rustfs_io_metrics::record_get_object_stage_duration( + metrics.path, + metrics.direct_read_copy_stage, + read_result.direct_read_copy_duration.as_secs_f64(), + ); + record_direct_read_page_fault_delta( + metrics.path, + metrics.direct_read_copy_stage, + read_result.direct_read_copy_fault_delta, + ); + } + } + } + let bytes = read_result.bytes; // Log successful mmap read metrics let duration_ms = start.elapsed().as_secs_f64() * 1000.0; @@ -2890,7 +3271,13 @@ impl DiskAPI for LocalDisk { // Record mmap read metrics rustfs_io_metrics::record_zero_copy_read(length, duration_ms); - debug!(size = length, duration_ms = duration_ms, "mmap_read_success"); + debug!( + size = length, + duration_ms = duration_ms, + mmap_populate = should_populate_mmap_read, + read_copy_method = ?read_copy_method, + "mmap_read_success" + ); return Ok(bytes); } @@ -2903,6 +3290,58 @@ impl DiskAPI for LocalDisk { debug!(reason = "non_unix_platform", "zero_copy_fallback"); + let access_check_start = metrics_enabled.then(std::time::Instant::now); + let volume_dir = self.get_bucket_path(volume)?; + if !skip_access_checks(volume) { + access(&volume_dir) + .await + .map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?; + } + if let Some(metrics) = metrics { + record_mmap_copy_stage(metrics, metrics.access_check_stage, access_check_start); + } + + let path_resolve_start = metrics_enabled.then(std::time::Instant::now); + let file_path = self.get_object_path(volume, path)?; + check_path_length(file_path.to_string_lossy().as_ref())?; + if let Some(metrics) = metrics { + record_mmap_copy_stage(metrics, metrics.path_resolve_stage, path_resolve_start); + } + + let file_path_clone = file_path.clone(); + let metadata_lookup_start = metrics_enabled.then(std::time::Instant::now); + let meta_result = tokio::task::spawn_blocking(move || std::fs::metadata(&file_path_clone).map_err(DiskError::from)) + .await + .map_err(DiskError::from) + .and_then(|result| result); + if let Some(metrics) = metrics { + record_mmap_copy_stage(metrics, metrics.metadata_lookup_stage, metadata_lookup_start); + } + let meta = meta_result?; + + let metadata_validate_start = metrics_enabled.then(std::time::Instant::now); + if meta.len() < end_offset as u64 { + if let Some(metrics) = metrics { + record_mmap_copy_stage(metrics, metrics.metadata_validate_stage, metadata_validate_start); + } + error!( + event = EVENT_DISK_LOCAL_READ_VERSION_FALLBACK, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + volume, + path, + offset, + length, + actual_size = meta.len(), + reason = "read_file_mmap_copy_out_of_bounds", + "Disk local read fallback failed" + ); + return Err(DiskError::FileCorrupt); + } + if let Some(metrics) = metrics { + record_mmap_copy_stage(metrics, metrics.metadata_validate_stage, metadata_validate_start); + } + let mut f = self.open_file(file_path, O_RDONLY, volume_dir).await?; if offset > 0 { @@ -5494,6 +5933,88 @@ mod test { }); } + #[test] + fn should_populate_mmap_read_respects_env() { + temp_env::with_var_unset(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, || { + assert!(!should_populate_mmap_read(512 * 1024)); + }); + + temp_env::with_var(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, Some("true"), || { + assert!(should_populate_mmap_read(512 * 1024)); + assert!(!should_populate_mmap_read(0)); + }); + + temp_env::with_var(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, Some("false"), || { + assert!(!should_populate_mmap_read(512 * 1024)); + }); + } + + #[test] + fn local_read_copy_method_respects_env() { + temp_env::with_var_unset(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, || { + assert_eq!(local_read_copy_method(), LocalReadCopyMethod::MmapCopy); + }); + + temp_env::with_var( + ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, + Some(RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY), + || { + assert_eq!(local_read_copy_method(), LocalReadCopyMethod::DirectReadCopy); + }, + ); + + temp_env::with_var(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, Some("unknown"), || { + assert_eq!(local_read_copy_method(), LocalReadCopyMethod::MmapCopy); + }); + } + + #[cfg(unix)] + #[test] + fn mmap_page_size_is_cached_positive() { + let first = mmap_page_size().expect("page size should be available"); + let second = mmap_page_size().expect("cached page size should be available"); + + assert!(first > 0); + assert_eq!(first, second); + } + + #[cfg(unix)] + #[tokio::test] + async fn read_file_mmap_copy_supports_direct_read_copy_method() { + use tempfile::tempdir; + + temp_env::async_with_vars( + [(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, Some(RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY))], + async { + let root_dir = tempdir().expect("operation should succeed"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("operation should succeed"); + let disk = LocalDisk::new(&endpoint, false).await.expect("operation should succeed"); + disk.make_volume("test-volume").await.expect("operation should succeed"); + disk.write_all("test-volume", "test-file.txt", Bytes::from_static(b"0123456789abcdef")) + .await + .expect("operation should succeed"); + + let data = disk + .read_file_mmap_copy("test-volume", "test-file.txt", 4, 6) + .await + .expect("operation should succeed"); + + assert_eq!(data, Bytes::from_static(b"456789")); + }, + ) + .await; + } + + #[cfg(unix)] + #[test] + fn mmap_page_fault_delta_clamps_non_monotonic_counts() { + let before = Some(MmapPageFaultCounts { minor: 10, major: 4 }); + let after = Some(MmapPageFaultCounts { minor: 7, major: 6 }); + + assert_eq!(mmap_page_fault_delta(before, after), MmapPageFaultDelta { minor: 0, major: 2 }); + assert_eq!(mmap_page_fault_delta(before, None), MmapPageFaultDelta::default()); + } + #[test] fn test_is_bitrot_size_mismatch_error_only_matches_target_message() { assert!(is_bitrot_size_mismatch_error(&std::io::Error::other("bitrot shard file size mismatch"))); diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 7dd192ca8..0ecf7b21b 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -59,6 +59,21 @@ pub type DiskStore = Arc; pub type FileReader = Box; pub type FileWriter = Box; +#[derive(Clone, Copy, Debug)] +pub struct MmapCopyStageMetrics { + pub(crate) path: &'static str, + pub(crate) access_check_stage: &'static str, + pub(crate) path_resolve_stage: &'static str, + pub(crate) metadata_lookup_stage: &'static str, + pub(crate) metadata_validate_stage: &'static str, + pub(crate) blocking_wait_stage: &'static str, + pub(crate) blocking_task_stage: &'static str, + pub(crate) file_open_stage: &'static str, + pub(crate) mmap_map_stage: &'static str, + pub(crate) mmap_copy_stage: &'static str, + pub(crate) direct_read_copy_stage: &'static str, +} + #[derive(Debug)] pub enum Disk { Local(Box), @@ -301,6 +316,24 @@ impl DiskAPI for Disk { } } + async fn read_file_mmap_copy_with_metrics( + &self, + volume: &str, + path: &str, + offset: usize, + length: usize, + metrics: Option, + ) -> Result { + match self { + Disk::Local(local_disk) => { + local_disk + .read_file_mmap_copy_with_metrics(volume, path, offset, length, metrics) + .await + } + Disk::Remote(remote_disk) => remote_disk.read_file_mmap_copy(volume, path, offset, length).await, + } + } + #[tracing::instrument(skip(self))] async fn append_file(&self, volume: &str, path: &str) -> Result { match self { @@ -564,6 +597,17 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { /// File read using mmap-then-copy on Unix or an efficient read on non-Unix. async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result; + async fn read_file_mmap_copy_with_metrics( + &self, + volume: &str, + path: &str, + offset: usize, + length: usize, + _metrics: Option, + ) -> Result { + self.read_file_mmap_copy(volume, path, offset, length).await + } + /// Historical name for `read_file_mmap_copy`. #[deprecated( since = "1.0.0-beta.8", diff --git a/crates/ecstore/src/io_support/bitrot.rs b/crates/ecstore/src/io_support/bitrot.rs index 817e3f3b2..b641dc204 100644 --- a/crates/ecstore/src/io_support/bitrot.rs +++ b/crates/ecstore/src/io_support/bitrot.rs @@ -12,7 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, error::DiskError}; +use crate::diagnostics::get::{ + GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, + GET_STAGE_READER_MMAP_COPY_BUFFER, GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, + GET_STAGE_READER_MMAP_MAP, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE, + GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS, + GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled, +}; +use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError}; use crate::erasure::coding::{BitrotReader, BitrotWriterWrapper, CustomWriter}; use bytes::Bytes; use rustfs_config::{DEFAULT_OBJECT_MMAP_READ_ENABLE, ENV_OBJECT_MMAP_READ_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE}; @@ -29,6 +36,14 @@ use tracing::debug; type BoxedObjectReader = Box; type OpenObjectReaderFuture = Pin>> + Send>>; +#[derive(Clone, Copy)] +pub(crate) struct BitrotReaderStageMetrics { + pub(crate) path: &'static str, + pub(crate) reader_construction_stage: &'static str, + pub(crate) file_open_stage: &'static str, + pub(crate) bitrot_reader_init_stage: &'static str, +} + pub(crate) fn object_mmap_read_enabled() -> bool { rustfs_utils::get_env_bool_with_aliases( ENV_OBJECT_MMAP_READ_ENABLE, @@ -46,6 +61,7 @@ struct BitrotReaderSource { offset: usize, length: usize, use_mmap_read: bool, + stage_metrics: Option, } impl BitrotReaderSource { @@ -56,15 +72,67 @@ impl BitrotReaderSource { rd.set_position(offset); Ok(Some(Box::new(rd))) } else if let Some(disk) = self.disk { - open_disk_reader(&disk, &self.bucket, &self.path, self.offset, self.length, self.use_mmap_read) - .await - .map(Some) + open_disk_reader( + &disk, + &self.bucket, + &self.path, + self.offset, + self.length, + self.use_mmap_read, + self.stage_metrics.map(|metrics| metrics.path), + ) + .await + .map(Some) } else { Ok(None) } } } +struct FirstReadMetricsReader { + inner: FileReader, + metrics_path: &'static str, + stage: &'static str, + started_at: Option, + recorded: bool, +} + +impl FirstReadMetricsReader { + fn new(inner: FileReader, metrics_path: &'static str, stage: &'static str) -> Self { + Self { + inner, + metrics_path, + stage, + started_at: None, + recorded: false, + } + } +} + +impl AsyncRead for FirstReadMetricsReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if self.recorded { + return Pin::new(&mut self.inner).poll_read(cx, buf); + } + + let filled_before = buf.filled().len(); + if self.started_at.is_none() { + self.started_at = Some(Instant::now()); + } + + match Pin::new(&mut self.inner).poll_read(cx, buf) { + Poll::Ready(Ok(())) => { + if buf.filled().len() > filled_before { + self.recorded = true; + record_get_stage_duration_if_enabled(self.metrics_path, self.stage, self.started_at.take()); + } + Poll::Ready(Ok(())) + } + other => other, + } + } +} + struct DeferredObjectReader { state: Mutex, } @@ -146,14 +214,38 @@ async fn open_disk_reader( offset: usize, length: usize, use_mmap_read: bool, + metrics_path: Option<&'static str>, ) -> disk::error::Result { + let metrics_path = metrics_path.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled()); + let stage_metrics_enabled = metrics_path.is_some(); + if use_mmap_read && disk.is_local() { - let start = Instant::now(); - match disk.read_file_mmap_copy(bucket, path, offset, length).await { + let start = stage_metrics_enabled.then(Instant::now); + let zero_copy_start = Instant::now(); + let mmap_metrics = metrics_path.map(|metrics_path| MmapCopyStageMetrics { + path: metrics_path, + access_check_stage: GET_STAGE_READER_MMAP_ACCESS_CHECK, + path_resolve_stage: GET_STAGE_READER_MMAP_PATH_RESOLVE, + metadata_lookup_stage: GET_STAGE_READER_MMAP_METADATA_LOOKUP, + metadata_validate_stage: GET_STAGE_READER_MMAP_METADATA_VALIDATE, + blocking_wait_stage: GET_STAGE_READER_MMAP_BLOCKING_WAIT, + blocking_task_stage: GET_STAGE_READER_MMAP_BLOCKING_TASK, + file_open_stage: GET_STAGE_READER_MMAP_FILE_OPEN, + mmap_map_stage: GET_STAGE_READER_MMAP_MAP, + mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER, + direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY, + }); + match disk + .read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics) + .await + { Ok(bytes) => { - let duration_ms = start.elapsed().as_secs_f64() * 1000.0; + let duration_ms = zero_copy_start.elapsed().as_secs_f64() * 1000.0; rustfs_io_metrics::record_zero_copy_read(bytes.len(), duration_ms); + if let Some(metrics_path) = metrics_path { + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS, start); + } debug!( size = bytes.len(), path = %path, @@ -163,6 +255,9 @@ async fn open_disk_reader( return Ok(Box::new(Cursor::new(bytes))); } Err(err) => { + if let Some(metrics_path) = metrics_path { + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, start); + } let reason = format!("{err:?}"); rustfs_io_metrics::record_zero_copy_fallback(&reason); debug!( @@ -171,15 +266,36 @@ async fn open_disk_reader( "zero_copy_fallback" ); - return match disk.read_file_stream(bucket, path, offset, length).await { - Ok(reader) => Ok(reader), + let stream_start = stage_metrics_enabled.then(Instant::now); + let stream_result = disk.read_file_stream(bucket, path, offset, length).await; + if let Some(metrics_path) = metrics_path { + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_STREAM, stream_start); + } + + return match stream_result { + Ok(reader) => Ok(wrap_first_read_metrics(reader, metrics_path)), Err(_) => Err(err), }; } } } - disk.read_file_stream(bucket, path, offset, length).await + let stream_start = stage_metrics_enabled.then(Instant::now); + let reader = disk.read_file_stream(bucket, path, offset, length).await?; + if let Some(metrics_path) = metrics_path { + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_STREAM, stream_start); + } + Ok(wrap_first_read_metrics(reader, metrics_path)) +} + +fn wrap_first_read_metrics(reader: FileReader, metrics_path: Option<&'static str>) -> FileReader { + if let Some(metrics_path) = metrics_path + && rustfs_io_metrics::get_stage_metrics_enabled() + { + return Box::new(FirstReadMetricsReader::new(reader, metrics_path, GET_STAGE_READER_STREAM_FIRST_READ)); + } + + reader } fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksum_algo: HashAlgorithm) -> (usize, usize) { @@ -215,21 +331,131 @@ pub async fn create_bitrot_reader( skip_verify: bool, use_mmap_read: bool, ) -> disk::error::Result>>> { + create_bitrot_reader_with_stage_metrics( + inline_data, + disk, + bucket, + path, + offset, + length, + shard_size, + checksum_algo, + skip_verify, + use_mmap_read, + None, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn create_bitrot_reader_with_stage_metrics( + inline_data: Option<&[u8]>, + disk: Option<&DiskStore>, + bucket: &str, + path: &str, + offset: usize, + length: usize, + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify: bool, + use_mmap_read: bool, + stage_metrics: Option, +) -> disk::error::Result>>> { + create_bitrot_reader_from_bytes_with_stage_metrics( + inline_data.map(Bytes::copy_from_slice), + disk, + bucket, + path, + offset, + length, + shard_size, + checksum_algo, + skip_verify, + use_mmap_read, + stage_metrics, + ) + .await +} + +/// Create a BitrotReader from owned inline Bytes or a disk file stream. +/// +/// Passing `Bytes` preserves the shared inline data buffer and avoids copying +/// shard payloads that are already owned by metadata. +#[allow(clippy::too_many_arguments)] +pub async fn create_bitrot_reader_from_bytes( + inline_data: Option, + disk: Option<&DiskStore>, + bucket: &str, + path: &str, + offset: usize, + length: usize, + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify: bool, + use_mmap_read: bool, +) -> disk::error::Result>>> { + create_bitrot_reader_from_bytes_with_stage_metrics( + inline_data, + disk, + bucket, + path, + offset, + length, + shard_size, + checksum_algo, + skip_verify, + use_mmap_read, + None, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn create_bitrot_reader_from_bytes_with_stage_metrics( + inline_data: Option, + disk: Option<&DiskStore>, + bucket: &str, + path: &str, + offset: usize, + length: usize, + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify: bool, + use_mmap_read: bool, + stage_metrics: Option, +) -> disk::error::Result>>> { + let stage_metrics = stage_metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled()); + let stage_metrics_enabled = stage_metrics.is_some(); + + let reader_construction_start = stage_metrics_enabled.then(Instant::now); let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone()); let source = BitrotReaderSource { - inline_data: inline_data.map(Bytes::copy_from_slice), + inline_data, disk: disk.cloned(), bucket: bucket.to_string(), path: path.to_string(), offset, length, use_mmap_read, + stage_metrics, }; + if let Some(metrics) = stage_metrics { + record_get_stage_duration_if_enabled(metrics.path, metrics.reader_construction_stage, reader_construction_start); + } - source - .open() - .await - .map(|reader| reader.map(|reader| BitrotReader::new(reader, shard_size, checksum_algo, skip_verify))) + let file_open_start = stage_metrics_enabled.then(Instant::now); + let reader = source.open().await?; + if let Some(metrics) = stage_metrics { + record_get_stage_duration_if_enabled(metrics.path, metrics.file_open_stage, file_open_start); + } + + let bitrot_reader_init_start = stage_metrics_enabled.then(Instant::now); + let reader = reader.map(|reader| BitrotReader::new(reader, shard_size, checksum_algo, skip_verify)); + if let Some(metrics) = stage_metrics { + record_get_stage_duration_if_enabled(metrics.path, metrics.bitrot_reader_init_stage, bitrot_reader_init_start); + } + + Ok(reader) } #[allow(clippy::too_many_arguments)] @@ -254,6 +480,7 @@ pub fn create_deferred_bitrot_reader( offset, length, use_mmap_read, + stage_metrics: None, }; BitrotReader::new(Box::new(DeferredObjectReader::new(source)), shard_size, checksum_algo, skip_verify) @@ -390,7 +617,7 @@ mod tests { None, "test-volume", "test-path", - payload.len() as i64, + i64::try_from(payload.len()).expect("test payload length should fit i64"), shard_size, checksum_algo.clone(), ) @@ -425,6 +652,52 @@ mod tests { assert_eq!(&out[..n], b"efgh"); } + #[tokio::test] + async fn test_create_bitrot_reader_from_bytes_preserves_inline_body() { + let shard_size = 4; + let checksum_algo = HashAlgorithm::HighwayHash256S; + let payload = b"abcdefghijkl"; + + let mut writer = create_bitrot_writer( + true, + None, + "test-volume", + "test-path", + payload.len() as i64, + shard_size, + checksum_algo.clone(), + ) + .await + .expect("inline bitrot writer"); + + for chunk in payload.chunks(shard_size) { + writer.write(chunk).await.expect("write chunk"); + } + + let inline_data = Bytes::from(writer.into_inline_data().expect("inline buffer")); + let mut reader = create_bitrot_reader_from_bytes( + Some(inline_data), + None, + "test-bucket", + "test-path", + shard_size, + shard_size, + shard_size, + checksum_algo, + false, + false, + ) + .await + .expect("create reader from bytes") + .expect("reader"); + + let mut out = [0u8; 4]; + let n = reader.read(&mut out).await.expect("read second shard from bytes"); + + assert_eq!(n, shard_size); + assert_eq!(&out[..n], b"efgh"); + } + #[tokio::test] async fn test_deferred_bitrot_reader_opens_inline_source_on_read() { let shard_size = 4; diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index 2d258a4af..c696d12c9 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -277,6 +277,7 @@ impl PutObjReader { pub struct GetObjectReader { pub stream: Box, pub object_info: ObjectInfo, + pub buffered_body: Option, } #[derive(Debug, Clone, Copy)] @@ -529,6 +530,7 @@ impl ReadPlan { GetObjectReader { stream: reader, object_info: oi.clone(), + buffered_body: None, }, self.storage_offset, self.storage_length, @@ -582,6 +584,7 @@ impl ReadPlan { GetObjectReader { stream: final_reader, object_info, + buffered_body: None, }, self.storage_offset, self.storage_length, @@ -682,6 +685,7 @@ impl ReadPlan { GetObjectReader { stream: final_reader, object_info, + buffered_body: None, }, self.storage_offset, self.storage_length, diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index abd0edaf0..87e1f267c 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -31,6 +31,9 @@ pub struct ObjectOptions { pub delete_prefix_object: bool, pub version_id: Option, pub no_lock: bool, + /// True when an upper layer already holds the object read lock before + /// forwarding a no_lock read to the set layer. + pub metadata_cache_safe: bool, pub versioned: bool, pub version_suspended: bool, diff --git a/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs index edbee54a0..cb61d0390 100644 --- a/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs +++ b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs @@ -148,6 +148,7 @@ impl MigrationBackendSpy { GetObjectReader { stream: Box::new(Cursor::new(vec![0_u8; 3])), object_info: ObjectInfo::default(), + buffered_body: None, } } } diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index ef9fda9e1..e3de808a0 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -27,10 +27,12 @@ use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl}; use crate::cluster::rpc::heal_bucket_local_on_disks; use crate::data_usage::record_compression_total_memory; use crate::diagnostics::get::{ - GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, - GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, - GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION, GET_STAGE_EMIT, GET_STAGE_METADATA, classify_storage_error, - record_get_object_pipeline_failure, + GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_CODEC_STREAMING, + GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY, + GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION, + GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE, GET_STAGE_LOCK_ACQUIRE, + GET_STAGE_METADATA, GET_STAGE_OBJECT_INFO, GET_STAGE_PATH_DECISION, GET_STAGE_READER_SETUP, classify_storage_error, + get_stage_timer_if_enabled, record_get_object_pipeline_failure, record_get_stage_duration_if_enabled, }; use crate::disk::error_reduce::{ BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs, @@ -47,7 +49,7 @@ use crate::erasure::codec::bridge::{ use crate::erasure::coding; use crate::error::{Error, Result, is_err_version_not_found}; use crate::error::{GenericError, ObjectApiError, is_err_object_not_found}; -use crate::io_support::bitrot::{create_bitrot_reader, create_bitrot_writer}; +use crate::io_support::bitrot::{create_bitrot_reader, create_bitrot_reader_from_bytes, create_bitrot_writer}; use crate::object_api::ObjectOptions; use crate::runtime::sources as runtime_sources; use crate::services::batch_processor::AsyncBatchProcessor; @@ -157,6 +159,7 @@ type ListObjectsV2Info = StorageListObjectsV2Info; type ListObjectVersionsInfo = StorageListObjectVersionsInfo; type ObjectInfoOrErr = StorageObjectInfoOrErr; type WalkOptions = StorageWalkOptions bool>; +type InlineBitrotReader = coding::BitrotReader>; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_SET_DISK: &str = "set_disk"; @@ -333,7 +336,8 @@ fn adaptive_duplex_buffer_size(object_size: i64) -> usize { const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1); const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750); const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s -const GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096 +const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096 +const ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: &str = "RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES"; // --- Codec Streaming Configuration --- @@ -361,6 +365,11 @@ const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE: bool = false; const ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS: &str = "RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS"; const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS: usize = 256; +const ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY: &str = "RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY"; +const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY: bool = false; +const ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: &str = "RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD"; +const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: usize = 128 * 1024; + // --- Metadata Early-Stop Configuration --- const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE"; @@ -618,6 +627,62 @@ fn get_codec_streaming_min_size() -> usize { rustfs_utils::get_env_usize(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE) } +fn get_object_metadata_cache_max_entries() -> usize { + #[cfg(test)] + { + rustfs_utils::get_env_usize( + ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES, + DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES, + ) + .max(1) + } + #[cfg(not(test))] + { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + rustfs_utils::get_env_usize( + ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES, + DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES, + ) + .max(1) + }) + } +} + +fn is_get_small_object_direct_memory_enabled() -> bool { + #[cfg(test)] + { + rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY, DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY) + } + #[cfg(not(test))] + { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY, DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY) + }) + } +} + +fn get_small_object_direct_memory_threshold() -> usize { + #[cfg(test)] + { + rustfs_utils::get_env_usize( + ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD, + DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD, + ) + } + #[cfg(not(test))] + { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + rustfs_utils::get_env_usize( + ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD, + DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD, + ) + }) + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum GetCodecStreamingEngine { Legacy, @@ -753,7 +818,11 @@ struct GetCodecStreamingGate { decision: GetCodecStreamingDecision, } -fn record_get_codec_streaming_gate_decision(object_class: GetCodecStreamingObjectClass, decision: GetCodecStreamingDecision) { +fn record_get_codec_streaming_gate_decision( + object_class: GetCodecStreamingObjectClass, + decision: GetCodecStreamingDecision, + size_bucket: &'static str, +) { let (outcome, reason) = match decision { GetCodecStreamingDecision::Use => ( crate::diagnostics::get::GET_CODEC_STREAMING_DECISION_USE, @@ -763,7 +832,18 @@ fn record_get_codec_streaming_gate_decision(object_class: GetCodecStreamingObjec (crate::diagnostics::get::GET_CODEC_STREAMING_DECISION_FALLBACK, reason.as_str()) } }; - rustfs_io_metrics::record_get_object_codec_streaming_decision(outcome, object_class.as_str(), reason); + let object_class = object_class.as_str(); + rustfs_io_metrics::record_get_object_codec_streaming_decision(outcome, object_class, reason); + rustfs_io_metrics::record_get_object_codec_streaming_decision_by_size(outcome, object_class, reason, size_bucket); +} + +fn record_get_object_reader_path_observation( + path: &'static str, + object_class: GetCodecStreamingObjectClass, + size_bucket: &'static str, +) { + rustfs_io_metrics::record_get_object_reader_path(path); + rustfs_io_metrics::record_get_object_reader_path_by_size(path, object_class.as_str(), size_bucket); } fn classify_get_codec_streaming_object_class( @@ -789,6 +869,59 @@ fn classify_get_codec_streaming_object_class( GetCodecStreamingObjectClass::PlainSinglePart } +fn is_get_small_object_direct_memory_eligible_with_threshold( + range: &Option, + object_info: &ObjectInfo, + fi: &FileInfo, + opts: &ObjectOptions, + threshold: usize, +) -> bool { + if threshold == 0 + || range.is_some() + || opts.part_number.is_some() + || opts.version_id.is_some() + || opts.versioned + || opts.version_suspended + || opts.incl_free_versions + || opts.skip_free_version + || opts.data_movement + || opts.raw_data_movement_read + || object_info.delete_marker + || object_info.metadata_only + || object_info.version_only + || object_info.is_encrypted() + || object_info.is_compressed() + || object_info.is_remote() + || object_info.parts.len() != 1 + || fi.parts.len() != 1 + || fi.size <= 0 + { + return false; + } + + let Ok(object_size) = usize::try_from(fi.size) else { + return false; + }; + + object_size <= threshold +} + +fn is_get_small_object_direct_memory_eligible( + range: &Option, + object_info: &ObjectInfo, + fi: &FileInfo, + opts: &ObjectOptions, +) -> bool { + is_get_small_object_direct_memory_enabled() + && is_get_small_object_direct_memory_eligible_with_threshold( + range, + object_info, + fi, + opts, + get_small_object_direct_memory_threshold(), + ) +} + fn get_codec_streaming_reader_gate( bucket: &str, object: &str, @@ -1048,7 +1181,7 @@ pub struct SetDisks { pub pool_index: usize, pub format: FormatV3, disk_health_cache: Arc>>>, - get_object_metadata_cache: moka::future::Cache, + get_object_metadata_cache: moka::future::Cache>, pub lockers: Vec>, local_lock_manager: Arc, } @@ -1207,7 +1340,7 @@ impl SetDisks { set_endpoints, disk_health_cache: Arc::new(RwLock::new(Vec::new())), get_object_metadata_cache: moka::future::Cache::builder() - .max_capacity(GET_OBJECT_METADATA_CACHE_MAX_ENTRIES as u64) + .max_capacity(get_object_metadata_cache_max_entries() as u64) .time_to_live(GET_OBJECT_METADATA_CACHE_TTL) .build(), lockers, @@ -1537,6 +1670,189 @@ fn classify_multipart_part_write_path(object_size: i64, block_size: usize) -> Sm } } +#[allow(clippy::too_many_arguments)] +async fn build_inline_bitrot_readers( + files: &[FileInfo], + total_shards: usize, + bucket: &str, + object: &str, + read_length: usize, + shard_size: usize, + checksum_algo: &HashAlgorithm, + skip_verify_bitrot: bool, +) -> disk::error::Result>> { + let mut readers = Vec::with_capacity(total_shards); + for file in files.iter().take(total_shards) { + let reader = if let Some(data) = &file.data { + create_bitrot_reader_from_bytes( + Some(data.clone()), + None, + bucket, + object, + 0, + read_length, + shard_size, + checksum_algo.clone(), + skip_verify_bitrot, + false, + ) + .await? + } else { + None + }; + readers.push(reader); + } + Ok(readers) +} + +#[allow(clippy::too_many_arguments)] +async fn build_inline_bitrot_readers_from_refs( + files: &[&FileInfo], + bucket: &str, + object: &str, + read_length: usize, + shard_size: usize, + checksum_algo: &HashAlgorithm, + skip_verify_bitrot: bool, +) -> disk::error::Result>> { + let mut readers = Vec::with_capacity(files.len()); + for file in files { + let reader = if let Some(data) = &file.data { + create_bitrot_reader_from_bytes( + Some(data.clone()), + None, + bucket, + object, + 0, + read_length, + shard_size, + checksum_algo.clone(), + skip_verify_bitrot, + false, + ) + .await? + } else { + None + }; + readers.push(reader); + } + Ok(readers) +} + +async fn try_read_inline_data_shards_direct( + readers: &mut [Option], + data_shards: usize, + read_length: usize, + object_size: usize, +) -> Option { + if object_size == 0 || read_length == 0 || readers.len() < data_shards { + return None; + } + + let mut body = Vec::with_capacity(object_size); + let mut remaining = object_size; + for reader in readers.iter_mut().take(data_shards) { + let reader = reader.as_mut()?; + let mut shard = vec![0u8; read_length]; + let Ok(read) = reader.read(&mut shard).await else { + return None; + }; + if read != read_length { + return None; + } + + let take = remaining.min(shard.len()); + body.extend_from_slice(&shard[..take]); + remaining -= take; + if remaining == 0 { + return Some(Bytes::from(body)); + } + } + + None +} + +fn can_try_inline_data_shards_direct(object_size: usize, block_size: usize) -> bool { + object_size > 0 && object_size <= block_size +} + +fn inline_erasure_shard_size(block_size: usize, data_shards: usize, uses_legacy: bool) -> usize { + if block_size == 0 || data_shards == 0 { + return 0; + } + if uses_legacy { + coding::calc_shard_size_legacy(block_size, data_shards) + } else { + coding::calc_shard_size(block_size, data_shards) + } +} + +fn inline_erasure_shard_file_size(total_length: usize, block_size: usize, data_shards: usize, uses_legacy: bool) -> usize { + if total_length == 0 || block_size == 0 || data_shards == 0 { + return 0; + } + + let shard_size = inline_erasure_shard_size(block_size, data_shards, uses_legacy); + let shard_size_fn = if uses_legacy { + coding::calc_shard_size_legacy + } else { + coding::calc_shard_size + }; + let num_shards = total_length / block_size; + let last_block_size = total_length % block_size; + let last_shard_size = shard_size_fn(last_block_size, data_shards); + num_shards * shard_size + last_shard_size +} + +fn inline_erasure_shard_file_offset( + start_offset: usize, + length: usize, + total_length: usize, + block_size: usize, + data_shards: usize, + uses_legacy: bool, +) -> usize { + if block_size == 0 || data_shards == 0 { + return 0; + } + + let shard_size = inline_erasure_shard_size(block_size, data_shards, uses_legacy); + let shard_file_size = inline_erasure_shard_file_size(total_length, block_size, data_shards, uses_legacy); + let end_shard = (start_offset + length) / block_size; + let till_offset = end_shard * shard_size + shard_size; + till_offset.min(shard_file_size) +} + +fn collect_inline_data_shard_fileinfos_by_index<'a>( + parts_metadata: &'a [FileInfo], + fi: &FileInfo, + data_shards: usize, + mut disk_is_online: impl FnMut(usize) -> bool, +) -> Option> { + let distribution = &fi.erasure.distribution; + let mut data_files = vec![None; data_shards]; + + for (disk_index, file_info) in parts_metadata.iter().enumerate() { + if !disk_is_online(disk_index) { + continue; + } + let block_index = *distribution.get(disk_index)?; + if block_index == 0 || block_index > data_shards { + continue; + } + if !file_info.is_valid() { + continue; + } + if file_info.data.as_ref().is_none_or(|data| data.is_empty()) { + continue; + } + + data_files[block_index - 1] = Some(file_info); + } + + data_files.into_iter().collect() +} + #[async_trait::async_trait] impl crate::storage_api_contracts::object::ObjectIO for SetDisks { type Error = Error; @@ -1556,6 +1872,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { h: HeaderMap, opts: &ObjectOptions, ) -> Result { + let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); // Check if lock optimization is enabled // When enabled, read locks are released after metadata read let lock_optimization_enabled = is_lock_optimization_enabled(); @@ -1563,6 +1880,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { // Acquire a shared read-lock early to protect read consistency let read_lock_guard = if !opts.no_lock { let acquire_start = Instant::now(); + let lock_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); // Record lock wait for deadlock detection if is_deadlock_detection_enabled() { @@ -1582,6 +1900,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { // Record lock statistics metrics::counter!("rustfs.lock.acquire.total", "type" => "read").increment(1); metrics::histogram!("rustfs.lock.acquire.duration.seconds").record(acquire_start.elapsed().as_secs_f64()); + record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_LOCK_ACQUIRE, lock_stage_start); Some(guard) } else { @@ -1590,17 +1909,27 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let metadata_stage_start = Instant::now(); let (fi, files, disks) = match self.get_object_fileinfo(bucket, object, opts, true).await { - Ok(result) => { - rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_stage_start.elapsed().as_secs_f64()); - result - } + Ok(result) => result, Err(err) => { rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_stage_start.elapsed().as_secs_f64()); record_get_object_pipeline_failure(GET_STAGE_METADATA, classify_storage_error(&err)); return Err(to_object_err(err, vec![bucket, object])); } }; + let object_info_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); let object_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended); + let object_class = classify_get_codec_streaming_object_class(&range, &object_info, &fi); + let size_bucket = rustfs_io_metrics::get_object_size_bucket(object_info.size); + record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_OBJECT_INFO, object_info_stage_start); + let metadata_elapsed = metadata_stage_start.elapsed().as_secs_f64(); + rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_elapsed); + rustfs_io_metrics::record_get_object_stage_duration_by_size( + GET_OBJECT_PATH_SET_DISK, + GET_STAGE_METADATA, + object_class.as_str(), + size_bucket, + metadata_elapsed, + ); if object_info.delete_marker { if opts.version_id.is_none() { @@ -1619,7 +1948,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { // } if object_info.size == 0 { - rustfs_io_metrics::record_get_object_reader_path(GET_OBJECT_PATH_EMPTY); + record_get_object_reader_path_observation(GET_OBJECT_PATH_EMPTY, object_class, size_bucket); // if let Some(rs) = range { // let _ = rs.get_offset_length(object_info.size)?; // } @@ -1627,6 +1956,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let reader = GetObjectReader { stream: Box::new(Cursor::new(Vec::new())), object_info, + buffered_body: Some(Bytes::new()), }; return Ok(reader); } @@ -1635,7 +1965,96 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { // Uses the shared predicate from ObjectInfo; additionally checks that // inline data is actually present and no range request is in flight. if object_info.is_inline_fast_path_eligible() && fi.data.is_some() && range.is_none() { + let mut inline_prepare_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); let data_shards = fi.erasure.data_blocks; + + let object_size = usize::try_from(fi.size) + .map_err(|_| to_object_err(Error::other("inline fast path object size is invalid"), vec![bucket, object]))?; + + let checksum_info = fi.erasure.get_checksum_info(fi.parts[0].number); + let checksum_algo = + if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S { + rustfs_utils::HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; + + if can_try_inline_data_shards_direct(object_size, fi.erasure.block_size) + && let Some(data_files) = collect_inline_data_shard_fileinfos_by_index(&files, &fi, data_shards, |index| { + disks.get(index).is_some_and(Option::is_some) + }) + { + let read_length = inline_erasure_shard_file_offset( + 0, + object_size, + object_size, + fi.erasure.block_size, + data_shards, + fi.uses_legacy_checksum, + ); + let shard_size = inline_erasure_shard_size(fi.erasure.block_size, data_shards, fi.uses_legacy_checksum); + if let Some(inline_prepare_stage_start) = inline_prepare_stage_start.take() { + rustfs_io_metrics::record_get_object_stage_duration_by_size( + GET_OBJECT_PATH_INLINE_DIRECT, + GET_STAGE_INLINE_PREPARE, + object_class.as_str(), + size_bucket, + inline_prepare_stage_start.elapsed().as_secs_f64(), + ); + } + let reader_setup_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now); + let mut readers = build_inline_bitrot_readers_from_refs( + &data_files, + bucket, + object, + read_length, + shard_size, + &checksum_algo, + opts.skip_verify_bitrot, + ) + .await?; + if let Some(reader_setup_stage_start) = reader_setup_stage_start { + rustfs_io_metrics::record_get_object_stage_duration_by_size( + GET_OBJECT_PATH_INLINE_DIRECT, + GET_STAGE_READER_SETUP, + object_class.as_str(), + size_bucket, + reader_setup_stage_start.elapsed().as_secs_f64(), + ); + } + + // Decode directly + let decode_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now); + if let Some(body) = try_read_inline_data_shards_direct(&mut readers, data_shards, read_length, object_size).await + { + if let Some(decode_stage_start) = decode_stage_start { + rustfs_io_metrics::record_get_object_stage_duration_by_size( + GET_OBJECT_PATH_INLINE_DIRECT, + GET_STAGE_DECODE, + object_class.as_str(), + size_bucket, + decode_stage_start.elapsed().as_secs_f64(), + ); + } + + record_get_object_reader_path_observation(GET_OBJECT_PATH_INLINE_DIRECT, object_class, size_bucket); + let reader = GetObjectReader { + stream: Box::new(Cursor::new(body.clone())), + object_info, + buffered_body: Some(body), + }; + return Ok(reader); + } + } + + let erasure = coding::Erasure::new_with_options( + fi.erasure.data_blocks, + fi.erasure.parity_blocks, + fi.erasure.block_size, + fi.uses_legacy_checksum, + ); + let read_length = erasure.shard_file_offset(0, object_size, object_size); + let total_shards = data_shards + fi.erasure.parity_blocks; let (_disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(&disks, &files, &fi); // Check if we have enough inline data shards @@ -1646,52 +2065,40 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { .count(); if inline_count >= data_shards { - // All data shards are inline - decode in memory - let erasure = coding::Erasure::new_with_options( - fi.erasure.data_blocks, - fi.erasure.parity_blocks, - fi.erasure.block_size, - fi.uses_legacy_checksum, - ); - let object_size = usize::try_from(fi.size) - .map_err(|_| to_object_err(Error::other("inline fast path object size is invalid"), vec![bucket, object]))?; - - let checksum_info = fi.erasure.get_checksum_info(fi.parts[0].number); - let checksum_algo = - if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S { - rustfs_utils::HashAlgorithm::HighwayHash256SLegacy - } else { - checksum_info.algorithm - }; - let read_length = erasure.shard_file_offset(0, object_size, object_size); - let mut readers: Vec>>> = - Vec::new(); - for file in files.iter().take(data_shards + fi.erasure.parity_blocks) { - if let Some(data) = &file.data { - readers.push( - create_bitrot_reader( - Some(data), - None, - bucket, - object, - 0, - read_length, - erasure.shard_size(), - checksum_algo.clone(), - opts.skip_verify_bitrot, - false, - ) - .await?, - ); - } else { - readers.push(None); - } + if let Some(inline_prepare_stage_start) = inline_prepare_stage_start.take() { + rustfs_io_metrics::record_get_object_stage_duration_by_size( + GET_OBJECT_PATH_INLINE_DIRECT, + GET_STAGE_INLINE_PREPARE, + object_class.as_str(), + size_bucket, + inline_prepare_stage_start.elapsed().as_secs_f64(), + ); + } + let reader_setup_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now); + let readers = build_inline_bitrot_readers( + &files, + total_shards, + bucket, + object, + read_length, + erasure.shard_size(), + &checksum_algo, + opts.skip_verify_bitrot, + ) + .await?; + if let Some(reader_setup_stage_start) = reader_setup_stage_start { + rustfs_io_metrics::record_get_object_stage_duration_by_size( + GET_OBJECT_PATH_INLINE_DIRECT, + GET_STAGE_READER_SETUP, + object_class.as_str(), + size_bucket, + reader_setup_stage_start.elapsed().as_secs_f64(), + ); } - // Decode directly + let decode_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now); let mut output = Cursor::new(Vec::with_capacity(object_size)); let (written, err) = erasure.decode(&mut output, readers, 0, object_size, object_size).await; - if let Some(e) = err { return Err(to_object_err(e.into(), vec![bucket, object])); } @@ -1701,25 +2108,42 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { vec![bucket, object], )); } + let body = Bytes::from(output.into_inner()); + if let Some(decode_stage_start) = decode_stage_start { + rustfs_io_metrics::record_get_object_stage_duration_by_size( + GET_OBJECT_PATH_INLINE_DIRECT, + GET_STAGE_DECODE, + object_class.as_str(), + size_bucket, + decode_stage_start.elapsed().as_secs_f64(), + ); + } - rustfs_io_metrics::record_get_object_reader_path(GET_OBJECT_PATH_INLINE_DIRECT); + record_get_object_reader_path_observation(GET_OBJECT_PATH_INLINE_DIRECT, object_class, size_bucket); let reader = GetObjectReader { - stream: Box::new(Cursor::new(output.into_inner())), + stream: Box::new(Cursor::new(body.clone())), object_info, + buffered_body: Some(body), }; return Ok(reader); } } + let path_decision_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); let codec_streaming_gate = get_codec_streaming_reader_gate(bucket, object, &range, &object_info, &fi, lock_optimization_enabled); + record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_PATH_DECISION, path_decision_stage_start); if object_info.is_remote() { if let GetCodecStreamingDecision::Fallback(reason) = codec_streaming_gate.decision { - record_get_codec_streaming_gate_decision(codec_streaming_gate.object_class, codec_streaming_gate.decision); + record_get_codec_streaming_gate_decision( + codec_streaming_gate.object_class, + codec_streaming_gate.decision, + size_bucket, + ); rustfs_io_metrics::record_get_object_codec_streaming_fallback(reason.as_str()); } - rustfs_io_metrics::record_get_object_reader_path(GET_OBJECT_PATH_REMOTE_TRANSITION); + record_get_object_reader_path_observation(GET_OBJECT_PATH_REMOTE_TRANSITION, object_class, size_bucket); let mut opts = opts.clone(); if object_info.parts.len() == 1 { opts.part_number = Some(1); @@ -1747,6 +2171,74 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { read_lock_guard }; + if is_get_small_object_direct_memory_eligible(&range, &object_info, &fi, opts) { + let object_size = usize::try_from(object_info.size) + .map_err(|_| to_object_err(Error::other("direct-memory GET object size is invalid"), vec![bucket, object]))?; + if let Some(body) = Self::try_get_object_direct_data_shards_with_fileinfo( + bucket, + object, + &fi, + &files, + &disks, + opts.skip_verify_bitrot, + object_class.as_str(), + size_bucket, + ) + .await? + { + if body.len() != object_size { + return Err(to_object_err( + Error::other("direct-memory GET decoded length mismatch"), + vec![bucket, object], + )); + } + + record_get_object_reader_path_observation(GET_OBJECT_PATH_DIRECT_MEMORY, object_class, size_bucket); + let reader = GetObjectReader { + stream: Box::new(Cursor::new(body.clone())), + object_info, + buffered_body: Some(body), + }; + return Ok(reader); + } + + let mut output = Vec::with_capacity(object_size); + Self::get_object_with_fileinfo( + bucket, + object, + 0, + object_info.size, + &mut output, + fi, + files, + &disks, + self.set_index, + self.pool_index, + opts.skip_verify_bitrot, + true, + GET_OBJECT_PATH_DIRECT_MEMORY, + object_class.as_str(), + size_bucket, + ) + .await?; + + if output.len() != object_size { + return Err(to_object_err( + Error::other("direct-memory GET decoded length mismatch"), + vec![bucket, object], + )); + } + + record_get_object_reader_path_observation(GET_OBJECT_PATH_DIRECT_MEMORY, object_class, size_bucket); + let body = Bytes::from(output); + let reader = GetObjectReader { + stream: Box::new(Cursor::new(body.clone())), + object_info, + buffered_body: Some(body), + }; + return Ok(reader); + } + match codec_streaming_gate.decision { GetCodecStreamingDecision::Use => { match Self::get_object_decode_reader_with_fileinfo( @@ -1765,8 +2257,9 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { record_get_codec_streaming_gate_decision( codec_streaming_gate.object_class, GetCodecStreamingDecision::Use, + size_bucket, ); - rustfs_io_metrics::record_get_object_reader_path(GET_OBJECT_PATH_CODEC_STREAMING); + record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket); let (reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?; return Ok(reader); } @@ -1774,18 +2267,23 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { record_get_codec_streaming_gate_decision( codec_streaming_gate.object_class, GetCodecStreamingDecision::Fallback(reason), + size_bucket, ); rustfs_io_metrics::record_get_object_codec_streaming_fallback(reason.as_str()); } } } GetCodecStreamingDecision::Fallback(reason) => { - record_get_codec_streaming_gate_decision(codec_streaming_gate.object_class, codec_streaming_gate.decision); + record_get_codec_streaming_gate_decision( + codec_streaming_gate.object_class, + codec_streaming_gate.decision, + size_bucket, + ); rustfs_io_metrics::record_get_object_codec_streaming_fallback(reason.as_str()); } } - rustfs_io_metrics::record_get_object_reader_path(GET_OBJECT_PATH_LEGACY_DUPLEX); + record_get_object_reader_path_observation(GET_OBJECT_PATH_LEGACY_DUPLEX, object_class, size_bucket); let duplex_buffer_size = adaptive_duplex_buffer_size(object_info.size); let (rd, wd) = tokio::io::duplex(duplex_buffer_size); @@ -1821,6 +2319,10 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { set_index, pool_index, skip_verify, + false, + GET_OBJECT_PATH_LEGACY_DUPLEX, + object_class.as_str(), + size_bucket, ) .await { @@ -3716,6 +4218,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let reader = ReaderImpl::ObjectBody(GetObjectReader { stream: Box::new(pr), object_info: oi, + buffered_body: None, }); let cloned_bucket = bucket.to_string(); @@ -3724,6 +4227,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let set_index = self.set_index; let pool_index = self.pool_index; let skip_verify = opts.skip_verify_bitrot; + let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(cloned_fi.size); tokio::spawn(async move { if let Err(e) = Self::get_object_with_fileinfo( &cloned_bucket, @@ -3737,6 +4241,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { set_index, pool_index, skip_verify, + false, + GET_OBJECT_PATH_LEGACY_DUPLEX, + GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, + metrics_size_bucket, ) .await { @@ -8461,6 +8969,396 @@ mod tests { assert_eq!(complete_part_checksum(&part, full_object_crc32), Some(Some("AAAAAA==".to_string()))); } + fn direct_memory_test_metadata(size: i64) -> (ObjectInfo, FileInfo, ObjectOptions) { + let part_size = usize::try_from(size).expect("test size should fit usize"); + let part = ObjectPartInfo { + number: 1, + size: part_size, + actual_size: size, + ..Default::default() + }; + let object_info = ObjectInfo { + size, + actual_size: size, + parts: Arc::new(vec![part]), + etag: Some("0123456789abcdef0123456789abcdef".to_string()), + ..Default::default() + }; + let mut fi = FileInfo::new("bucket/object", 1, 0); + fi.size = size; + fi.add_object_part(1, String::new(), part_size, None, size, None, None); + (object_info, fi, ObjectOptions::default()) + } + + #[test] + fn small_object_direct_memory_eligibility_is_conservative() { + let (object_info, fi, opts) = direct_memory_test_metadata(1024); + assert!(is_get_small_object_direct_memory_eligible_with_threshold( + &None, + &object_info, + &fi, + &opts, + 128 * 1024 + )); + + assert!(!is_get_small_object_direct_memory_eligible_with_threshold( + &Some(HTTPRangeSpec { + start: 0, + end: 10, + is_suffix_length: false, + }), + &object_info, + &fi, + &opts, + 128 * 1024 + )); + + let mut part_opts = opts.clone(); + part_opts.part_number = Some(1); + assert!(!is_get_small_object_direct_memory_eligible_with_threshold( + &None, + &object_info, + &fi, + &part_opts, + 128 * 1024 + )); + + let mut versioned_opts = opts.clone(); + versioned_opts.versioned = true; + assert!(!is_get_small_object_direct_memory_eligible_with_threshold( + &None, + &object_info, + &fi, + &versioned_opts, + 128 * 1024 + )); + + let mut remote = object_info; + remote.transitioned_object.status = TRANSITION_COMPLETE.to_string(); + remote.transitioned_object.tier = "remote-tier".to_string(); + assert!(!is_get_small_object_direct_memory_eligible_with_threshold( + &None, + &remote, + &fi, + &opts, + 128 * 1024 + )); + } + + #[test] + fn small_object_direct_memory_eligibility_respects_threshold_and_shape() { + let (object_info, fi, opts) = direct_memory_test_metadata(128 * 1024); + assert!(is_get_small_object_direct_memory_eligible_with_threshold( + &None, + &object_info, + &fi, + &opts, + 128 * 1024 + )); + assert!(!is_get_small_object_direct_memory_eligible_with_threshold( + &None, + &object_info, + &fi, + &opts, + (128 * 1024) - 1 + )); + + let mut multipart = object_info; + multipart.parts = Arc::new(vec![ObjectPartInfo::default(), ObjectPartInfo::default()]); + assert!(!is_get_small_object_direct_memory_eligible_with_threshold( + &None, + &multipart, + &fi, + &opts, + 128 * 1024 + )); + } + + async fn inline_bitrot_files_for_payload(payload: &[u8]) -> (coding::Erasure, Vec, usize, HashAlgorithm) { + let erasure = coding::Erasure::new(4, 2, 1024 * 1024); + let read_length = erasure.shard_file_offset(0, payload.len(), payload.len()); + let checksum_algo = HashAlgorithm::HighwayHash256S; + let shards = erasure.encode_data(payload).expect("payload should encode"); + let mut files = Vec::with_capacity(shards.len()); + + for shard in shards { + let mut writer = coding::BitrotWriterWrapper::new( + coding::CustomWriter::new_inline_buffer(), + erasure.shard_size(), + checksum_algo.clone(), + ); + writer.write(&shard).await.expect("inline shard should write"); + writer.shutdown().await.expect("inline writer should shutdown"); + let data = writer.into_inline_data().expect("inline data should be retained"); + let mut file = FileInfo::new("bucket/object", erasure.data_shards, erasure.parity_shards); + file.erasure.index = files.len() + 1; + file.data = Some(Bytes::from(data)); + files.push(file); + } + + (erasure, files, read_length, checksum_algo) + } + + fn inline_data_shard_fileinfo( + name: &str, + data_blocks: usize, + parity_blocks: usize, + erasure_index: usize, + distribution: &[usize], + data: Option<&'static [u8]>, + ) -> FileInfo { + let mut fi = FileInfo::new(name, data_blocks, parity_blocks); + fi.name = name.to_string(); + fi.erasure.index = erasure_index; + fi.erasure.distribution = distribution.to_vec(); + fi.data = data.map(Bytes::from_static); + fi + } + + #[test] + fn collect_inline_data_shards_by_index_uses_distribution_order() { + let distribution = vec![3, 1, 5, 2, 4, 6]; + let mut fi = FileInfo::new("object", 4, 2); + fi.erasure.distribution = distribution.clone(); + let files = vec![ + inline_data_shard_fileinfo("block-3", 4, 2, 3, &distribution, Some(b"c")), + inline_data_shard_fileinfo("block-1", 4, 2, 1, &distribution, Some(b"a")), + inline_data_shard_fileinfo("parity-5", 4, 2, 5, &distribution, Some(b"p")), + inline_data_shard_fileinfo("block-2", 4, 2, 2, &distribution, Some(b"b")), + inline_data_shard_fileinfo("block-4", 4, 2, 4, &distribution, Some(b"d")), + inline_data_shard_fileinfo("parity-6", 4, 2, 6, &distribution, Some(b"q")), + ]; + + let data_files = + collect_inline_data_shard_fileinfos_by_index(&files, &fi, 4, |_| true).expect("all data shards should be collected"); + + assert_eq!( + data_files.iter().map(|file| file.name.as_str()).collect::>(), + ["block-1", "block-2", "block-3", "block-4"] + ); + } + + #[test] + fn collect_inline_data_shards_by_index_rejects_missing_data_shard() { + let distribution = vec![1, 2, 3, 4]; + let mut fi = FileInfo::new("object", 2, 2); + fi.erasure.distribution = distribution.clone(); + let files = vec![ + inline_data_shard_fileinfo("block-1", 2, 2, 1, &distribution, Some(b"a")), + inline_data_shard_fileinfo("block-2", 2, 2, 2, &distribution, None), + inline_data_shard_fileinfo("parity-3", 2, 2, 3, &distribution, Some(b"p")), + inline_data_shard_fileinfo("parity-4", 2, 2, 4, &distribution, Some(b"q")), + ]; + + assert!(collect_inline_data_shard_fileinfos_by_index(&files, &fi, 2, |_| true).is_none()); + } + + #[tokio::test] + async fn inline_data_shards_direct_read_reassembles_payload() { + let payload = b"small inline object payload that spans data shards"; + let (erasure, files, read_length, checksum_algo) = inline_bitrot_files_for_payload(payload).await; + let mut readers = build_inline_bitrot_readers( + &files, + erasure.data_shards, + "bucket", + "object", + read_length, + erasure.shard_size(), + &checksum_algo, + false, + ) + .await + .expect("inline bitrot readers should build"); + assert_eq!(readers.len(), erasure.data_shards); + + let body = try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, payload.len()) + .await + .expect("data shard direct read should succeed"); + + assert_eq!(body.as_ref(), payload); + } + + #[tokio::test] + async fn inline_data_shards_direct_read_rejects_corrupt_shard() { + let payload = b"small inline object payload that will be corrupted"; + let (erasure, mut files, read_length, checksum_algo) = inline_bitrot_files_for_payload(payload).await; + let first = files[0].data.as_mut().expect("first shard should exist"); + let mut corrupted = first.to_vec(); + let last = corrupted.last_mut().expect("encoded shard should not be empty"); + *last ^= 0xff; + *first = Bytes::from(corrupted); + + let mut readers = build_inline_bitrot_readers( + &files, + erasure.total_shard_count(), + "bucket", + "object", + read_length, + erasure.shard_size(), + &checksum_algo, + false, + ) + .await + .expect("inline bitrot readers should build"); + + let body = try_read_inline_data_shards_direct(&mut readers, 4, read_length, payload.len()).await; + + assert!(body.is_none()); + } + + #[test] + fn inline_data_shards_direct_read_requires_single_block() { + assert!(can_try_inline_data_shards_direct(1024, 1024)); + assert!(!can_try_inline_data_shards_direct(0, 1024)); + assert!(!can_try_inline_data_shards_direct(1025, 1024)); + } + + #[test] + fn inline_erasure_offset_helpers_match_erasure_methods() { + for uses_legacy in [false, true] { + let erasure = coding::Erasure::new_with_options(4, 2, 1024 * 1024, uses_legacy); + for object_size in [1usize, 1024, 100 * 1024, 1024 * 1024] { + assert_eq!( + inline_erasure_shard_size(erasure.block_size, erasure.data_shards, uses_legacy), + erasure.shard_size() + ); + assert_eq!( + inline_erasure_shard_file_offset( + 0, + object_size, + object_size, + erasure.block_size, + erasure.data_shards, + uses_legacy, + ), + erasure.shard_file_offset(0, object_size, object_size) + ); + } + } + } + + #[tokio::test] + async fn direct_memory_inline_data_shards_direct_read_reassembles_single_block_payload() { + let tempdir = tempfile::tempdir().expect("tempdir should be created"); + let endpoint = + Endpoint::try_from(tempdir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("disk should be created"); + + let payload = vec![b'i'; 192 * 1024]; + let (erasure, files, _read_length, _checksum_algo) = inline_bitrot_files_for_payload(&payload).await; + let mut fi = FileInfo::new("bucket/object", erasure.data_shards, erasure.parity_shards); + fi.size = payload.len() as i64; + fi.data = files[0].data.clone(); + fi.add_object_part(1, String::new(), payload.len(), None, payload.len() as i64, None, None); + + let disks = vec![Some(disk); erasure.total_shard_count()]; + let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(fi.size); + + let body = SetDisks::try_get_object_direct_data_shards_with_fileinfo( + "bucket", + "object", + &fi, + &files, + &disks, + true, + GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, + metrics_size_bucket, + ) + .await + .expect("direct-memory inline data shard read should not fail") + .expect("inline data shard path should be used"); + + assert_eq!(body.as_ref(), payload); + } + + #[tokio::test] + async fn direct_memory_data_shards_direct_read_reassembles_single_block_payload() { + use uuid::Uuid; + + let tempdir = tempfile::tempdir().expect("tempdir should be created"); + let endpoint = + Endpoint::try_from(tempdir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("disk should be created"); + + let bucket = "bucket"; + let object = "object"; + let payload = vec![b'd'; 192 * 1024]; + + disk.make_volume(bucket).await.expect("bucket should be created"); + + let mut fi = FileInfo::new(&format!("{bucket}/{object}"), 1, 0); + let data_dir = Uuid::new_v4(); + fi.data_dir = Some(data_dir); + fi.size = payload.len() as i64; + fi.add_object_part(1, String::new(), payload.len(), None, payload.len() as i64, None, None); + + let erasure = coding::Erasure::new_with_options( + fi.erasure.data_blocks, + fi.erasure.parity_blocks, + fi.erasure.block_size, + fi.uses_legacy_checksum, + ); + let shard_path = format!("{object}/{data_dir}/part.1"); + let checksum_info = fi.erasure.get_checksum_info(1); + + let mut bitrot_writer = create_bitrot_writer( + true, + None, + bucket, + &shard_path, + payload.len() as i64, + erasure.shard_size(), + checksum_info.algorithm.clone(), + ) + .await + .expect("bitrot writer should be created"); + + for chunk in payload.chunks(erasure.shard_size()) { + bitrot_writer.write(chunk).await.expect("payload chunk should be written"); + } + + let encoded = bitrot_writer.into_inline_data().expect("bitrot encoded data should exist"); + disk.write_all(bucket, &shard_path, Bytes::from(encoded)) + .await + .expect("encoded shard should be stored"); + + let files = vec![fi.clone()]; + let disks = vec![Some(disk)]; + let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(fi.size); + + let body = SetDisks::try_get_object_direct_data_shards_with_fileinfo( + bucket, + object, + &fi, + &files, + &disks, + true, + GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, + metrics_size_bucket, + ) + .await + .expect("direct-memory data shard read should not fail") + .expect("single-block data shard path should be used"); + + assert_eq!(body.as_ref(), payload); + } + #[tokio::test] async fn range_reads_use_shard_span_length_for_non_zero_offsets() { use tokio::io::AsyncReadExt; @@ -8526,6 +9424,7 @@ mod tests { let files = vec![fi.clone()]; let disks = vec![Some(disk.clone())]; let (mut reader, mut writer) = tokio::io::duplex(range_length * 2); + let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(fi.size); let read_task = tokio::spawn(async move { SetDisks::get_object_with_fileinfo( @@ -8540,6 +9439,10 @@ mod tests { 0, 0, true, + false, + GET_OBJECT_PATH_LEGACY_DUPLEX, + GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, + metrics_size_bucket, ) .await }); diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 98c3c6042..73cdc8620 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -14,6 +14,14 @@ use super::*; use crate::diagnostics::get::{ + GET_DIRECT_MEMORY_SUBPATH_DISK_DATA_BLOCKS, GET_DIRECT_MEMORY_SUBPATH_INLINE_BUFFERED, GET_METADATA_CACHE_DECISION_HIT, + GET_METADATA_CACHE_DECISION_MISS, GET_METADATA_CACHE_DECISION_REJECT, GET_METADATA_CACHE_DECISION_SKIP, + GET_METADATA_CACHE_REASON_DATA_MOVEMENT, GET_METADATA_CACHE_REASON_DELETE_MARKER, GET_METADATA_CACHE_REASON_DIST_ERASURE, + GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM, + GET_METADATA_CACHE_REASON_META_BUCKET, GET_METADATA_CACHE_REASON_NO_LOCK, GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, + GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER, + GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, + GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, @@ -21,12 +29,17 @@ use crate::diagnostics::get::{ GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, - GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_DECODE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GetObjectFailureReason, - classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, + GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, + GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, + GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, + GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, + GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, }; use crate::erasure::coding::BitrotReader; -use crate::io_support::bitrot::{create_deferred_bitrot_reader, object_mmap_read_enabled}; +use crate::io_support::bitrot::{ + BitrotReaderStageMetrics, create_bitrot_reader_with_stage_metrics, create_deferred_bitrot_reader, object_mmap_read_enabled, +}; use crate::set_disk::shard_source::ShardReadCost; use futures::stream::{FuturesUnordered, StreamExt}; use metrics::counter; @@ -43,6 +56,9 @@ use tokio::sync::RwLock; use tokio::task::JoinSet; const EVENT_SET_DISK_READ: &str = "set_disk_read"; +const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP"; +const ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP: &str = + "RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP"; const SLOW_OBJECT_READ_LOG_THRESHOLD: Duration = Duration::from_secs(5); const READ_REPAIR_HEAL_DEDUP_TTL: Duration = Duration::from_secs(60); const READ_REPAIR_HEAL_DEDUP_MAX_ENTRIES: usize = 4096; @@ -426,6 +442,13 @@ impl MetadataQuorumAccumulator { } } +#[derive(Clone, Debug)] +enum MetadataCacheLookup { + Hit(Arc), + Miss, + RejectedInsufficientQuorum, +} + fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool { left.volume == right.volume && left.name == right.name @@ -797,12 +820,27 @@ async fn submit_read_repair_heal_with_submitter( } type ObjectBitrotReader = BitrotReader>; +type BitrotReaderTask<'a> = + Pin, DiskError>)> + Send + 'a>>; + +const DIRECT_MEMORY_BITROT_READER_STAGE_METRICS: BitrotReaderStageMetrics = BitrotReaderStageMetrics { + path: GET_OBJECT_PATH_DIRECT_MEMORY, + reader_construction_stage: GET_STAGE_READER_TASK_READER_CONSTRUCTION, + file_open_stage: GET_STAGE_READER_TASK_FILE_OPEN, + bitrot_reader_init_stage: GET_STAGE_READER_TASK_BITROT_READER_INIT, +}; struct BitrotReaderSetup { readers: Vec>, errors: Vec>, + scheduled: Vec, attempted: Vec, ready: Vec, + scheduled_count: usize, + attempted_count: usize, + ready_count: usize, + failed_count: usize, + deferred_count: usize, } #[derive(Clone, Copy)] @@ -811,9 +849,78 @@ enum BitrotReaderSetupMode { VerifyReconstruction, } +#[derive(Clone, Copy, PartialEq, Eq)] +enum BitrotReaderSetupStrategy { + AllShards, + DataBlocksFirst, + DataBlocksOnly, +} + +impl BitrotReaderSetupMode { + fn as_str(self) -> &'static str { + match self { + BitrotReaderSetupMode::ReadQuorum => "read_quorum", + BitrotReaderSetupMode::VerifyReconstruction => "verify_reconstruction", + } + } +} + +impl BitrotReaderSetupStrategy { + fn as_str(self) -> &'static str { + match self { + BitrotReaderSetupStrategy::AllShards => "all_shards", + BitrotReaderSetupStrategy::DataBlocksFirst => "data_blocks_first", + BitrotReaderSetupStrategy::DataBlocksOnly => "data_blocks_only", + } + } +} + +fn get_bitrot_reader_setup_strategy(mode: BitrotReaderSetupMode, prefer_data_blocks_first: bool) -> BitrotReaderSetupStrategy { + match mode { + BitrotReaderSetupMode::ReadQuorum + if prefer_data_blocks_first || rustfs_utils::get_env_bool(ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, false) => + { + BitrotReaderSetupStrategy::DataBlocksFirst + } + BitrotReaderSetupMode::VerifyReconstruction + if rustfs_utils::get_env_bool(ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, false) => + { + BitrotReaderSetupStrategy::DataBlocksFirst + } + _ => BitrotReaderSetupStrategy::AllShards, + } +} + impl BitrotReaderSetup { + fn new(shards: usize) -> Self { + Self { + readers: (0..shards).map(|_| None).collect(), + errors: vec![Some(DiskError::DiskNotFound); shards], + scheduled: vec![false; shards], + attempted: vec![false; shards], + ready: vec![false; shards], + scheduled_count: 0, + attempted_count: 0, + ready_count: 0, + failed_count: 0, + deferred_count: 0, + } + } + + fn scheduled_shards(&self) -> usize { + self.scheduled_count + } + + fn attempted_shards(&self) -> usize { + self.attempted_count + } + + fn pending_scheduled_shards(&self) -> usize { + self.scheduled_count.saturating_sub(self.attempted_count) + } + fn available_shards(&self) -> usize { - self.ready.iter().filter(|ready| **ready).count() + self.ready_count } fn available_data_shards(&self, data_shards: usize) -> usize { @@ -821,11 +928,7 @@ impl BitrotReaderSetup { } fn completed_failed_shards(&self) -> usize { - self.attempted - .iter() - .zip(&self.errors) - .filter(|(attempted, err)| **attempted && err.is_some()) - .count() + self.failed_count } fn data_shards_attempted(&self, data_shards: usize) -> bool { @@ -842,12 +945,271 @@ impl BitrotReaderSetup { } fn has_setup_quorum(&self, data_shards: usize, parity_shards: usize, mode: BitrotReaderSetupMode) -> bool { - let target = match mode { + self.available_shards() >= self.setup_target(data_shards, parity_shards, mode) + } + + fn setup_target(&self, data_shards: usize, parity_shards: usize, mode: BitrotReaderSetupMode) -> usize { + match mode { BitrotReaderSetupMode::ReadQuorum => data_shards, BitrotReaderSetupMode::VerifyReconstruction => self.reconstruction_verification_target(data_shards, parity_shards), - }; - self.available_shards() >= target + } } + + fn deferred_shards(&self) -> usize { + self.deferred_count + } + + fn mark_scheduled(&mut self, idx: usize) -> bool { + if self.scheduled[idx] { + return false; + } + self.scheduled[idx] = true; + self.scheduled_count = self.scheduled_count.saturating_add(1); + true + } + + fn apply_reader_result(&mut self, idx: usize, result: std::result::Result, DiskError>) { + self.attempted[idx] = true; + self.attempted_count = self.attempted_count.saturating_add(1); + match result { + Ok(Some(reader)) => { + self.readers[idx] = Some(reader); + self.errors[idx] = None; + self.ready[idx] = true; + self.ready_count = self.ready_count.saturating_add(1); + } + Ok(None) => { + self.readers[idx] = None; + self.errors[idx] = Some(DiskError::DiskNotFound); + self.ready[idx] = false; + self.failed_count = self.failed_count.saturating_add(1); + } + Err(e) => { + self.readers[idx] = None; + self.errors[idx] = Some(e); + self.ready[idx] = false; + self.failed_count = self.failed_count.saturating_add(1); + } + } + } + + fn retain_deferred_reader(&mut self, idx: usize, reader: ObjectBitrotReader) { + self.readers[idx] = Some(reader); + self.errors[idx] = None; + self.deferred_count = self.deferred_count.saturating_add(1); + } +} + +#[allow(clippy::too_many_arguments)] +fn schedule_bitrot_reader_task<'a>( + reader_tasks: &mut FuturesUnordered>, + setup: &mut BitrotReaderSetup, + idx: usize, + files: &'a [FileInfo], + disks: &'a [Option], + bucket: &'a str, + object: &'a str, + part_number: usize, + read_offset: usize, + read_length: usize, + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify_bitrot: bool, + use_mmap_read: bool, + stage_metrics: Option, +) { + if idx >= disks.len() || !setup.mark_scheduled(idx) { + return; + } + + let inline_data = files[idx].data.as_deref(); + let data_dir = files[idx].data_dir.unwrap_or_default(); + let disk = disks[idx].as_ref(); + let path = format!("{object}/{data_dir}/part.{part_number}"); + + reader_tasks.push(Box::pin(async move { + let result = create_bitrot_reader_with_stage_metrics( + inline_data, + disk, + bucket, + &path, + read_offset, + read_length, + shard_size, + checksum_algo, + skip_verify_bitrot, + use_mmap_read, + stage_metrics, + ) + .await; + (idx, result) + })); +} + +fn next_unscheduled_reader_index(setup: &BitrotReaderSetup, total_shards: usize, data_shards: usize) -> Option { + (data_shards..total_shards) + .chain(0..data_shards.min(total_shards)) + .find(|idx| !setup.scheduled[*idx]) +} + +#[allow(clippy::too_many_arguments)] +fn fill_deferred_bitrot_readers( + setup: &mut BitrotReaderSetup, + files: &[FileInfo], + disks: &[Option], + bucket: &str, + object: &str, + part_number: usize, + read_offset: usize, + read_length: usize, + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify_bitrot: bool, + use_mmap_read: bool, + data_shards: usize, + parity_shards: usize, + mode: BitrotReaderSetupMode, +) { + if !setup.has_setup_quorum(data_shards, parity_shards, mode) { + return; + } + + for idx in 0..disks.len() { + if setup.attempted[idx] { + continue; + } + + let inline_data = files[idx].data.clone(); + let disk = disks[idx].clone(); + let data_dir = files[idx].data_dir.unwrap_or_default(); + let path = format!("{object}/{data_dir}/part.{part_number}"); + setup.retain_deferred_reader( + idx, + create_deferred_bitrot_reader( + inline_data, + disk, + bucket, + &path, + read_offset, + read_length, + shard_size, + checksum_algo.clone(), + skip_verify_bitrot, + use_mmap_read, + ), + ); + } +} + +fn record_bitrot_reader_setup_fanout( + strategy: BitrotReaderSetupStrategy, + mode: BitrotReaderSetupMode, + setup: &BitrotReaderSetup, +) { + rustfs_io_metrics::record_get_object_reader_setup_fanout( + strategy.as_str(), + mode.as_str(), + setup.scheduled_shards(), + setup.attempted_shards(), + setup.available_shards(), + setup.completed_failed_shards(), + setup.deferred_shards(), + ); +} + +#[allow(clippy::too_many_arguments)] +async fn create_bitrot_readers_until_quorum_all_shards( + files: &[FileInfo], + disks: &[Option], + bucket: &str, + object: &str, + part_number: usize, + read_offset: usize, + read_length: usize, + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify_bitrot: bool, + use_mmap_read: bool, + data_shards: usize, + parity_shards: usize, + mode: BitrotReaderSetupMode, + stage_metrics: Option, +) -> BitrotReaderSetup { + let strategy = BitrotReaderSetupStrategy::AllShards; + let mut setup = BitrotReaderSetup::new(disks.len()); + let mut reader_tasks = FuturesUnordered::new(); + let stage_metrics = stage_metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled()); + + rustfs_io_metrics::record_get_object_reader_setup_strategy(strategy.as_str(), mode.as_str()); + + let schedule_stage_start = stage_metrics.map(|_| Instant::now()); + for (idx, disk_op) in disks.iter().enumerate() { + setup.mark_scheduled(idx); + let inline_data = files[idx].data.as_deref(); + let data_dir = files[idx].data_dir.unwrap_or_default(); + let disk = disk_op.as_ref(); + let path = format!("{object}/{data_dir}/part.{part_number}"); + let checksum_algo = checksum_algo.clone(); + + reader_tasks.push(async move { + let result = create_bitrot_reader_with_stage_metrics( + inline_data, + disk, + bucket, + &path, + read_offset, + read_length, + shard_size, + checksum_algo, + skip_verify_bitrot, + use_mmap_read, + stage_metrics, + ) + .await; + (idx, result) + }); + } + if let Some(stage_metrics) = stage_metrics { + record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_SCHEDULE, schedule_stage_start); + } + + let wait_quorum_stage_start = stage_metrics.map(|_| Instant::now()); + while let Some((idx, result)) = reader_tasks.next().await { + setup.apply_reader_result(idx, result); + + if setup.has_setup_quorum(data_shards, parity_shards, mode) { + break; + } + } + if let Some(stage_metrics) = stage_metrics { + record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_WAIT_QUORUM, wait_quorum_stage_start); + } + + fill_deferred_bitrot_readers( + &mut setup, + files, + disks, + bucket, + object, + part_number, + read_offset, + read_length, + shard_size, + checksum_algo, + skip_verify_bitrot, + use_mmap_read, + data_shards, + parity_shards, + mode, + ); + let drop_pending_stage_start = stage_metrics.map(|_| Instant::now()); + drop(reader_tasks); + if let Some(stage_metrics) = stage_metrics { + record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_DROP_PENDING, drop_pending_stage_start); + } + record_bitrot_reader_setup_fanout(strategy, mode, &setup); + + setup } #[allow(clippy::too_many_arguments)] @@ -866,91 +1228,254 @@ async fn create_bitrot_readers_until_quorum( data_shards: usize, parity_shards: usize, mode: BitrotReaderSetupMode, + stage_metrics: Option, ) -> BitrotReaderSetup { - let mut setup = BitrotReaderSetup { - readers: (0..disks.len()).map(|_| None).collect(), - errors: vec![Some(DiskError::DiskNotFound); disks.len()], - attempted: vec![false; disks.len()], - ready: vec![false; disks.len()], - }; - let mut reader_tasks = FuturesUnordered::new(); + create_bitrot_readers_until_quorum_with_preference( + files, + disks, + bucket, + object, + part_number, + read_offset, + read_length, + shard_size, + checksum_algo, + skip_verify_bitrot, + use_mmap_read, + data_shards, + parity_shards, + mode, + false, + stage_metrics, + ) + .await +} - for (idx, disk_op) in disks.iter().enumerate() { - let inline_data = files[idx].data.as_deref(); - let data_dir = files[idx].data_dir.unwrap_or_default(); - let disk = disk_op.as_ref(); - let path = format!("{object}/{data_dir}/part.{part_number}"); - let checksum_algo = checksum_algo.clone(); - - reader_tasks.push(async move { - let result = create_bitrot_reader( - inline_data, - disk, - bucket, - &path, - read_offset, - read_length, - shard_size, - checksum_algo, - skip_verify_bitrot, - use_mmap_read, - ) - .await; - (idx, result) - }); +#[allow(clippy::too_many_arguments)] +async fn create_bitrot_readers_until_quorum_with_preference( + files: &[FileInfo], + disks: &[Option], + bucket: &str, + object: &str, + part_number: usize, + read_offset: usize, + read_length: usize, + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify_bitrot: bool, + use_mmap_read: bool, + data_shards: usize, + parity_shards: usize, + mode: BitrotReaderSetupMode, + prefer_data_blocks_first: bool, + stage_metrics: Option, +) -> BitrotReaderSetup { + let strategy = get_bitrot_reader_setup_strategy(mode, prefer_data_blocks_first); + if strategy == BitrotReaderSetupStrategy::AllShards { + return create_bitrot_readers_until_quorum_all_shards( + files, + disks, + bucket, + object, + part_number, + read_offset, + read_length, + shard_size, + checksum_algo, + skip_verify_bitrot, + use_mmap_read, + data_shards, + parity_shards, + mode, + stage_metrics, + ) + .await; } + let mut setup = BitrotReaderSetup::new(disks.len()); + let mut reader_tasks: FuturesUnordered> = FuturesUnordered::new(); + let total_shards = disks.len(); + let stage_metrics = stage_metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled()); + + rustfs_io_metrics::record_get_object_reader_setup_strategy(strategy.as_str(), mode.as_str()); + + let schedule_stage_start = stage_metrics.map(|_| Instant::now()); + for idx in 0..data_shards.min(total_shards) { + schedule_bitrot_reader_task( + &mut reader_tasks, + &mut setup, + idx, + files, + disks, + bucket, + object, + part_number, + read_offset, + read_length, + shard_size, + checksum_algo.clone(), + skip_verify_bitrot, + use_mmap_read, + stage_metrics, + ); + } + + if data_shards < total_shards { + schedule_bitrot_reader_task( + &mut reader_tasks, + &mut setup, + data_shards, + files, + disks, + bucket, + object, + part_number, + read_offset, + read_length, + shard_size, + checksum_algo.clone(), + skip_verify_bitrot, + use_mmap_read, + stage_metrics, + ); + } + if let Some(stage_metrics) = stage_metrics { + record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_SCHEDULE, schedule_stage_start); + } + + let wait_quorum_stage_start = stage_metrics.map(|_| Instant::now()); while let Some((idx, result)) = reader_tasks.next().await { - setup.attempted[idx] = true; - match result { - Ok(Some(reader)) => { - setup.readers[idx] = Some(reader); - setup.errors[idx] = None; - setup.ready[idx] = true; - } - Ok(None) => { - setup.readers[idx] = None; - setup.errors[idx] = Some(DiskError::DiskNotFound); - setup.ready[idx] = false; - } - Err(e) => { - setup.readers[idx] = None; - setup.errors[idx] = Some(e); - setup.ready[idx] = false; - } - } + setup.apply_reader_result(idx, result); if setup.has_setup_quorum(data_shards, parity_shards, mode) { break; } - } - if setup.has_setup_quorum(data_shards, parity_shards, mode) { - for idx in 0..disks.len() { - if setup.attempted[idx] { - continue; - } - - let inline_data = files[idx].data.clone(); - let disk = disks[idx].clone(); - let data_dir = files[idx].data_dir.unwrap_or_default(); - let path = format!("{object}/{data_dir}/part.{part_number}"); - setup.readers[idx] = Some(create_deferred_bitrot_reader( - inline_data, - disk, + let target = setup.setup_target(data_shards, parity_shards, mode); + while setup.available_shards().saturating_add(setup.pending_scheduled_shards()) < target { + let Some(next_idx) = next_unscheduled_reader_index(&setup, total_shards, data_shards) else { + break; + }; + schedule_bitrot_reader_task( + &mut reader_tasks, + &mut setup, + next_idx, + files, + disks, bucket, - &path, + object, + part_number, read_offset, read_length, shard_size, checksum_algo.clone(), skip_verify_bitrot, use_mmap_read, - )); - setup.errors[idx] = None; + stage_metrics, + ); } } + if let Some(stage_metrics) = stage_metrics { + record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_WAIT_QUORUM, wait_quorum_stage_start); + } + + fill_deferred_bitrot_readers( + &mut setup, + files, + disks, + bucket, + object, + part_number, + read_offset, + read_length, + shard_size, + checksum_algo, + skip_verify_bitrot, + use_mmap_read, + data_shards, + parity_shards, + mode, + ); + let drop_pending_stage_start = stage_metrics.map(|_| Instant::now()); drop(reader_tasks); + if let Some(stage_metrics) = stage_metrics { + record_get_stage_duration_if_enabled(stage_metrics.path, GET_STAGE_READER_SETUP_DROP_PENDING, drop_pending_stage_start); + } + record_bitrot_reader_setup_fanout(strategy, mode, &setup); + + setup +} + +#[allow(clippy::too_many_arguments)] +async fn create_data_block_bitrot_readers( + files: &[FileInfo], + disks: &[Option], + bucket: &str, + object: &str, + part_number: usize, + read_offset: usize, + read_length: usize, + shard_size: usize, + checksum_algo: HashAlgorithm, + skip_verify_bitrot: bool, + use_mmap_read: bool, + data_shards: usize, +) -> BitrotReaderSetup { + let strategy = BitrotReaderSetupStrategy::DataBlocksOnly; + let total_shards = disks.len().min(files.len()); + let mut setup = BitrotReaderSetup::new(total_shards); + let mut reader_tasks: FuturesUnordered> = FuturesUnordered::new(); + let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let reader_stage_metrics = stage_metrics_enabled.then_some(DIRECT_MEMORY_BITROT_READER_STAGE_METRICS); + + rustfs_io_metrics::record_get_object_reader_setup_strategy(strategy.as_str(), BitrotReaderSetupMode::ReadQuorum.as_str()); + + let schedule_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); + for idx in 0..data_shards.min(total_shards) { + schedule_bitrot_reader_task( + &mut reader_tasks, + &mut setup, + idx, + files, + disks, + bucket, + object, + part_number, + read_offset, + read_length, + shard_size, + checksum_algo.clone(), + skip_verify_bitrot, + use_mmap_read, + reader_stage_metrics, + ); + } + record_get_stage_duration_if_enabled(GET_OBJECT_PATH_DIRECT_MEMORY, GET_STAGE_READER_SETUP_SCHEDULE, schedule_stage_start); + + let wait_quorum_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); + while let Some((idx, result)) = reader_tasks.next().await { + setup.apply_reader_result(idx, result); + if setup.available_data_shards(data_shards) >= data_shards { + break; + } + } + record_get_stage_duration_if_enabled( + GET_OBJECT_PATH_DIRECT_MEMORY, + GET_STAGE_READER_SETUP_WAIT_QUORUM, + wait_quorum_stage_start, + ); + + let drop_pending_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); + drop(reader_tasks); + record_get_stage_duration_if_enabled( + GET_OBJECT_PATH_DIRECT_MEMORY, + GET_STAGE_READER_SETUP_DROP_PENDING, + drop_pending_stage_start, + ); + + // The direct-memory path only consumes the data shard readers. If one of + // them is missing, the caller falls back to the regular GET path. + record_bitrot_reader_setup_fanout(strategy, BitrotReaderSetupMode::ReadQuorum, &setup); setup } @@ -1036,17 +1561,37 @@ where } impl SetDisks { - async fn is_get_object_metadata_cache_enabled(&self, bucket: &str, opts: &ObjectOptions, read_data: bool) -> bool { - is_get_object_metadata_cache_request_eligible(bucket, opts, read_data) && !runtime_sources::setup_is_dist_erasure().await + async fn get_object_metadata_cache_bypass_reason( + bucket: &str, + opts: &ObjectOptions, + read_data: bool, + ) -> Option<&'static str> { + if let Some(reason) = get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data) { + return Some(reason); + } + runtime_sources::setup_is_dist_erasure() + .await + .then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE) } async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option { + match self.lookup_cached_get_object_fileinfo(bucket, object).await { + MetadataCacheLookup::Hit(entry) => Some((*entry).clone()), + MetadataCacheLookup::Miss | MetadataCacheLookup::RejectedInsufficientQuorum => None, + } + } + + async fn lookup_cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> MetadataCacheLookup { let key = GetObjectMetadataCacheKey::new(bucket, object); // moka handles TTL expiry automatically; no is_fresh() check needed - self.get_object_metadata_cache - .get(&key) - .await - .filter(|entry| entry.online_disks.iter().filter(|disk| disk.is_some()).count() >= entry.read_quorum) + let Some(entry) = self.get_object_metadata_cache.get(&key).await else { + return MetadataCacheLookup::Miss; + }; + if entry.online_disks.iter().filter(|disk| disk.is_some()).count() >= entry.read_quorum { + MetadataCacheLookup::Hit(entry) + } else { + MetadataCacheLookup::RejectedInsufficientQuorum + } } async fn cache_get_object_fileinfo( @@ -1067,13 +1612,13 @@ impl SetDisks { self.get_object_metadata_cache .insert( key, - GetObjectMetadataCacheEntry { + Arc::new(GetObjectMetadataCacheEntry { created_at: Instant::now(), fi: fi.clone(), parts_metadata: parts_metadata.to_vec(), online_disks: online_disks.to_vec(), read_quorum, - }, + }), ) .await; } @@ -1199,10 +1744,8 @@ impl SetDisks { observe: bool, default_parity_count: usize, ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { - let early_stop_enabled = observe && is_get_metadata_early_stop_enabled(); - let allow_early_stop = observe - && ((is_get_metadata_early_stop_enabled() && version_id.is_empty() && !healing && !incl_free_versions) - || (is_version_early_stop_enabled() && !version_id.is_empty() && !healing)); + let early_stop_enabled = observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled()); + let allow_early_stop = observe && should_allow_metadata_early_stop(read_data, version_id, healing, incl_free_versions); if allow_early_stop { return Self::read_all_fileinfo_early_stop( disks, @@ -1403,6 +1946,7 @@ impl SetDisks { GET_OBJECT_PATH_LEGACY_DUPLEX, saved_responses, ); + while join_set.join_next().await.is_some() {} let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); return Ok((ress, errors, diagnostics)); } @@ -1745,14 +2289,53 @@ impl SetDisks { read_data: bool, ) -> Result<(FileInfo, Vec, Vec>)> { let vid = opts.version_id.clone().unwrap_or_default(); + let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); - let use_metadata_cache = self.is_get_object_metadata_cache_enabled(bucket, opts, read_data).await; - if use_metadata_cache - && vid.is_empty() - && let Some(cached) = self.cached_get_object_fileinfo(bucket, object).await - { - return Ok((cached.fi, cached.parts_metadata, cached.online_disks)); + let metadata_cache_lookup_start = get_stage_timer_if_enabled(stage_metrics_enabled); + let cache_bypass_reason = Self::get_object_metadata_cache_bypass_reason(bucket, opts, read_data).await; + let use_metadata_cache = cache_bypass_reason.is_none(); + if let Some(reason) = cache_bypass_reason { + rustfs_io_metrics::record_get_object_metadata_cache_decision( + GET_OBJECT_PATH_SET_DISK, + GET_METADATA_CACHE_DECISION_SKIP, + reason, + ); + } else if vid.is_empty() { + match self.lookup_cached_get_object_fileinfo(bucket, object).await { + MetadataCacheLookup::Hit(cached) => { + rustfs_io_metrics::record_get_object_metadata_cache_decision( + GET_OBJECT_PATH_SET_DISK, + GET_METADATA_CACHE_DECISION_HIT, + GET_METADATA_CACHE_REASON_USABLE, + ); + record_get_stage_duration_if_enabled( + GET_OBJECT_PATH_SET_DISK, + GET_STAGE_METADATA_CACHE_LOOKUP, + metadata_cache_lookup_start, + ); + return Ok((cached.fi.clone(), cached.parts_metadata.clone(), cached.online_disks.clone())); + } + MetadataCacheLookup::Miss => { + rustfs_io_metrics::record_get_object_metadata_cache_decision( + GET_OBJECT_PATH_SET_DISK, + GET_METADATA_CACHE_DECISION_MISS, + GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, + ); + } + MetadataCacheLookup::RejectedInsufficientQuorum => { + rustfs_io_metrics::record_get_object_metadata_cache_decision( + GET_OBJECT_PATH_SET_DISK, + GET_METADATA_CACHE_DECISION_REJECT, + GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM, + ); + } + } } + record_get_stage_duration_if_enabled( + GET_OBJECT_PATH_SET_DISK, + GET_STAGE_METADATA_CACHE_LOOKUP, + metadata_cache_lookup_start, + ); let disks = self.disks.read().await; @@ -1772,17 +2355,24 @@ impl SetDisks { ) .await?; metadata_fanout_diagnostics.record(GET_OBJECT_PATH_LEGACY_DUPLEX); + let metadata_fanout_complete = metadata_fanout_diagnostics.total_responses() >= disks.len(); // warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata); // warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs); let _min_disks = self.set_drive_count - self.default_parity_count; + let metadata_resolve_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); let (read_quorum, _) = match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count) .map_err(|err| to_object_err(err.into(), vec![bucket, object])) { Ok(v) => v, Err(e) => { // error!("Self::object_quorum_from_meta: {:?}, bucket: {}, object: {}", &e, bucket, object); + record_get_stage_duration_if_enabled( + GET_OBJECT_PATH_SET_DISK, + GET_STAGE_METADATA_RESOLVE, + metadata_resolve_stage_start, + ); return Err(e); } }; @@ -1792,6 +2382,11 @@ impl SetDisks { if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object); + record_get_stage_duration_if_enabled( + GET_OBJECT_PATH_SET_DISK, + GET_STAGE_METADATA_RESOLVE, + metadata_resolve_stage_start, + ); return Err(to_object_err(err.into(), vec![bucket, object])); } @@ -1810,10 +2405,11 @@ impl SetDisks { "metadata_read_error", ) .await; - } else if use_metadata_cache { + } else if use_metadata_cache && metadata_fanout_complete { self.cache_get_object_fileinfo(bucket, object, &fi, &parts_metadata, &op_online_disks, read_quorum) .await; } + record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_METADATA_RESOLVE, metadata_resolve_stage_start); // debug!("get_object_fileinfo pick fi {:?}", &fi); // let online_disks: Vec> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect(); @@ -1862,6 +2458,156 @@ impl SetDisks { (oi, write_quorum, None) } + #[allow(clippy::too_many_arguments)] + pub(super) async fn try_get_object_direct_data_shards_with_fileinfo( + bucket: &str, + object: &str, + fi: &FileInfo, + files: &[FileInfo], + disks: &[Option], + skip_verify_bitrot: bool, + metrics_object_class: &'static str, + metrics_size_bucket: &'static str, + ) -> Result> { + if fi.parts.len() != 1 || !object_fits_single_block(fi.size, fi.erasure.block_size) { + return Ok(None); + } + + let object_size = usize::try_from(fi.size) + .map_err(|_| to_object_err(Error::other("direct-memory GET object size is invalid"), vec![bucket, object]))?; + let Some(part) = fi.parts.first() else { + return Ok(None); + }; + if part.size != object_size { + return Ok(None); + } + + let erasure = coding::Erasure::new_with_options( + fi.erasure.data_blocks, + fi.erasure.parity_blocks, + fi.erasure.block_size, + fi.uses_legacy_checksum, + ); + if erasure.data_shards == 0 { + return Ok(None); + } + + let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, files, fi); + let checksum_info = fi.erasure.get_checksum_info(part.number); + let checksum_algo = if fi.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { + HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; + let read_length = erasure.shard_file_offset(0, object_size, object_size); + + if fi.data.is_some() { + let Some(data_files) = collect_inline_data_shard_fileinfos_by_index(&files, fi, erasure.data_shards, |index| { + disks.get(index).is_some_and(Option::is_some) + }) else { + return Ok(None); + }; + + let reader_setup_stage_start = Instant::now(); + let mut readers = build_inline_bitrot_readers_from_refs( + &data_files, + bucket, + object, + read_length, + erasure.shard_size(), + &checksum_algo, + skip_verify_bitrot, + ) + .await?; + let reader_setup_elapsed = reader_setup_stage_start.elapsed(); + rustfs_io_metrics::record_get_object_shard_reader_setup_duration(reader_setup_elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_stage_duration_by_size( + GET_OBJECT_PATH_DIRECT_MEMORY, + GET_STAGE_READER_SETUP, + metrics_object_class, + metrics_size_bucket, + reader_setup_elapsed.as_secs_f64(), + ); + + let decode_stage_start = Instant::now(); + let body = try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await; + let decode_elapsed = decode_stage_start.elapsed(); + rustfs_io_metrics::record_get_object_decode_duration(decode_elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_stage_duration_by_size( + GET_OBJECT_PATH_DIRECT_MEMORY, + GET_STAGE_DECODE, + metrics_object_class, + metrics_size_bucket, + decode_elapsed.as_secs_f64(), + ); + + if body.is_some() { + rustfs_io_metrics::record_get_object_direct_memory_subpath( + GET_DIRECT_MEMORY_SUBPATH_INLINE_BUFFERED, + metrics_object_class, + metrics_size_bucket, + ); + } + + return Ok(body); + } + + let use_mmap_read = object_mmap_read_enabled(); + + let reader_setup_stage_start = Instant::now(); + let mut reader_setup = create_data_block_bitrot_readers( + &files, + &disks, + bucket, + object, + part.number, + 0, + read_length, + erasure.shard_size(), + checksum_algo, + skip_verify_bitrot, + use_mmap_read, + erasure.data_shards, + ) + .await; + let reader_setup_elapsed = reader_setup_stage_start.elapsed(); + rustfs_io_metrics::record_get_object_shard_reader_setup_duration(reader_setup_elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_stage_duration_by_size( + GET_OBJECT_PATH_DIRECT_MEMORY, + GET_STAGE_READER_SETUP, + metrics_object_class, + metrics_size_bucket, + reader_setup_elapsed.as_secs_f64(), + ); + + if reader_setup.available_data_shards(erasure.data_shards) < erasure.data_shards { + return Ok(None); + } + + let decode_stage_start = Instant::now(); + let body = + try_read_inline_data_shards_direct(&mut reader_setup.readers, erasure.data_shards, read_length, object_size).await; + let decode_elapsed = decode_stage_start.elapsed(); + rustfs_io_metrics::record_get_object_decode_duration(decode_elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_stage_duration_by_size( + GET_OBJECT_PATH_DIRECT_MEMORY, + GET_STAGE_DECODE, + metrics_object_class, + metrics_size_bucket, + decode_elapsed.as_secs_f64(), + ); + + if body.is_some() { + rustfs_io_metrics::record_get_object_direct_memory_subpath( + GET_DIRECT_MEMORY_SUBPATH_DISK_DATA_BLOCKS, + metrics_object_class, + metrics_size_bucket, + ); + } + + Ok(body) + } + #[allow(clippy::too_many_arguments)] pub(super) async fn get_object_with_fileinfo( // &self, @@ -1876,6 +2622,10 @@ impl SetDisks { set_index: usize, pool_index: usize, skip_verify_bitrot: bool, + prefer_data_blocks_first_reader_setup: bool, + metrics_path: &'static str, + metrics_object_class: &'static str, + metrics_size_bucket: &'static str, ) -> Result<()> where W: AsyncWrite + Send + Sync + Unpin + 'static, @@ -2019,7 +2769,7 @@ impl SetDisks { .iter() .map(|disk| shard_read_cost_for_disk(disk.as_ref())) .collect::>(); - let reader_setup = create_bitrot_readers_until_quorum( + let reader_setup = create_bitrot_readers_until_quorum_with_preference( &files, &disks, bucket, @@ -2034,10 +2784,19 @@ impl SetDisks { erasure.data_shards, erasure.parity_shards, BitrotReaderSetupMode::ReadQuorum, + prefer_data_blocks_first_reader_setup, + None, ) .await; let reader_setup_elapsed = reader_setup_stage_start.elapsed(); rustfs_io_metrics::record_get_object_shard_reader_setup_duration(reader_setup_elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_stage_duration_by_size( + metrics_path, + GET_STAGE_READER_SETUP, + metrics_object_class, + metrics_size_bucket, + reader_setup_elapsed.as_secs_f64(), + ); let setup_available_readers = reader_setup.available_shards(); if reader_setup_elapsed >= SLOW_OBJECT_READ_LOG_THRESHOLD { warn!( @@ -2172,6 +2931,13 @@ impl SetDisks { .await; let decode_elapsed = decode_stage_start.elapsed(); rustfs_io_metrics::record_get_object_decode_duration(decode_elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_stage_duration_by_size( + metrics_path, + GET_STAGE_DECODE, + metrics_object_class, + metrics_size_bucket, + decode_elapsed.as_secs_f64(), + ); if decode_elapsed >= SLOW_OBJECT_READ_LOG_THRESHOLD || err.is_some() { warn!( event = EVENT_SET_DISK_READ, @@ -2399,6 +3165,13 @@ impl SetDisks { let read_length = till_offset.saturating_sub(read_offset); let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let metrics_path = get_codec_streaming_metrics_path(); + let reader_stage_metrics = stage_metrics_enabled.then_some(BitrotReaderStageMetrics { + path: metrics_path, + reader_construction_stage: GET_STAGE_READER_TASK_READER_CONSTRUCTION, + file_open_stage: GET_STAGE_READER_TASK_FILE_OPEN, + bitrot_reader_init_stage: GET_STAGE_READER_TASK_BITROT_READER_INIT, + }); let reader_setup_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); let read_costs = disks .iter() @@ -2419,9 +3192,9 @@ impl SetDisks { erasure.data_shards, erasure.parity_shards, BitrotReaderSetupMode::VerifyReconstruction, + reader_stage_metrics, ) .await; - let metrics_path = get_codec_streaming_metrics_path(); record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_SETUP, reader_setup_stage_start); let available_shards = reader_setup.available_shards(); @@ -2458,18 +3231,53 @@ impl SetDisks { } } +fn should_allow_metadata_early_stop(read_data: bool, version_id: &str, healing: bool, incl_free_versions: bool) -> bool { + if read_data { + return false; + } + + (is_get_metadata_early_stop_enabled() && version_id.is_empty() && !healing && !incl_free_versions) + || (is_version_early_stop_enabled() && !version_id.is_empty() && !healing) +} + +fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOptions, read_data: bool) -> Option<&'static str> { + if !read_data { + return Some(GET_METADATA_CACHE_REASON_NOT_READ_DATA); + } + if opts.no_lock && !opts.metadata_cache_safe { + return Some(GET_METADATA_CACHE_REASON_NO_LOCK); + } + if opts.version_id.is_some() { + return Some(GET_METADATA_CACHE_REASON_VERSION_ID); + } + if opts.versioned { + return Some(GET_METADATA_CACHE_REASON_VERSIONED); + } + if opts.version_suspended { + return Some(GET_METADATA_CACHE_REASON_VERSION_SUSPENDED); + } + if opts.incl_free_versions { + return Some(GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS); + } + if opts.delete_marker { + return Some(GET_METADATA_CACHE_REASON_DELETE_MARKER); + } + if opts.part_number.is_some() { + return Some(GET_METADATA_CACHE_REASON_PART_NUMBER); + } + if opts.data_movement { + return Some(GET_METADATA_CACHE_REASON_DATA_MOVEMENT); + } + if opts.raw_data_movement_read { + return Some(GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ); + } + bucket + .starts_with(RUSTFS_META_BUCKET) + .then_some(GET_METADATA_CACHE_REASON_META_BUCKET) +} + fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOptions, read_data: bool) -> bool { - read_data - && !opts.no_lock - && opts.version_id.is_none() - && !opts.versioned - && !opts.version_suspended - && !opts.incl_free_versions - && !opts.delete_marker - && opts.part_number.is_none() - && !opts.data_movement - && !opts.raw_data_movement_read - && !bucket.starts_with(RUSTFS_META_BUCKET) + get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data).is_none() } #[cfg(test)] @@ -2524,6 +3332,22 @@ mod metadata_cache_tests { .await } + #[test] + #[serial] + fn get_object_metadata_cache_capacity_uses_default_and_env_override() { + temp_env::with_var(ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES, None::<&str>, || { + assert_eq!(get_object_metadata_cache_max_entries(), DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES); + }); + + temp_env::with_var(ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES, Some("16384"), || { + assert_eq!(get_object_metadata_cache_max_entries(), 16_384); + }); + + temp_env::with_var(ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES, Some("0"), || { + assert_eq!(get_object_metadata_cache_max_entries(), 1); + }); + } + fn valid_test_fileinfo(object: &str) -> FileInfo { let mut fi = FileInfo::new(object, 2, 2); fi.volume = "bucket".to_string(); @@ -2538,62 +3362,110 @@ mod metadata_cache_tests { fn get_object_metadata_cache_request_eligibility_is_conservative() { let opts = ObjectOptions::default(); assert!(is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!(get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), None); assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, false)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason("bucket", &opts, false), + Some(GET_METADATA_CACHE_REASON_NOT_READ_DATA) + ); assert!(!is_get_object_metadata_cache_request_eligible(RUSTFS_META_BUCKET, &opts, true)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason(RUSTFS_META_BUCKET, &opts, true), + Some(GET_METADATA_CACHE_REASON_META_BUCKET) + ); let mut opts = ObjectOptions { no_lock: true, ..Default::default() }; assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), + Some(GET_METADATA_CACHE_REASON_NO_LOCK) + ); + opts.metadata_cache_safe = true; + assert!(is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!(get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), None); opts = ObjectOptions { version_id: Some("version".to_string()), ..Default::default() }; assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), + Some(GET_METADATA_CACHE_REASON_VERSION_ID) + ); opts = ObjectOptions { versioned: true, ..Default::default() }; assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), + Some(GET_METADATA_CACHE_REASON_VERSIONED) + ); opts = ObjectOptions { version_suspended: true, ..Default::default() }; assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), + Some(GET_METADATA_CACHE_REASON_VERSION_SUSPENDED) + ); opts = ObjectOptions { incl_free_versions: true, ..Default::default() }; assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), + Some(GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS) + ); opts = ObjectOptions { delete_marker: true, ..Default::default() }; assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), + Some(GET_METADATA_CACHE_REASON_DELETE_MARKER) + ); opts = ObjectOptions { part_number: Some(1), ..Default::default() }; assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), + Some(GET_METADATA_CACHE_REASON_PART_NUMBER) + ); opts = ObjectOptions { data_movement: true, ..Default::default() }; assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), + Some(GET_METADATA_CACHE_REASON_DATA_MOVEMENT) + ); opts = ObjectOptions { raw_data_movement_read: true, ..Default::default() }; assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), + Some(GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ) + ); } #[tokio::test] @@ -2794,13 +3666,13 @@ mod metadata_cache_tests { set.get_object_metadata_cache .insert( GetObjectMetadataCacheKey::new("bucket", "object"), - GetObjectMetadataCacheEntry { + Arc::new(GetObjectMetadataCacheEntry { created_at: Instant::now(), fi: fi.clone(), parts_metadata: vec![fi], online_disks: vec![None], read_quorum: 1, - }, + }), ) .await; @@ -2845,7 +3717,7 @@ mod metadata_cache_tests { #[tokio::test] async fn get_object_metadata_cache_prunes_when_capacity_is_reached() { - // moka handles capacity eviction automatically via max_capacity(1024). + // moka handles capacity eviction automatically via the configured max_capacity. // This test verifies that the cache can hold entries and that insertion works. let set = new_metadata_cache_test_set().await; let fresh_fi = valid_test_fileinfo("fresh-object"); @@ -3342,6 +4214,22 @@ mod tests { }); } + #[test] + fn metadata_early_stop_rejects_data_reads() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")), + (ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE, Some("true")), + ], + || { + assert!(!should_allow_metadata_early_stop(true, "", false, false)); + assert!(!should_allow_metadata_early_stop(true, "version-id", false, false)); + assert!(should_allow_metadata_early_stop(false, "", false, false)); + assert!(should_allow_metadata_early_stop(false, "version-id", false, false)); + }, + ); + } + #[test] fn version_early_stop_gate_defaults_to_disabled() { temp_env::with_var(ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE, None::<&str>, || { @@ -3607,11 +4495,85 @@ mod tests { data_shards: usize, parity_shards: usize, mode: BitrotReaderSetupMode, + ) -> BitrotReaderSetup { + setup_inline_bitrot_readers_with_env(data, data_shards, parity_shards, mode, false).await + } + + async fn setup_inline_bitrot_readers_with_env( + data: Vec>, + data_shards: usize, + parity_shards: usize, + mode: BitrotReaderSetupMode, + data_blocks_first: bool, + ) -> BitrotReaderSetup { + setup_inline_bitrot_readers_with_reader_setup_env(data, data_shards, parity_shards, mode, data_blocks_first, false).await + } + + async fn setup_inline_bitrot_readers_with_codec_reader_setup_env( + data: Vec>, + data_shards: usize, + parity_shards: usize, + mode: BitrotReaderSetupMode, + codec_data_blocks_first: bool, + ) -> BitrotReaderSetup { + setup_inline_bitrot_readers_with_reader_setup_env(data, data_shards, parity_shards, mode, false, codec_data_blocks_first) + .await + } + + async fn setup_inline_bitrot_readers_with_reader_setup_env( + data: Vec>, + data_shards: usize, + parity_shards: usize, + mode: BitrotReaderSetupMode, + data_blocks_first: bool, + codec_data_blocks_first: bool, ) -> BitrotReaderSetup { let files = data.into_iter().map(inline_reader_setup_fileinfo).collect::>(); let disks = vec![None; files.len()]; - create_bitrot_readers_until_quorum( + temp_env::async_with_vars( + [ + (ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, data_blocks_first.then_some("true")), + ( + ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, + codec_data_blocks_first.then_some("true"), + ), + ], + async { + create_bitrot_readers_until_quorum( + &files, + &disks, + "bucket", + "object", + 1, + 0, + 4, + 4, + HashAlgorithm::None, + false, + false, + data_shards, + parity_shards, + mode, + None, + ) + .await + }, + ) + .await + } + + async fn setup_inline_bitrot_readers_with_preference( + data: Vec>, + data_shards: usize, + parity_shards: usize, + mode: BitrotReaderSetupMode, + prefer_data_blocks_first: bool, + ) -> BitrotReaderSetup { + let files = data.into_iter().map(inline_reader_setup_fileinfo).collect::>(); + let disks = vec![None; files.len()]; + + create_bitrot_readers_until_quorum_with_preference( &files, &disks, "bucket", @@ -3626,6 +4588,8 @@ mod tests { data_shards, parity_shards, mode, + prefer_data_blocks_first, + None, ) .await } @@ -3678,6 +4642,124 @@ mod tests { assert_eq!(&out[..n], [b"aaaa", b"bbbb", b"cccc", b"dddd"][fallback_index]); } + #[tokio::test] + async fn bitrot_reader_setup_data_blocks_first_keeps_deferred_fallback_readers() { + let mut setup = setup_inline_bitrot_readers_with_env( + vec![Some(b"aaaa"), Some(b"bbbb"), Some(b"cccc"), Some(b"dddd")], + 2, + 2, + BitrotReaderSetupMode::ReadQuorum, + true, + ) + .await; + + assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::ReadQuorum)); + assert_eq!(setup.available_shards(), 2); + assert!(setup.scheduled_shards() < 4); + assert_eq!(setup.readers.iter().filter(|reader| reader.is_some()).count(), 4); + + let fallback_index = setup + .attempted + .iter() + .position(|attempted| !*attempted) + .expect("data-blocks-first setup should leave at least one deferred fallback"); + let mut fallback = setup.readers[fallback_index] + .take() + .expect("deferred fallback reader should be retained"); + let mut out = [0u8; 4]; + let n = fallback + .read(&mut out) + .await + .expect("deferred fallback reader should open on read"); + + assert_eq!(n, 4); + assert_eq!(&out[..n], [b"aaaa", b"bbbb", b"cccc", b"dddd"][fallback_index]); + } + + #[tokio::test] + async fn bitrot_reader_setup_preference_uses_data_blocks_first_without_env() { + let setup = setup_inline_bitrot_readers_with_preference( + vec![Some(b"aaaa"), Some(b"bbbb"), Some(b"cccc"), Some(b"dddd")], + 2, + 2, + BitrotReaderSetupMode::ReadQuorum, + true, + ) + .await; + + assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::ReadQuorum)); + assert_eq!(setup.available_shards(), 2); + assert!(setup.scheduled_shards() < 4); + assert_eq!(setup.deferred_shards(), 2); + } + + #[tokio::test] + async fn bitrot_reader_setup_data_blocks_first_schedules_parity_after_missing_data() { + let setup = setup_inline_bitrot_readers_with_env( + vec![None, Some(b"bbbb"), Some(b"cccc"), Some(b"dddd")], + 2, + 2, + BitrotReaderSetupMode::ReadQuorum, + true, + ) + .await; + + assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::ReadQuorum)); + assert_eq!(setup.available_shards(), 2); + assert_eq!(setup.completed_failed_shards(), 1); + assert!(setup.ready.iter().skip(2).any(|ready| *ready)); + } + + #[tokio::test] + async fn bitrot_reader_setup_data_blocks_first_does_not_apply_to_verify_mode() { + let setup = setup_inline_bitrot_readers_with_env( + vec![None, Some(b"bbbb"), Some(b"cccc"), Some(b"dddd")], + 2, + 2, + BitrotReaderSetupMode::VerifyReconstruction, + true, + ) + .await; + + assert_eq!(setup.available_shards(), 3); + assert_eq!(setup.scheduled_shards(), 4); + assert_eq!(setup.completed_failed_shards(), 1); + } + + #[tokio::test] + async fn bitrot_reader_setup_codec_data_blocks_first_can_apply_to_verify_mode() { + let setup = setup_inline_bitrot_readers_with_codec_reader_setup_env( + vec![Some(b"aaaa"), Some(b"bbbb"), Some(b"cccc"), Some(b"dddd")], + 2, + 2, + BitrotReaderSetupMode::VerifyReconstruction, + true, + ) + .await; + + assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::VerifyReconstruction)); + assert_eq!(setup.available_data_shards(2), 2); + assert!(setup.scheduled_shards() < 4); + } + + #[tokio::test] + async fn bitrot_reader_setup_codec_data_blocks_first_collects_extra_source_for_reconstruction() { + let setup = setup_inline_bitrot_readers_with_codec_reader_setup_env( + vec![None, Some(b"bbbb"), Some(b"cccc"), Some(b"dddd")], + 2, + 2, + BitrotReaderSetupMode::VerifyReconstruction, + true, + ) + .await; + + assert!(setup.has_setup_quorum(2, 2, BitrotReaderSetupMode::VerifyReconstruction)); + assert_eq!(setup.available_shards(), 3); + assert_eq!(setup.available_data_shards(2), 1); + assert!(setup.data_shards_attempted(2)); + assert_eq!(setup.completed_failed_shards(), 1); + } + #[tokio::test] async fn bitrot_reader_setup_verify_mode_stops_when_data_quorum_is_available() { let setup = setup_inline_bitrot_readers( diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 2d062d7f9..9d1b8a186 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -332,8 +332,8 @@ impl ECStore { pool_meta: RwLock::new(pool_meta), rebalance_meta: RwLock::new(None), decommission_cancelers, - start_gate: tokio::sync::Mutex::new(()), - pool_meta_save_gate: tokio::sync::Mutex::new(()), + start_gate: Mutex::new(()), + pool_meta_save_gate: Mutex::new(()), }); // Only set it when the global deployment ID is not yet configured @@ -545,6 +545,7 @@ mod tests { Ok(GetObjectReader { stream: Box::new(Cursor::new(self.read_payload.clone())), object_info: self.object_info(bucket, object, self.read_payload.len()), + buffered_body: None, }) } diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index c4025a214..184740701 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -428,6 +428,7 @@ impl ECStore { diag_enabled, ); opts.no_lock = true; + opts.metadata_cache_safe = true; Ok(Some(ObjectLockDiagGuard::new( guard, @@ -1301,6 +1302,10 @@ impl ECStore { mod tests { use super::*; use crate::bucket::lifecycle::core::TRANSITION_COMPLETE; + use crate::layout::{ + endpoints::{Endpoints, PoolEndpoints}, + format::FormatV3, + }; use bytes::Bytes; use std::io::Cursor; use std::sync::Arc; @@ -1818,6 +1823,76 @@ mod tests { assert!(lookup_opts.no_lock); } + async fn new_read_lock_test_store() -> ECStore { + let format = FormatV3::new(1, 2); + let endpoints = vec![ + Endpoint::try_from("http://127.0.0.1:9000/data0").expect("first endpoint should parse"), + Endpoint::try_from("http://127.0.0.1:9001/data1").expect("second endpoint should parse"), + ]; + let pool_endpoints = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 2, + endpoints: Endpoints::from(endpoints), + cmd_line: "read-lock-metadata-cache-safe-test".to_string(), + platform: "test".to_string(), + }; + let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone()]); + let sets = Sets::new(vec![None, None], &pool_endpoints, &format, 0, 1) + .await + .expect("test sets should be created with empty disks"); + + ECStore { + id: Uuid::new_v4(), + disk_map: HashMap::new(), + pools: vec![sets], + peer_sys: S3PeerSys::new(&endpoint_pools), + pool_meta: RwLock::new(PoolMeta::default()), + rebalance_meta: RwLock::new(None), + decommission_cancelers: RwLock::new(Vec::new()), + start_gate: Mutex::new(()), + pool_meta_save_gate: Mutex::new(()), + } + } + + #[tokio::test] + async fn acquired_read_lock_marks_metadata_cache_safe_for_set_layer() { + let store = new_read_lock_test_store().await; + let mut opts = ObjectOptions::default(); + + let guard = store + .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts) + .await + .expect("read lock should be acquired"); + + assert!(guard.is_some(), "read lock should be held by the outer store layer"); + assert!(opts.no_lock, "set layer should not reacquire the object lock"); + assert!( + opts.metadata_cache_safe, + "metadata cache is safe only because the outer store layer acquired the read lock" + ); + } + + #[tokio::test] + async fn prelocked_read_request_does_not_mark_metadata_cache_safe() { + let store = new_read_lock_test_store().await; + let mut opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + let guard = store + .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts) + .await + .expect("prelocked read should not acquire another lock"); + + assert!(guard.is_none(), "prelocked caller should keep lock ownership outside ECStore"); + assert!( + !opts.metadata_cache_safe, + "generic no_lock callers must stay ineligible for metadata cache unless explicitly marked safe" + ); + } + #[tokio::test] #[serial_test::serial] async fn reader_lock_is_held_when_optimization_is_disabled() { @@ -1841,6 +1916,7 @@ mod tests { let reader = GetObjectReader { stream: Box::new(Cursor::new(Vec::::new())), object_info: ObjectInfo::default(), + buffered_body: None, }; let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard)); @@ -1879,6 +1955,7 @@ mod tests { let reader = GetObjectReader { stream: Box::new(Cursor::new(vec![1, 2, 3])), object_info: ObjectInfo::default(), + buffered_body: None, }; let mut reader = ECStore::attach_read_lock_guard(reader, Some(read_guard)); diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index 543048798..b6fd3a9db 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -186,6 +186,25 @@ const SHARD_READ_COST_SAME_NODE: &str = "same_node"; const SHARD_READ_COST_UNKNOWN: &str = "unknown"; const LOW_COST_QUORUM_CANDIDATE_FALSE: &str = "false"; const LOW_COST_QUORUM_CANDIDATE_TRUE: &str = "true"; +pub const GET_OBJECT_SIZE_BUCKET_LE_4_KIB: &str = "le_4kib"; +pub const GET_OBJECT_SIZE_BUCKET_LE_16_KIB: &str = "le_16kib"; +pub const GET_OBJECT_SIZE_BUCKET_LE_64_KIB: &str = "le_64kib"; +pub const GET_OBJECT_SIZE_BUCKET_LE_128_KIB: &str = "le_128kib"; +pub const GET_OBJECT_SIZE_BUCKET_LE_1_MIB: &str = "le_1mib"; +pub const GET_OBJECT_SIZE_BUCKET_GT_1_MIB: &str = "gt_1mib"; + +/// Return the bounded size bucket used by small-object GET diagnostics. +#[inline(always)] +pub const fn get_object_size_bucket(size_bytes: i64) -> &'static str { + match size_bytes { + ..=4_096 => GET_OBJECT_SIZE_BUCKET_LE_4_KIB, + 4_097..=16_384 => GET_OBJECT_SIZE_BUCKET_LE_16_KIB, + 16_385..=65_536 => GET_OBJECT_SIZE_BUCKET_LE_64_KIB, + 65_537..=131_072 => GET_OBJECT_SIZE_BUCKET_LE_128_KIB, + 131_073..=1_048_576 => GET_OBJECT_SIZE_BUCKET_LE_1_MIB, + _ => GET_OBJECT_SIZE_BUCKET_GT_1_MIB, + } +} fn saturating_sub_atomic(counter: &AtomicU64, bytes: u64) -> u64 { let mut current = counter.load(Ordering::Relaxed); @@ -404,6 +423,39 @@ pub fn record_get_object_reader_stream_poll( .record(duration_secs); } +/// Record a poll of the single-chunk in-memory GetObject handoff stream. +#[inline(always)] +pub fn record_get_object_memory_body_stream_poll(source: &'static str, outcome: &'static str, bytes: usize, duration_secs: f64) { + if !get_stage_metrics_enabled() { + return; + } + let bytes_counter = u64::try_from(bytes).unwrap_or(u64::MAX); + counter!( + "rustfs_io_get_object_memory_body_stream_poll_total", + "source" => source, + "outcome" => outcome + ) + .increment(1); + counter!( + "rustfs_io_get_object_memory_body_stream_poll_bytes_total", + "source" => source, + "outcome" => outcome + ) + .increment(bytes_counter); + histogram!( + "rustfs_io_get_object_memory_body_stream_poll_bytes", + "source" => source, + "outcome" => outcome + ) + .record(usize_to_f64(bytes)); + histogram!( + "rustfs_io_get_object_memory_body_stream_poll_duration_seconds", + "source" => source, + "outcome" => outcome + ) + .record(duration_secs); +} + /// Record I/O queue congestion observation. #[inline(always)] pub fn record_io_queue_congestion() { @@ -446,6 +498,28 @@ pub fn record_get_object_stage_duration(path: &'static str, stage: &'static str, histogram!("rustfs_io_get_object_stage_duration_seconds", "path" => path, "stage" => stage).record(duration_secs); } +/// Record GetObject stage duration with bounded object class and size labels. +#[inline(always)] +pub fn record_get_object_stage_duration_by_size( + path: &'static str, + stage: &'static str, + object_class: &'static str, + size_bucket: &'static str, + duration_secs: f64, +) { + if !get_stage_metrics_enabled() { + return; + } + histogram!( + "rustfs_io_get_object_stage_duration_seconds_by_size", + "path" => path, + "stage" => stage, + "object_class" => object_class, + "size_bucket" => size_bucket + ) + .record(duration_secs); +} + /// Record GetObject metadata fanout duration. #[inline(always)] pub fn record_get_object_metadata_fanout_duration(path: &'static str, duration_secs: f64) { @@ -485,6 +559,16 @@ pub fn record_get_object_metadata_response(path: &'static str, outcome: &'static counter!("rustfs_io_get_object_metadata_response_total", "path" => path, "outcome" => outcome).increment(1); } +/// Record one bounded metadata cache decision. +#[inline(always)] +pub fn record_get_object_metadata_cache_decision(path: &'static str, decision: &'static str, reason: &'static str) { + if !get_stage_metrics_enabled() { + return; + } + counter!("rustfs_io_get_object_metadata_cache_total", "path" => path, "decision" => decision, "reason" => reason) + .increment(1); +} + /// Record aggregate metadata fanout shape for one GetObject metadata read. #[inline(always)] pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize, valid: usize, ignored: usize, errors: usize) { @@ -603,6 +687,37 @@ pub fn record_get_object_reader_path(path: &'static str) { counter!("rustfs_io_get_object_reader_path_total", "path" => path).increment(1); } +/// Record the selected GetObject reader path with bounded object class and size labels. +#[inline(always)] +pub fn record_get_object_reader_path_by_size(path: &'static str, object_class: &'static str, size_bucket: &'static str) { + if !get_stage_metrics_enabled() { + return; + } + counter!( + "rustfs_io_get_object_reader_path_by_size_total", + "path" => path, + "object_class" => object_class, + "size_bucket" => size_bucket + ) + .increment(1); +} + +/// Record the concrete subpath used by the direct-memory GetObject reader. +#[inline(always)] +pub fn record_get_object_direct_memory_subpath(subpath: &'static str, object_class: &'static str, size_bucket: &'static str) { + if !get_stage_metrics_enabled() { + return; + } + counter!("rustfs_io_get_object_direct_memory_subpath_total", "subpath" => subpath).increment(1); + counter!( + "rustfs_io_get_object_direct_memory_subpath_by_size_total", + "subpath" => subpath, + "object_class" => object_class, + "size_bucket" => size_bucket + ) + .increment(1); +} + /// Record why the codec streaming reader was not selected. #[inline(always)] pub fn record_get_object_codec_streaming_fallback(reason: &'static str) { @@ -627,6 +742,27 @@ pub fn record_get_object_codec_streaming_decision(outcome: &'static str, object_ .increment(1); } +/// Record the final codec-streaming rollout decision with bounded size attribution. +#[inline(always)] +pub fn record_get_object_codec_streaming_decision_by_size( + outcome: &'static str, + object_class: &'static str, + reason: &'static str, + size_bucket: &'static str, +) { + if !get_stage_metrics_enabled() { + return; + } + counter!( + "rustfs_io_get_object_codec_streaming_decision_by_size_total", + "outcome" => outcome, + "object_class" => object_class, + "reason" => reason, + "size_bucket" => size_bucket + ) + .increment(1); +} + /// Record one decoded reader stripe processed by a GetObject read path. #[inline(always)] pub fn record_get_object_reader_stripe(path: &'static str) { @@ -974,6 +1110,66 @@ pub fn record_get_object_shard_read_fanout( histogram!("rustfs_io_get_object_shard_read_failed", "path" => path).record(shard_read_fanout_to_f64(failed)); } +/// Record the bitrot reader setup scheduling strategy selected for a GET read. +#[inline(always)] +pub fn record_get_object_reader_setup_strategy(strategy: &'static str, mode: &'static str) { + if !get_stage_metrics_enabled() { + return; + } + counter!( + "rustfs_io_get_object_reader_setup_strategy_total", + "strategy" => strategy, + "mode" => mode + ) + .increment(1); +} + +/// Record the final bitrot reader setup fanout shape for a GET read. +#[inline(always)] +pub fn record_get_object_reader_setup_fanout( + strategy: &'static str, + mode: &'static str, + scheduled: usize, + attempted: usize, + ready: usize, + failed: usize, + deferred: usize, +) { + if !get_stage_metrics_enabled() { + return; + } + histogram!( + "rustfs_io_get_object_reader_setup_scheduled", + "strategy" => strategy, + "mode" => mode + ) + .record(shard_read_fanout_to_f64(scheduled)); + histogram!( + "rustfs_io_get_object_reader_setup_attempted", + "strategy" => strategy, + "mode" => mode + ) + .record(shard_read_fanout_to_f64(attempted)); + histogram!( + "rustfs_io_get_object_reader_setup_ready", + "strategy" => strategy, + "mode" => mode + ) + .record(shard_read_fanout_to_f64(ready)); + histogram!( + "rustfs_io_get_object_reader_setup_failed", + "strategy" => strategy, + "mode" => mode + ) + .record(shard_read_fanout_to_f64(failed)); + histogram!( + "rustfs_io_get_object_reader_setup_deferred", + "strategy" => strategy, + "mode" => mode + ) + .record(shard_read_fanout_to_f64(deferred)); +} + /// Record GetObject metadata resolution duration. #[inline(always)] pub fn record_get_object_metadata_phase_duration(duration_secs: f64) { @@ -1694,10 +1890,13 @@ mod tests { #[test] fn test_record_get_object_stage_metrics() { record_get_object_stage_duration("s3_handler", "request_context", 0.001); + record_get_object_stage_duration_by_size("legacy_duplex", "metadata", "plain_single_part", "le_4kib", 0.001); record_get_object_reader_path("codec_streaming"); + record_get_object_reader_path_by_size("codec_streaming", "plain_single_part", "le_1mib"); record_get_object_codec_streaming_fallback("range"); record_get_object_codec_streaming_decision("fallback", "range", "range"); record_get_object_codec_streaming_decision("use", "plain_single_part", "none"); + record_get_object_codec_streaming_decision_by_size("fallback", "plain_single_part", "below_min_size", "le_128kib"); record_get_object_reader_stripe("codec_streaming"); record_get_object_reader_bytes("codec_streaming", 1024); record_get_object_reader_buffer("codec_streaming", "output", 1024); @@ -1735,6 +1934,8 @@ mod tests { record_get_object_pipeline_failure_for_path("codec_streaming", "decode", "read_quorum"); record_get_object_shard_read_observation("codec_streaming", 0, "data", "local", "success", "none", 1024, 0.004, 0.001); record_get_object_shard_read_cost_summary("codec_streaming", 3, 1, 2, 0, 4, 4, 4, true); + record_get_object_reader_setup_strategy("data_blocks_first", "read_quorum"); + record_get_object_reader_setup_fanout("data_blocks_first", "read_quorum", 3, 2, 2, 0, 2); assert!(0.005_f64.is_sign_positive()); } @@ -1750,6 +1951,7 @@ mod tests { record_get_object_reader_prefetch_bytes("codec_streaming", "single_inflight", 4096); record_get_object_reader_stream_buffer_size("standard", "selected", 131072); record_get_object_reader_stream_poll("standard", "selected", "ready_data", 8192, 4096, 0.0002); + record_get_object_memory_body_stream_poll("buffered_body", "ready_data", 4096, 0.0001); assert!(0.0003_f64.is_sign_positive()); } @@ -1794,8 +1996,11 @@ mod tests { let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); set_get_stage_metrics_enabled(true); record_get_object_stage_duration("s3_handler", "request_context", 0.001); + record_get_object_stage_duration_by_size("legacy_duplex", "metadata", "plain_single_part", "le_4kib", 0.001); record_get_object_reader_path("codec_streaming"); + record_get_object_reader_path_by_size("codec_streaming", "plain_single_part", "le_1mib"); record_get_object_codec_streaming_fallback("range"); + record_get_object_codec_streaming_decision_by_size("fallback", "plain_single_part", "below_min_size", "le_128kib"); record_get_object_reader_stripe("codec_streaming"); record_get_object_reader_bytes("codec_streaming", 1024); record_get_object_reader_buffer("codec_streaming", "output", 1024); @@ -1838,8 +2043,11 @@ mod tests { let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); set_get_stage_metrics_enabled(false); record_get_object_stage_duration("s3_handler", "request_context", 0.001); + record_get_object_stage_duration_by_size("legacy_duplex", "metadata", "plain_single_part", "le_4kib", 0.001); record_get_object_reader_path("codec_streaming"); + record_get_object_reader_path_by_size("codec_streaming", "plain_single_part", "le_1mib"); record_get_object_codec_streaming_fallback("range"); + record_get_object_codec_streaming_decision_by_size("fallback", "plain_single_part", "below_min_size", "le_128kib"); record_get_object_reader_stripe("codec_streaming"); record_get_object_reader_bytes("codec_streaming", 1024); record_get_object_reader_buffer("codec_streaming", "output", 1024); @@ -1879,6 +2087,22 @@ mod tests { assert!(!get_stage_metrics_enabled()); } + #[test] + fn test_get_object_size_buckets_match_issue714_matrix() { + assert_eq!(get_object_size_bucket(0), GET_OBJECT_SIZE_BUCKET_LE_4_KIB); + assert_eq!(get_object_size_bucket(1024), GET_OBJECT_SIZE_BUCKET_LE_4_KIB); + assert_eq!(get_object_size_bucket(4096), GET_OBJECT_SIZE_BUCKET_LE_4_KIB); + assert_eq!(get_object_size_bucket(4097), GET_OBJECT_SIZE_BUCKET_LE_16_KIB); + assert_eq!(get_object_size_bucket(10 * 1024), GET_OBJECT_SIZE_BUCKET_LE_16_KIB); + assert_eq!(get_object_size_bucket(16 * 1024), GET_OBJECT_SIZE_BUCKET_LE_16_KIB); + assert_eq!(get_object_size_bucket((16 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_64_KIB); + assert_eq!(get_object_size_bucket(100 * 1024), GET_OBJECT_SIZE_BUCKET_LE_128_KIB); + assert_eq!(get_object_size_bucket(128 * 1024), GET_OBJECT_SIZE_BUCKET_LE_128_KIB); + assert_eq!(get_object_size_bucket((128 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_1_MIB); + assert_eq!(get_object_size_bucket(1024 * 1024), GET_OBJECT_SIZE_BUCKET_LE_1_MIB); + assert_eq!(get_object_size_bucket((1024 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_GT_1_MIB); + } + #[test] fn test_record_stage_duration_generic() { // Generic stage duration should always record (no gating flag) diff --git a/crates/obs/src/telemetry/otel.rs b/crates/obs/src/telemetry/otel.rs index ec36ced24..f2688f420 100644 --- a/crates/obs/src/telemetry/otel.rs +++ b/crates/obs/src/telemetry/otel.rs @@ -54,7 +54,7 @@ use opentelemetry_otlp::{Compression, Protocol, WithExportConfig, WithHttpConfig use opentelemetry_sdk::propagation::{BaggagePropagator, TraceContextPropagator}; use opentelemetry_sdk::{ logs::SdkLoggerProvider, - metrics::{PeriodicReader, SdkMeterProvider}, + metrics::{Aggregation, Instrument, PeriodicReader, SdkMeterProvider, Stream}, trace::{RandomIdGenerator, Sampler, SdkTracerProvider}, }; use percent_encoding::percent_decode_str; @@ -70,6 +70,19 @@ use tracing_error::ErrorLayer; use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer}; use tracing_subscriber::{Layer, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt}; +const GET_OBJECT_DURATION_HISTOGRAM_METRICS: &[&str] = &[ + "rustfs_io_get_object_request_duration_seconds", + "rustfs_io_get_object_total_duration_seconds", + "rustfs_io_get_object_total_duration_seconds_with_path", + "rustfs_io_get_object_stage_duration_seconds", + "rustfs_io_get_object_stage_duration_seconds_by_size", +]; + +const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[ + 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002, 0.003, 0.004, 0.005, 0.0075, 0.01, 0.015, 0.02, 0.03, 0.05, 0.075, + 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, +]; + /// Initialize the full OpenTelemetry HTTP pipeline (traces + metrics + logs). /// /// This function is invoked when at least one OTLP endpoint has been @@ -418,11 +431,14 @@ fn build_meter_provider( let (provider, recorder) = Recorder::builder(service_name.to_string()) .with_meter_provider(|b: opentelemetry_sdk::metrics::MeterProviderBuilder| { - let b = b.with_resource(res).with_reader( - PeriodicReader::builder(exporter) - .with_interval(Duration::from_secs(meter_interval)) - .build(), - ); + let b = b + .with_resource(res) + .with_reader( + PeriodicReader::builder(exporter) + .with_interval(Duration::from_secs(meter_interval)) + .build(), + ) + .with_view(get_object_duration_histogram_view); if use_stdout { b.with_reader(create_periodic_reader(meter_interval)) } else { @@ -437,6 +453,24 @@ fn build_meter_provider( Ok(Some(provider)) } +fn get_object_duration_histogram_view(instrument: &Instrument) -> Option { + if !is_get_object_duration_histogram_metric(instrument.name()) { + return None; + } + + Stream::builder() + .with_aggregation(Aggregation::ExplicitBucketHistogram { + boundaries: GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.to_vec(), + record_min_max: true, + }) + .build() + .ok() +} + +fn is_get_object_duration_histogram_metric(name: &str) -> bool { + GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) +} + /// Build an optional [`SdkLoggerProvider`] for the given log endpoint. /// /// Returns `None` when the endpoint is empty or log export is disabled. @@ -687,4 +721,19 @@ mod tests { assert_eq!(resolve_signal_timeout(Some(0), None), None); assert_eq!(resolve_signal_timeout(None, Some(0)), None); } + + #[test] + fn test_get_object_duration_histogram_metric_match_is_scoped() { + assert!(is_get_object_duration_histogram_metric("rustfs_io_get_object_stage_duration_seconds")); + assert!(is_get_object_duration_histogram_metric("rustfs_io_get_object_request_duration_seconds")); + assert!(!is_get_object_duration_histogram_metric("rustfs_io_put_object_request_duration_seconds")); + assert!(!is_get_object_duration_histogram_metric("rustfs_io_get_object_response_size_bytes")); + } + + #[test] + fn test_get_object_duration_histogram_buckets_are_sorted() { + assert!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.windows(2).all(|pair| pair[0] < pair[1])); + assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.first(), Some(&0.0001)); + assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.last(), Some(&10.0)); + } } diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 9ddd95753..d5d3af617 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1215,6 +1215,7 @@ mod tests { Ok(GetObjectReader { stream: Box::new(Cursor::new(data)), object_info: ObjectInfo::default(), + buffered_body: None, }) } diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 8b8659a09..1794b1a6f 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -191,10 +191,32 @@ const LOG_SUBSYSTEM_OBJECT: &str = "object"; const EVENT_PUT_OBJECT_STORE_INFLIGHT_SLOW: &str = "put_object_store_inflight_slow"; const EVENT_PUT_OBJECT_STORE_RETURNED: &str = "put_object_store_returned"; const EVENT_GET_OBJECT_STREAM_BODY: &str = "get_object_stream_body"; +const GET_OBJECT_STAGE_PATH_S3_HANDLER: &str = "s3_handler"; +const GET_OBJECT_STAGE_REQUEST_INGRESS_TO_CONTEXT: &str = "request_ingress_to_context"; +const GET_OBJECT_STAGE_OUTPUT_STRATEGY: &str = "output_strategy"; +const GET_OBJECT_STAGE_BODY_BUILD: &str = "body_build"; +const GET_OBJECT_STAGE_BODY_ENCRYPTED_BUFFER_READ: &str = "body_encrypted_buffer_read"; +const GET_OBJECT_STAGE_BODY_MEMORY_BLOB: &str = "body_memory_blob"; +const GET_OBJECT_STAGE_BODY_SEEK_BUFFER_READ: &str = "body_seek_buffer_read"; +const GET_OBJECT_STAGE_BODY_STREAM_STRATEGY: &str = "body_stream_strategy"; +const GET_OBJECT_STAGE_BODY_STREAMING_BLOB: &str = "body_streaming_blob"; +const GET_OBJECT_STAGE_CHECKSUM_HEADERS: &str = "checksum_headers"; +const GET_OBJECT_STAGE_LIFECYCLE_EXPIRATION: &str = "lifecycle_expiration"; +const GET_OBJECT_STAGE_METADATA_FILTER: &str = "metadata_filter"; const PUT_OBJECT_STORE_WARN_THRESHOLD: Duration = Duration::from_secs(5); const GET_OBJECT_STREAM_WARN_THRESHOLD: Duration = Duration::from_secs(5); static GET_OBJECT_BUFFER_THRESHOLD_WARNED: AtomicBool = AtomicBool::new(false); +fn record_get_object_s3_handler_stage_duration(stage: &'static str, start: Option) { + if let Some(start) = start { + rustfs_io_metrics::record_get_object_stage_duration( + GET_OBJECT_STAGE_PATH_S3_HANDLER, + stage, + start.elapsed().as_secs_f64(), + ); + } +} + fn decoded_content_length_from_headers(headers: &HeaderMap) -> S3Result> { let Some(val) = headers.get(AMZ_DECODED_CONTENT_LENGTH) else { return Ok(None); @@ -235,6 +257,23 @@ impl DeadlockRequestGuard { request_id, } } + + fn register_if_enabled( + deadlock_detector: Arc, + request_id: &str, + description: F, + ) -> Option + where + F: FnOnce() -> String, + { + if !deadlock_detector.is_enabled() { + return None; + } + + let request_id = request_id.to_string(); + deadlock_detector.register_request(&request_id, description()); + Some(Self::new(deadlock_detector, request_id)) + } } impl Drop for DeadlockRequestGuard { @@ -248,7 +287,7 @@ struct GetObjectBootstrap { wrapper: RequestTimeoutWrapper, request_start: std::time::Instant, request_guard: GetObjectGuard, - _deadlock_request_guard: DeadlockRequestGuard, + _deadlock_request_guard: Option, concurrent_requests: usize, } @@ -273,6 +312,7 @@ struct GetObjectReadSetup { info: ObjectInfo, event_info: ObjectInfo, final_stream: DynReader, + buffered_body: Option, rs: Option, content_type: Option, last_modified: Option, @@ -332,6 +372,7 @@ const LARGE_SEQUENTIAL_GET_STREAM_BUFFER_CAP_BYTES: usize = 4 * MI_B; const LARGE_SEQUENTIAL_GET_READAHEAD_MULTIPLIER: usize = 2; const LARGE_BODY_READER_STREAM_BUFFER_FLOOR_BYTES: usize = MI_B; const LARGE_BODY_READER_STREAM_BUFFER_THRESHOLD_BYTES: i64 = 4 * MI_B as i64; +const ENV_RUSTFS_GET_SEEK_BUFFER_ENABLE: &str = "RUSTFS_GET_SEEK_BUFFER_ENABLE"; const ENV_RUSTFS_GET_READER_STREAM_BUFFER_SIZE: &str = "RUSTFS_GET_READER_STREAM_BUFFER_SIZE"; const ENV_RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE: &str = "RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE"; const GET_READER_STREAM_BUFFER_SOURCE_SELECTED: &str = "selected"; @@ -340,6 +381,9 @@ const GET_READER_STREAM_POLL_PENDING: &str = "pending"; const GET_READER_STREAM_POLL_READY_DATA: &str = "ready_data"; const GET_READER_STREAM_POLL_READY_EMPTY: &str = "ready_empty"; const GET_READER_STREAM_POLL_READY_ERROR: &str = "ready_error"; +const GET_MEMORY_BODY_SOURCE_BUFFERED_BODY: &str = "buffered_body"; +const GET_MEMORY_BODY_SOURCE_SEEK_BUFFER: &str = "seek_buffer"; +const GET_MEMORY_BODY_SOURCE_ENCRYPTED_BUFFER: &str = "encrypted_buffer"; fn get_reader_stream_buffer_size_override() -> Option { static GET_READER_STREAM_BUFFER_SIZE_OVERRIDE: OnceLock> = OnceLock::new(); @@ -356,6 +400,11 @@ fn is_get_output_handoff_attribution_enabled() -> bool { *ENABLED.get_or_init(|| rustfs_utils::get_env_bool(ENV_RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE, false)) } +fn is_get_seek_buffer_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SEEK_BUFFER_ENABLE, false)) +} + fn resolve_reader_stream_buffer_size(selected_size: usize, override_size: Option) -> (usize, &'static str) { if let Some(override_size) = override_size.filter(|value| *value > 0) { return (override_size, GET_READER_STREAM_BUFFER_SOURCE_ENV_OVERRIDE); @@ -437,6 +486,8 @@ pin_project! { struct MemoryTrackedBytesStream { bytes: Bytes, emitted: bool, + started: std::time::Instant, + source: &'static str, _guard: Option, } } @@ -483,10 +534,12 @@ pin_project! { } impl MemoryTrackedBytesStream { - fn new(bytes: Bytes, guard: Option) -> Self { + fn new(bytes: Bytes, source: &'static str, guard: Option) -> Self { Self { bytes, emitted: false, + started: std::time::Instant::now(), + source, _guard: guard, } } @@ -516,11 +569,32 @@ impl futures::Stream for MemoryTrackedBytesStream { fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { let this = self.project(); + let poll_start = is_get_output_handoff_attribution_enabled().then(std::time::Instant::now); if *this.emitted { + if let Some(poll_start) = poll_start { + rustfs_io_metrics::record_get_object_memory_body_stream_poll( + this.source, + GET_READER_STREAM_POLL_READY_EMPTY, + 0, + poll_start.elapsed().as_secs_f64(), + ); + } return Poll::Ready(None); } + let first_byte_elapsed = (!this.bytes.is_empty()).then(|| this.started.elapsed()); *this.emitted = true; + if let Some(elapsed) = first_byte_elapsed { + rustfs_io_metrics::record_get_object_first_byte_latency(GET_OBJECT_STAGE_PATH_S3_HANDLER, elapsed.as_secs_f64()); + } + if let Some(poll_start) = poll_start { + rustfs_io_metrics::record_get_object_memory_body_stream_poll( + this.source, + GET_READER_STREAM_POLL_READY_DATA, + this.bytes.len(), + poll_start.elapsed().as_secs_f64(), + ); + } Poll::Ready(Some(Ok(this.bytes.clone()))) } } @@ -662,6 +736,10 @@ impl AsyncRead for GetObjectStreamingReader { if !self.first_byte_reported { self.first_byte_reported = true; let elapsed = self.elapsed(); + rustfs_io_metrics::record_get_object_first_byte_latency( + GET_OBJECT_STAGE_PATH_S3_HANDLER, + elapsed.as_secs_f64(), + ); if elapsed >= GET_OBJECT_STREAM_WARN_THRESHOLD { warn!( event = EVENT_GET_OBJECT_STREAM_BODY, @@ -1149,6 +1227,7 @@ fn should_buffer_get_object_in_memory( has_range, configured_threshold, concurrent_requests, + is_get_seek_buffer_enabled(), ) } @@ -1159,8 +1238,12 @@ fn should_buffer_get_object_in_memory_with_threshold( has_range: bool, configured_threshold: i64, concurrent_requests: usize, + seek_buffer_enabled: bool, ) -> bool { - if part_number.is_some() || has_range || response_content_length <= 0 || configured_threshold <= 0 { + if !seek_buffer_enabled || part_number.is_some() || has_range || response_content_length <= 0 || configured_threshold <= 0 { + return false; + } + if usize::try_from(response_content_length).is_err() { return false; } @@ -1188,6 +1271,8 @@ fn should_buffer_get_object_in_memory_with_threshold( mod deadlock_request_guard_tests { use super::DeadlockRequestGuard; use crate::app::storage_api::object_usecase::deadlock_detector::{DeadlockDetector, RequestHangDetectionPolicy}; + use std::cell::Cell; + use std::rc::Rc; use std::sync::Arc; #[test] @@ -1208,6 +1293,24 @@ mod deadlock_request_guard_tests { assert_eq!(detector.tracked_count(), 0); } + + #[test] + fn deadlock_request_guard_skips_disabled_detector() { + let detector = Arc::new(DeadlockDetector::new(RequestHangDetectionPolicy { + enabled: false, + ..RequestHangDetectionPolicy::default() + })); + let description_built = Rc::new(Cell::new(false)); + let description_built_for_closure = Rc::clone(&description_built); + + let guard = DeadlockRequestGuard::register_if_enabled(detector, "test-request-id", || { + description_built_for_closure.set(true); + "test request".to_string() + }); + + assert!(guard.is_none()); + assert!(!description_built.get()); + } } async fn maybe_enqueue_transition_immediate(obj_info: &ObjectInfo, src: LcEventSrc) { @@ -1970,13 +2073,39 @@ impl DefaultObjectUsecase { } } - fn build_memory_blob(buf: Vec, response_content_length: i64, _optimal_buffer_size: usize) -> Option { - let guard = rustfs_io_metrics::track_get_object_buffered_bytes(buf.len()); - let bytes = Bytes::from(buf); - Some(StreamingBlob::wrap(bytes_stream( - MemoryTrackedBytesStream::new(bytes, guard), - response_content_length as usize, - ))) + fn build_memory_bytes_blob( + bytes: Bytes, + response_content_length: i64, + _optimal_buffer_size: usize, + source: &'static str, + ) -> Option { + let get_stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let memory_blob_start = get_stage_metrics_enabled.then(std::time::Instant::now); + let handoff_start = get_stage_metrics_enabled.then(std::time::Instant::now); + let bytes_len = bytes.len(); + let guard = rustfs_io_metrics::track_get_object_buffered_bytes(bytes_len); + let remaining = usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX); + let blob = StreamingBlob::wrap(bytes_stream(MemoryTrackedBytesStream::new(bytes, source, guard), remaining)); + if let Some(handoff_start) = handoff_start { + rustfs_io_metrics::record_get_object_response_handoff( + "single_chunk", + source, + bytes_len, + response_content_length, + handoff_start.elapsed().as_secs_f64(), + ); + } + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_MEMORY_BLOB, memory_blob_start); + Some(blob) + } + + fn build_memory_blob( + buf: Vec, + response_content_length: i64, + optimal_buffer_size: usize, + source: &'static str, + ) -> Option { + Self::build_memory_bytes_blob(Bytes::from(buf), response_content_length, optimal_buffer_size, source) } fn select_stream_buffer_strategy( @@ -2007,6 +2136,7 @@ impl DefaultObjectUsecase { where R: AsyncRead + Send + Sync + Unpin + 'static, { + let streaming_blob_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let expected = usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX); let tuned_stream_buffer_size = tune_reader_stream_buffer_size(stream_buffer_size, response_content_length, stream_strategy); @@ -2033,20 +2163,21 @@ impl DefaultObjectUsecase { handoff_start.elapsed().as_secs_f64(), ); } + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAMING_BLOB, streaming_blob_start); Some(blob) } fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result { - let timeout_config = GetObjectTimeoutPolicy::from_env(); + let timeout_config = GetObjectTimeoutPolicy::cached_from_env(); let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.to_string()); let request_start = std::time::Instant::now(); let request_guard = ConcurrencyManager::track_request(); let concurrent_requests = GetObjectGuard::concurrent_requests(); let deadlock_detector = deadlock_detector::get_deadlock_detector(); - let request_id = wrapper.request_id().to_string(); - deadlock_detector.register_request(&request_id, format!("GetObject {bucket}/{key}")); - let deadlock_request_guard = DeadlockRequestGuard::new(deadlock_detector, request_id); + let deadlock_request_guard = DeadlockRequestGuard::register_if_enabled(deadlock_detector, wrapper.request_id(), || { + format!("GetObject {bucket}/{key}") + }); Self::ensure_get_object_not_timed_out(&wrapper, &timeout_config, bucket, key, GetObjectTimeoutStage::BeforeProcessing)?; @@ -2255,6 +2386,8 @@ impl DefaultObjectUsecase { } let info = reader.object_info; + let stream = reader.stream; + let buffered_body = reader.buffered_body; let read_duration = read_start.elapsed(); @@ -2342,6 +2475,7 @@ impl DefaultObjectUsecase { ssekms_key_id, encryption_applied, final_stream, + buffered_body, ) = match sse_decryption(decryption_request).await? { Some(material) => { let server_side_encryption = Some(material.server_side_encryption.clone()); @@ -2353,10 +2487,11 @@ impl DefaultObjectUsecase { sse_customer_key_md5, material.kms_key_id, true, - wrap_reader(reader.stream), + wrap_reader(stream), + None, ) } - None => (None, None, None, None, false, wrap_reader(reader.stream)), + None => (None, None, None, None, false, wrap_reader(stream), buffered_body), }; // Detect inline fast path: data is in memory, no disk I/O semaphore needed. @@ -2367,6 +2502,7 @@ impl DefaultObjectUsecase { info, event_info, final_stream, + buffered_body, rs, content_type, last_modified, @@ -2536,6 +2672,7 @@ impl DefaultObjectUsecase { part_number: Option, has_range: bool, encryption_applied: bool, + buffered_body: Option, bucket: &str, key: &str, ) -> S3Result> @@ -2548,7 +2685,10 @@ impl DefaultObjectUsecase { if should_buffer_encrypted_object { let mut buf = Vec::with_capacity(response_content_length as usize); - if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { + let buffer_read_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let read_result = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_ENCRYPTED_BUFFER_READ, buffer_read_start); + if let Err(e) = read_result { error!(error = %e, "GetObject decrypted object buffering failed"); return Err(ApiError::from(StorageError::other(format!("Failed to read decrypted object: {e}"))).into()); } @@ -2561,12 +2701,19 @@ impl DefaultObjectUsecase { ); } - return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size)); + return Ok(Self::build_memory_blob( + buf, + response_content_length, + optimal_buffer_size, + GET_MEMORY_BODY_SOURCE_ENCRYPTED_BUFFER, + )); } debug!(buffer_size = optimal_buffer_size, "Encrypted object uses streaming decrypt path"); + let stream_strategy_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let (stream_buffer_size, stream_strategy) = Self::select_stream_buffer_strategy(response_content_length, optimal_buffer_size, enable_readahead, has_range); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAM_STRATEGY, stream_strategy_start); return Ok(Self::build_reader_blob( final_stream, response_content_length, @@ -2577,12 +2724,32 @@ impl DefaultObjectUsecase { )); } + if let Some(buffered_body) = buffered_body { + if buffered_body.len() != usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX) { + warn!( + expected = response_content_length, + actual = buffered_body.len(), + "Buffered GetObject body size mismatch" + ); + } + + return Ok(Self::build_memory_bytes_blob( + buffered_body, + response_content_length, + optimal_buffer_size, + GET_MEMORY_BODY_SOURCE_BUFFERED_BODY, + )); + } + let should_provide_seek_support = should_buffer_get_object_in_memory(info, response_content_length, part_number, has_range, concurrent_requests); if should_provide_seek_support { let mut buf = Vec::with_capacity(response_content_length as usize); - match tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { + let buffer_read_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let read_result = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_SEEK_BUFFER_READ, buffer_read_start); + match read_result { Ok(_) => { if buf.len() != response_content_length as usize { warn!( @@ -2592,7 +2759,12 @@ impl DefaultObjectUsecase { ); } - return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size)); + return Ok(Self::build_memory_blob( + buf, + response_content_length, + optimal_buffer_size, + GET_MEMORY_BODY_SOURCE_SEEK_BUFFER, + )); } Err(e) => { error!(error = %e, "GetObject seek-support buffering failed"); @@ -2600,8 +2772,10 @@ impl DefaultObjectUsecase { } } + let stream_strategy_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let (stream_buffer_size, stream_strategy) = Self::select_stream_buffer_strategy(response_content_length, optimal_buffer_size, enable_readahead, has_range); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAM_STRATEGY, stream_strategy_start); Ok(Self::build_reader_blob( final_stream, response_content_length, @@ -3297,6 +3471,7 @@ impl DefaultObjectUsecase { info: ObjectInfo, event_info: ObjectInfo, final_stream: DynReader, + buffered_body: Option, rs: Option, content_type: Option, last_modified: Option, @@ -3314,6 +3489,7 @@ impl DefaultObjectUsecase { part_number: Option, versioned: bool, ) -> S3Result { + let strategy_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let strategy = self.finalize_get_object_strategy( manager, bucket, @@ -3326,12 +3502,14 @@ impl DefaultObjectUsecase { queue_status, concurrent_requests, ); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_OUTPUT_STRATEGY, strategy_start); let GetObjectStrategyContext { io_strategy: _, optimal_buffer_size, enable_readahead, } = strategy; + let body_build_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let body = Self::build_get_object_body( final_stream, &info, @@ -3342,12 +3520,16 @@ impl DefaultObjectUsecase { part_number, rs.is_some(), encryption_applied, + buffered_body, bucket, key, ) .await?; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_BUILD, body_build_start); + let checksum_headers_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let checksums = Self::build_get_object_checksums(&info, &req.headers, part_number, rs.as_ref())?; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_CHECKSUM_HEADERS, checksum_headers_start); let output_version_id = if versioned { info.version_id.map(|vid| { @@ -3368,10 +3550,16 @@ impl DefaultObjectUsecase { }); // x-amz-expiration: predict from lifecycle configuration + let lifecycle_expiration_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let expiration = resolve_put_object_expiration(bucket, &info).await; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_LIFECYCLE_EXPIRATION, lifecycle_expiration_start); let storage_class = response_storage_class(&info, &info.user_defined); let content_disposition = info.user_defined.get("content-disposition").cloned(); + let metadata_filter_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let metadata = filter_object_metadata(&info.user_defined); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_METADATA_FILTER, metadata_filter_start); + let output = GetObjectOutput { body, content_length: Some(response_content_length), @@ -3382,7 +3570,7 @@ impl DefaultObjectUsecase { accept_ranges: Some(ACCEPT_RANGES_BYTES.to_string()), content_range, e_tag: info.etag.map(|etag| to_s3s_etag(&etag)), - metadata: filter_object_metadata(&info.user_defined), + metadata, server_side_encryption, sse_customer_algorithm, sse_customer_key_md5, @@ -3418,11 +3606,19 @@ impl DefaultObjectUsecase { let _ = context.object_store(); } - let request_id = req - .extensions - .get::() + let inbound_request_context = req.extensions.get::(); + let request_id = inbound_request_context .map(|ctx| ctx.request_id.clone()) .unwrap_or_else(|| request_context::RequestContext::fallback().request_id); + if rustfs_io_metrics::get_stage_metrics_enabled() + && let Some(context) = inbound_request_context + { + rustfs_io_metrics::record_get_object_stage_duration( + GET_OBJECT_STAGE_PATH_S3_HANDLER, + GET_OBJECT_STAGE_REQUEST_INGRESS_TO_CONTEXT, + context.start_time.elapsed().as_secs_f64(), + ); + } let bootstrap = Self::init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?; let timeout_config = bootstrap.timeout_config; let wrapper = bootstrap.wrapper; @@ -3477,6 +3673,7 @@ impl DefaultObjectUsecase { info, event_info, final_stream, + buffered_body, rs, content_type, last_modified, @@ -3495,15 +3692,6 @@ impl DefaultObjectUsecase { final_stream }; - let versioning_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); - let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; - if let Some(versioning_start) = versioning_start { - rustfs_io_metrics::record_get_object_stage_duration( - "s3_handler", - "versioning_lookup", - versioning_start.elapsed().as_secs_f64(), - ); - } let output_build_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let output_context = self .build_get_object_output_context( @@ -3514,6 +3702,7 @@ impl DefaultObjectUsecase { info, event_info, final_stream, + buffered_body, rs, content_type, last_modified, @@ -3529,7 +3718,7 @@ impl DefaultObjectUsecase { &queue_status, concurrent_requests, part_number, - versioned, + opts.versioned, ) .await?; if let Some(output_build_start) = output_build_start { @@ -6007,7 +6196,7 @@ mod tests { let configured_threshold = 20_i64 * 1024 * 1024 * 1024; let response_len = 80_i64 * 1024 * 1024; let should_buffer = - should_buffer_get_object_in_memory_with_threshold(&info, response_len, None, false, configured_threshold, 1); + should_buffer_get_object_in_memory_with_threshold(&info, response_len, None, false, configured_threshold, 1, true); assert!( !should_buffer, @@ -6026,7 +6215,8 @@ mod tests { None, false, configured_threshold, - 1 + 1, + true )); assert!(!should_buffer_get_object_in_memory_with_threshold( &info, @@ -6034,7 +6224,8 @@ mod tests { Some(1), false, configured_threshold, - 1 + 1, + true )); assert!(!should_buffer_get_object_in_memory_with_threshold( &info, @@ -6042,7 +6233,24 @@ mod tests { None, true, configured_threshold, - 1 + 1, + true + )); + } + + #[test] + fn should_buffer_get_object_in_memory_requires_seek_buffer_opt_in() { + let info = ObjectInfo::default(); + let configured_threshold = 10_i64 * 1024 * 1024; + + assert!(!should_buffer_get_object_in_memory_with_threshold( + &info, + 1024, + None, + false, + configured_threshold, + 1, + false )); } @@ -6057,7 +6265,8 @@ mod tests { None, false, configured_threshold, - 1 + 1, + true )); assert!(!should_buffer_get_object_in_memory_with_threshold( &info, @@ -6065,7 +6274,8 @@ mod tests { None, false, configured_threshold, - 1 + 1, + true )); } @@ -6080,7 +6290,8 @@ mod tests { None, false, configured_threshold, - 1 + 1, + true )); assert!(!should_buffer_get_object_in_memory_with_threshold( &info, @@ -6088,9 +6299,10 @@ mod tests { None, false, configured_threshold, - 1 + 1, + true )); - assert!(!should_buffer_get_object_in_memory_with_threshold(&info, 1024, None, false, 0, 1)); + assert!(!should_buffer_get_object_in_memory_with_threshold(&info, 1024, None, false, 0, 1, true)); } #[test] @@ -6104,7 +6316,8 @@ mod tests { None, false, configured_threshold, - 1 + 1, + true )); assert!(!should_buffer_get_object_in_memory_with_threshold( &info, @@ -6112,7 +6325,8 @@ mod tests { None, false, configured_threshold, - 32 + 32, + true )); assert!(should_buffer_get_object_in_memory_with_threshold( &info, @@ -6120,7 +6334,8 @@ mod tests { None, false, configured_threshold, - rustfs_config::DEFAULT_OBJECT_HIGH_CONCURRENCY_THRESHOLD + rustfs_config::DEFAULT_OBJECT_HIGH_CONCURRENCY_THRESHOLD, + true )); } @@ -6194,6 +6409,7 @@ mod tests { None, false, false, + None, "test-bucket", "large-object", ) @@ -6229,6 +6445,7 @@ mod tests { None, false, true, + None, "test-bucket", "large-encrypted-object", ) @@ -6243,6 +6460,78 @@ mod tests { ); } + #[tokio::test] + async fn build_get_object_body_uses_buffered_body_without_reader_preread() { + let reads = Arc::new(AtomicUsize::new(0)); + let reader = ReadProbeReader { + reads: Arc::clone(&reads), + }; + let info = ObjectInfo { + size: 4, + ..Default::default() + }; + + let body = DefaultObjectUsecase::build_get_object_body( + reader, + &info, + 4, + 128 * 1024, + false, + 1, + None, + false, + false, + Some(Bytes::from_static(b"test")), + "test-bucket", + "direct-memory-object", + ) + .await + .expect("build_get_object_body should consume buffered body"); + + assert!(body.is_some()); + assert_eq!( + reads.load(AtomicOrdering::Relaxed), + 0, + "buffered GetObject body must not be read from the fallback reader" + ); + } + + #[tokio::test] + async fn build_get_object_body_keeps_small_plain_objects_on_streaming_path_by_default() { + let reads = Arc::new(AtomicUsize::new(0)); + let reader = ReadProbeReader { + reads: Arc::clone(&reads), + }; + let info = ObjectInfo { + size: 4, + ..Default::default() + }; + + let body = DefaultObjectUsecase::build_get_object_body( + reader, + &info, + 4, + 128 * 1024, + false, + 1, + None, + false, + false, + None, + "test-bucket", + "small-plain-object", + ) + .await + .expect("build_get_object_body should keep small plain object on streaming path"); + + assert!(body.is_some()); + assert_eq!( + reads.load(AtomicOrdering::Relaxed), + 0, + "default GetObject response construction should not pre-read small plain object data" + ); + } + #[test] fn select_stream_buffer_strategy_expands_large_sequential_gets() { let (buffer_size, strategy) = @@ -6887,6 +7176,7 @@ mod tests { None, None, None, + None, 0, None, None, diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 01c1152e1..278539d58 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -20,7 +20,8 @@ use rustfs_filemeta::ReplicationStatusType; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_FORCE_DELETE, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, get_header, - insert_header_map, is_encryption_metadata_key, is_internal_key, + insert_header_map, + metadata_compat::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX}, }; use rustfs_utils::http::{ AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, @@ -493,6 +494,49 @@ pub fn extract_metadata_from_mime_with_object_name( } } +fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool { + value + .get(..prefix.len()) + .is_some_and(|head| head.eq_ignore_ascii_case(prefix)) +} + +fn should_skip_object_metadata_key(key: &str, value: &str, excluded_headers: &[&str]) -> bool { + const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-"; + const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-"; + const X_AMZ_PREFIX: &str = "x-amz-"; + + // Skip internal/reserved metadata (x-rustfs-internal-* or x-minio-internal-*) + if starts_with_ignore_ascii_case(key, RUSTFS_INTERNAL_PREFIX) || starts_with_ignore_ascii_case(key, MINIO_INTERNAL_PREFIX) { + return true; + } + + // Skip internal encryption metadata (x-rustfs-encryption-* or x-minio-encryption-*) + if starts_with_ignore_ascii_case(key, RUSTFS_ENCRYPTION_PREFIX) || starts_with_ignore_ascii_case(key, MINIO_ENCRYPTION_PREFIX) + { + return true; + } + + // Skip empty object lock values + if value.is_empty() + && (key.eq_ignore_ascii_case(X_AMZ_OBJECT_LOCK_MODE.as_str()) + || key.eq_ignore_ascii_case(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str())) + { + return true; + } + + if key.eq_ignore_ascii_case(AMZ_META_UNENCRYPTED_CONTENT_MD5) || key.eq_ignore_ascii_case(AMZ_META_UNENCRYPTED_CONTENT_LENGTH) + { + return true; + } + + if excluded_headers.iter().any(|excluded| key.eq_ignore_ascii_case(excluded)) { + return true; + } + + // User metadata is stored without the x-amz-meta- prefix by extract_metadata_from_mime. + starts_with_ignore_ascii_case(key, X_AMZ_PREFIX) +} + pub(crate) fn filter_object_metadata(metadata: &HashMap) -> Option> { // HTTP headers that should NOT be returned in the Metadata field. // These headers are returned as separate response headers, not user metadata. @@ -513,48 +557,19 @@ pub(crate) fn filter_object_metadata(metadata: &HashMap) -> Opti "x-amz-server-side-encryption-aws-kms-key-id", ]; - let mut filtered_metadata = HashMap::new(); + let mut filtered_metadata = None; for (k, v) in metadata { - let lower_key = k.to_ascii_lowercase(); - // Skip internal/reserved metadata (x-rustfs-internal-* or x-minio-internal-*) - if is_internal_key(&lower_key) { - continue; - } - - // Skip internal encryption metadata (x-rustfs-encryption-* or x-minio-encryption-*) - if is_encryption_metadata_key(&lower_key) { - continue; - } - - // Skip empty object lock values - if v.is_empty() && (k == &X_AMZ_OBJECT_LOCK_MODE.to_string() || k == &X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string()) { - continue; - } - - // Skip UNENCRYPTED metadata placeholders - if k == AMZ_META_UNENCRYPTED_CONTENT_MD5 || k == AMZ_META_UNENCRYPTED_CONTENT_LENGTH { - continue; - } - - // Skip excluded HTTP headers (they are returned as separate headers, not metadata) - if EXCLUDED_HEADERS.contains(&lower_key.as_str()) { - continue; - } - - // Skip any x-amz-* headers that are not user metadata - // User metadata was stored WITHOUT the x-amz-meta- prefix by extract_metadata_from_mime - if lower_key.starts_with("x-amz-") { + if should_skip_object_metadata_key(k, v, EXCLUDED_HEADERS) { continue; } // Include user-defined metadata (keys like "meta1", "custom-key", etc.) - filtered_metadata.insert(k.clone(), v.clone()); - } - if filtered_metadata.is_empty() { - None - } else { - Some(filtered_metadata) + filtered_metadata + .get_or_insert_with(HashMap::new) + .insert(k.clone(), v.clone()); } + + filtered_metadata } /// Detects content type from object name based on file extension. @@ -1504,6 +1519,21 @@ mod tests { assert!(filtered.is_none(), "content-type must not be exposed as user metadata"); } + #[test] + fn test_filter_object_metadata_excludes_case_insensitive_system_headers() { + let mut metadata = HashMap::new(); + metadata.insert("Content-Type".to_string(), "application/octet-stream".to_string()); + metadata.insert("X-Amz-Storage-Class".to_string(), "STANDARD".to_string()); + metadata.insert("X-RustFS-Internal-Healing".to_string(), "true".to_string()); + metadata.insert("X-Minio-Encryption-Iv".to_string(), "secret".to_string()); + metadata.insert("custom-key".to_string(), "custom-value".to_string()); + + let filtered = filter_object_metadata(&metadata).expect("user metadata should remain"); + + assert_eq!(filtered.len(), 1); + assert_eq!(filtered.get("custom-key"), Some(&"custom-value".to_string())); + } + #[test] fn test_detect_content_type_from_object_name() { // Test Parquet files (our custom handling) diff --git a/rustfs/src/storage/timeout_wrapper.rs b/rustfs/src/storage/timeout_wrapper.rs index 582c4c7e4..f0ca0f882 100644 --- a/rustfs/src/storage/timeout_wrapper.rs +++ b/rustfs/src/storage/timeout_wrapper.rs @@ -53,6 +53,8 @@ //! } //! ``` +#[cfg(not(test))] +use std::sync::OnceLock; use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use tracing::{debug, warn}; @@ -118,6 +120,19 @@ impl GetObjectTimeoutPolicy { } } + /// Load the process-wide GetObject timeout policy once for hot request paths. + #[cfg(not(test))] + pub fn cached_from_env() -> Self { + static POLICY: OnceLock = OnceLock::new(); + POLICY.get_or_init(Self::from_env).clone() + } + + /// Load the GetObject timeout policy from the current test environment. + #[cfg(test)] + pub fn cached_from_env() -> Self { + Self::from_env() + } + /// Check if timeout is enabled (timeout > 0). pub fn is_timeout_enabled(&self) -> bool { self.get_object_timeout > Duration::ZERO @@ -475,6 +490,28 @@ mod tests { assert_eq!(config.get_object_timeout, Duration::from_secs(30)); } + #[test] + fn test_cached_timeout_config_is_stable() { + let first = GetObjectTimeoutPolicy::cached_from_env(); + let second = GetObjectTimeoutPolicy::cached_from_env(); + + assert_eq!(first.get_object_timeout, second.get_object_timeout); + assert_eq!(first.enable_dynamic_timeout, second.enable_dynamic_timeout); + assert_eq!(first.bytes_per_second, second.bytes_per_second); + assert_eq!(first.min_timeout, second.min_timeout); + assert_eq!(first.max_timeout, second.max_timeout); + } + + #[test] + fn test_cached_timeout_config_honors_test_env_overrides() { + temp_env::with_var(rustfs_config::ENV_OBJECT_GET_TIMEOUT, Some("7"), || { + assert_eq!(GetObjectTimeoutPolicy::cached_from_env().get_object_timeout, Duration::from_secs(7)); + }); + temp_env::with_var(rustfs_config::ENV_OBJECT_GET_TIMEOUT, Some("11"), || { + assert_eq!(GetObjectTimeoutPolicy::cached_from_env().get_object_timeout, Duration::from_secs(11)); + }); + } + #[test] fn test_timeout_config_is_enabled() { let config = GetObjectTimeoutPolicy::default(); diff --git a/scripts/run_get_codec_streaming_smoke.sh b/scripts/run_get_codec_streaming_smoke.sh index cba5a89bd..bd3d13c6f 100755 --- a/scripts/run_get_codec_streaming_smoke.sh +++ b/scripts/run_get_codec_streaming_smoke.sh @@ -23,9 +23,18 @@ DURATION="30s" ROUNDS=3 RETRY_PER_ROUND=1 ROUND_COOLDOWN_SECS=20 +WARP_OBJECTS="" +WARP_OBJECT_LIFECYCLE="per-round" +WARP_PREPARE_DURATION="1s" +GET_OBJECT_METADATA_CACHE_MAX_ENTRIES="" +GET_SMALL_OBJECT_DIRECT_MEMORY="" +GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD="" +LOCAL_READ_COPY_METHOD="" MODE="both" +PROFILE_ORDER="normal" CODEC_ENGINES="legacy" CODEC_MAX_INFLIGHT=1 +CODEC_READER_SETUP="all_shards" CODEC_MULTIPART="off" CODEC_MULTIPART_MAX_PARTS=256 METADATA_EARLY_STOP="off" @@ -40,6 +49,11 @@ DIAGNOSTIC_OBS_ENDPOINT="${RUSTFS_OBS_ENDPOINT:-}" DIAGNOSTIC_OBS_METRIC_ENDPOINT="${RUSTFS_OBS_METRIC_ENDPOINT:-}" DIAGNOSTIC_OBS_METER_INTERVAL="${RUSTFS_OBS_METER_INTERVAL:-1}" DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX="${RUSTFS_OBS_SERVICE_NAME:-RustFS-get-codec}" +DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS="${RUSTFS_DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS:-5}" +DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS="${RUSTFS_DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS:-1}" +DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS="${RUSTFS_DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS:-2}" +DIAGNOSTIC_METRICS_MAX_TIME_SECS="${RUSTFS_DIAGNOSTIC_METRICS_MAX_TIME_SECS:-15}" +DIAGNOSTIC_METRICS_FILTER_REGEX="${RUSTFS_DIAGNOSTIC_METRICS_FILTER_REGEX:-rustfs_io_get_object_}" SERVICE_METRIC_PREFIX="rustfs_io_get_object_" COMPRESSED_FALLBACK_PROBE=false COMPRESSED_PROBE_EXTENSION=".compressed-probe.txt" @@ -53,8 +67,12 @@ RUST_LOG="warn" HEALTH_TIMEOUT_SECS=60 COMPAT_OBJECT_KEY="__rustfs_get_v2_pr24_compat/object.bin" COMPAT_OBJECT_SIZE=65536 +SKIP_COMPAT_PROBE=false +RESOURCE_SAMPLE_INTERVAL_SECS="${RUSTFS_GET_BENCH_RESOURCE_SAMPLE_INTERVAL_SECS:-5}" DRY_RUN=false SKIP_BUILD=false +DETACH=false +DETACH_LOG="" PROFILE_FAILURES=0 declare -a ORIGINAL_ARGS=() @@ -73,11 +91,17 @@ Purpose: Core options: --mode Which profile(s) to run (default: both) + --profile-order + Profile execution order after --mode expansion + (default: normal) --codec-engine Codec engine(s) for codec profiles (default: legacy) --metadata-early-stop Enable metadata early-stop observe/opt-in env (default: off) --shard-locality-preference Enable shard locality preference env (default: off) --codec-max-inflight RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT (default: 1) + --codec-reader-setup + RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP + for codec verify reader setup (default: all_shards) --codec-multipart Enable multipart codec streaming opt-in for codec profiles (default: off) --codec-multipart-max-parts @@ -96,6 +120,20 @@ Core options: --diagnostic-metrics-settle-secs Seconds to wait before the after snapshot so OTLP periodic metrics can export probe counters (default: 2) + --diagnostic-metrics-capture-attempts + Retry attempts for each direct /metrics scrape (default: 5) + --diagnostic-metrics-capture-retry-secs + Sleep seconds between failed direct /metrics scrape attempts + (default: 1) + --diagnostic-metrics-connect-timeout-secs + Curl connect timeout for each direct /metrics scrape + (default: 2) + --diagnostic-metrics-max-time-secs + Curl max time for each direct /metrics scrape + (default: 15) + --diagnostic-metrics-filter-regex + Regex for retained direct /metrics lines + (default: rustfs_io_get_object_) --diagnostic-obs-endpoint RUSTFS_OBS_ENDPOINT passed to RustFS during diagnostic runs --diagnostic-obs-metric-endpoint @@ -111,6 +149,20 @@ Core options: --sizes Object sizes (default: 1MiB,4MiB,10MiB) --concurrency warp concurrency (default: 16) --duration warp duration per round (default: 30s) + --warp-objects Number of objects prepared by warp for each size + (default: warp default) + --warp-object-lifecycle Object lifecycle mode for warp GET: + per-round|prepare-once|existing-only + (default: per-round) + --warp-prepare-duration Duration used by the prepare-once warmup + warp GET run (default: 1s) + --metadata-cache-max-entries + RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES for RustFS + --direct-memory RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY for RustFS + --direct-memory-threshold + RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD for RustFS + --local-read-copy-method + RUSTFS_OBJECT_MMAP_READ_METHOD for RustFS --rounds rounds per size (default: 3) --retry-per-round failed-attempt retries per round (default: 1) --round-cooldown-secs cooldown seconds after each completed round (default: 20) @@ -123,7 +175,14 @@ Binary/options: --codec-min-size RUSTFS_GET_CODEC_STREAMING_MIN_SIZE (default: 1) --compat-object-key Object key used by the compatibility probe --compat-object-size Object size used by the compatibility probe (default: 65536) + --skip-compat-probe skip post-benchmark compatibility/fallback probes + --resource-sample-interval-secs + Process resource sampler interval (default: 5) --skip-build do not run cargo build --release -p rustfs + --detach start this script under nohup and exit after + writing detached pid/log metadata + --detach-log log file used with --detach + (default: /detached.log) --dry-run print benchmark commands without starting RustFS Credentials: @@ -147,6 +206,11 @@ Output: /compat_summary.csv when legacy and codec profiles both run /metrics_summary.csv /service_metrics_summary.csv when --diagnostic-metrics is set + /service_metrics_round_summary.csv per-round stage/page-fault deltas when direct --diagnostic-metrics is set + /service_metrics_stage_distribution.csv + per-round stage histogram bucket deltas when direct --diagnostic-metrics is set + /service_metrics_round_percentiles.csv + per-round request/stage histogram percentile bucket upper bounds /service_metrics_acceptance.csv when --diagnostic-metrics is set /fallback_probe_summary.csv /body_sha256_legacy.txt when legacy profile runs @@ -158,7 +222,12 @@ Output: //manifest.env //metrics_summary.csv //service_metrics_summary.csv + //service_metrics_round_summary.csv + //service_metrics_stage_distribution.csv + //service_metrics_round_percentiles.csv //service-metrics/*.prom before/after snapshots when --diagnostic-metrics is set + //service-metrics/rounds/*.prom + per-round snapshots when direct --diagnostic-metrics is set //compat/compat_summary.csv //compat/fallback_probe_summary.csv //compat/response_headers.json @@ -207,10 +276,12 @@ parse_args() { while [[ $# -gt 0 ]]; do case "$1" in --mode) MODE="$2"; shift 2 ;; + --profile-order) PROFILE_ORDER="$2"; shift 2 ;; --codec-engine) CODEC_ENGINES="$2"; shift 2 ;; --metadata-early-stop) METADATA_EARLY_STOP="$2"; shift 2 ;; --shard-locality-preference) SHARD_LOCALITY_PREFERENCE="$2"; shift 2 ;; --codec-max-inflight) CODEC_MAX_INFLIGHT="$2"; shift 2 ;; + --codec-reader-setup) CODEC_READER_SETUP="$2"; shift 2 ;; --codec-multipart) CODEC_MULTIPART="$2"; shift 2 ;; --codec-multipart-max-parts) CODEC_MULTIPART_MAX_PARTS="$2"; shift 2 ;; --handoff-attribution) OUTPUT_HANDOFF_ATTRIBUTION=true; shift ;; @@ -219,6 +290,11 @@ parse_args() { --diagnostic-prometheus-query-url) DIAGNOSTIC_PROMETHEUS_QUERY_URL="$2"; shift 2 ;; --diagnostic-prometheus-query) DIAGNOSTIC_PROMETHEUS_QUERY="$2"; shift 2 ;; --diagnostic-metrics-settle-secs) DIAGNOSTIC_METRICS_SETTLE_SECS="$2"; shift 2 ;; + --diagnostic-metrics-capture-attempts) DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS="$2"; shift 2 ;; + --diagnostic-metrics-capture-retry-secs) DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS="$2"; shift 2 ;; + --diagnostic-metrics-connect-timeout-secs) DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS="$2"; shift 2 ;; + --diagnostic-metrics-max-time-secs) DIAGNOSTIC_METRICS_MAX_TIME_SECS="$2"; shift 2 ;; + --diagnostic-metrics-filter-regex) DIAGNOSTIC_METRICS_FILTER_REGEX="$2"; shift 2 ;; --diagnostic-obs-endpoint) DIAGNOSTIC_OBS_ENDPOINT="$2"; shift 2 ;; --diagnostic-obs-metric-endpoint) DIAGNOSTIC_OBS_METRIC_ENDPOINT="$2"; shift 2 ;; --diagnostic-obs-meter-interval) DIAGNOSTIC_OBS_METER_INTERVAL="$2"; shift 2 ;; @@ -229,6 +305,13 @@ parse_args() { --sizes) SIZES="$2"; shift 2 ;; --concurrency) CONCURRENCY="$2"; shift 2 ;; --duration) DURATION="$2"; shift 2 ;; + --warp-objects) WARP_OBJECTS="$2"; shift 2 ;; + --warp-object-lifecycle) WARP_OBJECT_LIFECYCLE="$2"; shift 2 ;; + --warp-prepare-duration) WARP_PREPARE_DURATION="$2"; shift 2 ;; + --metadata-cache-max-entries) GET_OBJECT_METADATA_CACHE_MAX_ENTRIES="$2"; shift 2 ;; + --direct-memory) GET_SMALL_OBJECT_DIRECT_MEMORY="$2"; shift 2 ;; + --direct-memory-threshold) GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD="$2"; shift 2 ;; + --local-read-copy-method) LOCAL_READ_COPY_METHOD="$2"; shift 2 ;; --rounds) ROUNDS="$2"; shift 2 ;; --retry-per-round) RETRY_PER_ROUND="$2"; shift 2 ;; --round-cooldown-secs) ROUND_COOLDOWN_SECS="$2"; shift 2 ;; @@ -239,10 +322,14 @@ parse_args() { --codec-min-size) CODEC_MIN_SIZE="$2"; shift 2 ;; --compat-object-key) COMPAT_OBJECT_KEY="$2"; shift 2 ;; --compat-object-size) COMPAT_OBJECT_SIZE="$2"; shift 2 ;; + --skip-compat-probe) SKIP_COMPAT_PROBE=true; shift ;; + --resource-sample-interval-secs) RESOURCE_SAMPLE_INTERVAL_SECS="$2"; shift 2 ;; --access-key) ACCESS_KEY="$2"; shift 2 ;; --secret-key) SECRET_KEY="$2"; shift 2 ;; --region) REGION="$2"; shift 2 ;; --skip-build) SKIP_BUILD=true; shift ;; + --detach) DETACH=true; shift ;; + --detach-log) DETACH_LOG="$2"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; -h|--help) usage; exit 0 ;; *) @@ -259,6 +346,11 @@ validate_args() { *) die "--mode must be legacy, codec, or both" ;; esac + case "$PROFILE_ORDER" in + normal|reverse) ;; + *) die "--profile-order must be normal or reverse" ;; + esac + case "$METADATA_EARLY_STOP" in on|off) ;; *) die "--metadata-early-stop must be on or off" ;; @@ -292,14 +384,53 @@ validate_args() { [[ -n "$SIZES" ]] || die "--sizes must not be empty" validate_positive_int "$CONCURRENCY" "--concurrency" validate_positive_int "$CODEC_MAX_INFLIGHT" "--codec-max-inflight" + case "$CODEC_READER_SETUP" in + all_shards|data_blocks_first) ;; + *) die "--codec-reader-setup must be all_shards or data_blocks_first" ;; + esac validate_positive_int "$CODEC_MULTIPART_MAX_PARTS" "--codec-multipart-max-parts" validate_positive_int "$ROUNDS" "--rounds" validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round" validate_non_negative_int "$ROUND_COOLDOWN_SECS" "--round-cooldown-secs" + if [[ -n "$WARP_OBJECTS" ]]; then + validate_positive_int "$WARP_OBJECTS" "--warp-objects" + fi + case "$WARP_OBJECT_LIFECYCLE" in + per-round|prepare-once|existing-only) ;; + *) die "--warp-object-lifecycle must be per-round, prepare-once, or existing-only" ;; + esac + if [[ "$WARP_OBJECT_LIFECYCLE" != "per-round" ]]; then + local size_count + size_count="$(printf '%s\n' "$SIZES" | awk -F',' '{ count=0; for (i=1; i<=NF; i++) { gsub(/^[ \t]+|[ \t]+$/, "", $i); if ($i != "") count++ } print count }')" + [[ "$size_count" -eq 1 ]] || die "--warp-object-lifecycle=${WARP_OBJECT_LIFECYCLE} currently requires a single --sizes value to avoid mixing object sizes in one bucket" + fi + if [[ -n "$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES" ]]; then + validate_positive_int "$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES" "--metadata-cache-max-entries" + fi + if [[ -n "$GET_SMALL_OBJECT_DIRECT_MEMORY" ]]; then + case "$GET_SMALL_OBJECT_DIRECT_MEMORY" in + on|off) ;; + *) die "--direct-memory must be on or off" ;; + esac + fi + if [[ -n "$GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD" ]]; then + validate_positive_int "$GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD" "--direct-memory-threshold" + fi + if [[ -n "$LOCAL_READ_COPY_METHOD" ]]; then + case "$LOCAL_READ_COPY_METHOD" in + mmap_copy|direct_read_copy) ;; + *) die "--local-read-copy-method must be mmap_copy or direct_read_copy" ;; + esac + fi validate_positive_int "$CODEC_MIN_SIZE" "--codec-min-size" validate_positive_int "$COMPAT_OBJECT_SIZE" "--compat-object-size" + validate_positive_int "$RESOURCE_SAMPLE_INTERVAL_SECS" "--resource-sample-interval-secs" validate_positive_int "$HEALTH_TIMEOUT_SECS" "--health-timeout-secs" validate_non_negative_int "$DIAGNOSTIC_METRICS_SETTLE_SECS" "--diagnostic-metrics-settle-secs" + validate_positive_int "$DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS" "--diagnostic-metrics-capture-attempts" + validate_non_negative_int "$DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS" "--diagnostic-metrics-capture-retry-secs" + validate_positive_int "$DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS" "--diagnostic-metrics-connect-timeout-secs" + validate_positive_int "$DIAGNOSTIC_METRICS_MAX_TIME_SECS" "--diagnostic-metrics-max-time-secs" validate_positive_int "$DIAGNOSTIC_OBS_METER_INTERVAL" "--diagnostic-obs-meter-interval" [[ -n "$COMPAT_OBJECT_KEY" ]] || die "--compat-object-key must not be empty" [[ -n "$DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX" ]] || die "--diagnostic-obs-service-name-prefix must not be empty" @@ -313,6 +444,9 @@ validate_args() { require_cmd cargo require_cmd "$PYTHON_BIN" fi + if [[ "$DETACH" == "true" && "$DRY_RUN" != "true" ]]; then + require_cmd nohup + fi } setup_output() { @@ -330,6 +464,14 @@ bool_from_on_off() { esac } +codec_reader_setup_data_blocks_first() { + case "$CODEC_READER_SETUP" in + all_shards) echo "false" ;; + data_blocks_first) echo "true" ;; + *) die "expected codec reader setup strategy, got: $CODEC_READER_SETUP" ;; + esac +} + dry_run_multipart_expected_reason() { if [[ "$CODEC_MULTIPART" != "on" ]]; then echo "multipart" @@ -344,6 +486,71 @@ command_line_string() { printf '%q ' "${BASH_SOURCE[0]}" "${ORIGINAL_ARGS[@]}" } +detached_child_args() { + local skip_next=false + local arg + + for arg in "${ORIGINAL_ARGS[@]}"; do + if [[ "$skip_next" == "true" ]]; then + skip_next=false + continue + fi + + case "$arg" in + --detach) + ;; + --detach-log) + skip_next=true + ;; + *) + printf '%s\0' "$arg" + ;; + esac + done +} + +run_detached_if_requested() { + if [[ "$DETACH" != "true" ]]; then + return 0 + fi + + if [[ -z "$DETACH_LOG" ]]; then + DETACH_LOG="${OUT_DIR}/detached.log" + fi + + local script_path="${PROJECT_ROOT}/scripts/run_get_codec_streaming_smoke.sh" + local pid_file="${OUT_DIR}/detached.pid" + local command_file="${OUT_DIR}/detached_command.txt" + local -a child_args=() + + while IFS= read -r -d '' arg; do + child_args+=("$arg") + done < <(detached_child_args) + + { + printf '%q ' "$script_path" "${child_args[@]}" + printf '\n' + } >"$command_file" + + if [[ "$DRY_RUN" == "true" ]]; then + log "[DRY-RUN] detach requested" + log "[DRY-RUN] detached log: ${DETACH_LOG}" + log "[DRY-RUN] detached command: ${command_file}" + return 0 + fi + + nohup "$script_path" "${child_args[@]}" >"$DETACH_LOG" 2>&1 < /dev/null & + local detached_pid="$!" + printf '%s\n' "$detached_pid" >"$pid_file" + + log "Detached GET codec streaming smoke started." + log "PID: ${detached_pid}" + log "Output dir: ${OUT_DIR}" + log "Log: ${DETACH_LOG}" + log "Command: ${command_file}" + exit 0 +} + sanitize_metric_label_value() { printf '%s' "$1" | tr -c '[:alnum:]_-' '-' } @@ -492,10 +699,12 @@ git_head=${git_head} git_dirty_count=${git_dirty_count} profiles=${profiles_csv} mode=${MODE} +profile_order=${PROFILE_ORDER} codec_engines=${CODEC_ENGINES} metadata_early_stop=${METADATA_EARLY_STOP} shard_locality_preference=${SHARD_LOCALITY_PREFERENCE} codec_max_inflight=${CODEC_MAX_INFLIGHT} +codec_reader_setup=${CODEC_READER_SETUP} codec_rollout_codec_profile=benchmark codec_body_compat_confirmed_codec_profile=true codec_header_compat_confirmed_codec_profile=true @@ -507,6 +716,11 @@ diagnostic_metrics_url=${DIAGNOSTIC_METRICS_URL} diagnostic_prometheus_query_url=${DIAGNOSTIC_PROMETHEUS_QUERY_URL} diagnostic_prometheus_query=${DIAGNOSTIC_PROMETHEUS_QUERY} diagnostic_metrics_settle_secs=${DIAGNOSTIC_METRICS_SETTLE_SECS} +diagnostic_metrics_capture_attempts=${DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS} +diagnostic_metrics_capture_retry_secs=${DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS} +diagnostic_metrics_connect_timeout_secs=${DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS} +diagnostic_metrics_max_time_secs=${DIAGNOSTIC_METRICS_MAX_TIME_SECS} +diagnostic_metrics_filter_regex=${DIAGNOSTIC_METRICS_FILTER_REGEX} diagnostic_obs_endpoint=${DIAGNOSTIC_OBS_ENDPOINT} diagnostic_obs_metric_endpoint=${DIAGNOSTIC_OBS_METRIC_ENDPOINT} diagnostic_obs_meter_interval=${DIAGNOSTIC_OBS_METER_INTERVAL} @@ -524,6 +738,11 @@ duration=${DURATION} rounds=${ROUNDS} retry_per_round=${RETRY_PER_ROUND} round_cooldown_secs=${ROUND_COOLDOWN_SECS} +warp_objects=${WARP_OBJECTS} +RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES=${GET_OBJECT_METADATA_CACHE_MAX_ENTRIES} +RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY=${GET_SMALL_OBJECT_DIRECT_MEMORY} +RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD=${GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD} +RUSTFS_OBJECT_MMAP_READ_METHOD=${LOCAL_READ_COPY_METHOD} skip_build=${SKIP_BUILD} dry_run=${DRY_RUN} rustfs_bin=${RUSTFS_BIN} @@ -614,6 +833,18 @@ endpoint_url() { echo "http://${ADDRESS}" } +single_benchmark_size() { + printf '%s\n' "$SIZES" | awk -F',' '{ + for (i=1; i<=NF; i++) { + gsub(/^[ \t]+|[ \t]+$/, "", $i) + if ($i != "") { + print $i + exit + } + } + }' +} + write_manifest() { local profile="$1" local profile_dir="$2" @@ -645,6 +876,9 @@ duration=${DURATION} rounds=${ROUNDS} retry_per_round=${RETRY_PER_ROUND} round_cooldown_secs=${ROUND_COOLDOWN_SECS} +warp_objects=${WARP_OBJECTS} +warp_object_lifecycle=${WARP_OBJECT_LIFECYCLE} +warp_prepare_duration=${WARP_PREPARE_DURATION} rustfs_bin=${RUSTFS_BIN} warp_bin=${WARP_BIN} python_bin=${PYTHON_BIN} @@ -660,10 +894,15 @@ RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED=${header_compat_confirmed} RUSTFS_GET_METADATA_EARLY_STOP_ENABLE=$(bool_from_on_off "$METADATA_EARLY_STOP") RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE=$(bool_from_on_off "$SHARD_LOCALITY_PREFERENCE") RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT=${CODEC_MAX_INFLIGHT} +RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP=$(codec_reader_setup_data_blocks_first) RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE=$(bool_from_on_off "$CODEC_MULTIPART") RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS=${CODEC_MULTIPART_MAX_PARTS} RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE=${OUTPUT_HANDOFF_ATTRIBUTION} RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=${CODEC_MIN_SIZE} +RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES=${GET_OBJECT_METADATA_CACHE_MAX_ENTRIES} +RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY=${GET_SMALL_OBJECT_DIRECT_MEMORY} +RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD=${GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD} +RUSTFS_OBJECT_MMAP_READ_METHOD=${LOCAL_READ_COPY_METHOD} RUSTFS_OBS_METRICS_EXPORT_ENABLED=${DIAGNOSTIC_METRICS} RUSTFS_OBS_ENDPOINT=${DIAGNOSTIC_OBS_ENDPOINT} RUSTFS_OBS_METRIC_ENDPOINT=${DIAGNOSTIC_OBS_METRIC_ENDPOINT} @@ -677,6 +916,11 @@ diagnostic_metrics_url=${DIAGNOSTIC_METRICS_URL} diagnostic_prometheus_query_url=${DIAGNOSTIC_PROMETHEUS_QUERY_URL} diagnostic_prometheus_query=${DIAGNOSTIC_PROMETHEUS_QUERY} diagnostic_metrics_settle_secs=${DIAGNOSTIC_METRICS_SETTLE_SECS} +diagnostic_metrics_capture_attempts=${DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS} +diagnostic_metrics_capture_retry_secs=${DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS} +diagnostic_metrics_connect_timeout_secs=${DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS} +diagnostic_metrics_max_time_secs=${DIAGNOSTIC_METRICS_MAX_TIME_SECS} +diagnostic_metrics_filter_regex=${DIAGNOSTIC_METRICS_FILTER_REGEX} service_metric_prefix=${SERVICE_METRIC_PREFIX} metrics_path=${metrics_path} RUSTFS_SCANNER_ENABLED=false @@ -684,6 +928,8 @@ RUSTFS_SCANNER_START_DELAY_SECS=3600 RUSTFS_SCANNER_CYCLE=3600 RUSTFS_CONSOLE_ENABLE=false RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true +resource_sample_interval_secs=${RESOURCE_SAMPLE_INTERVAL_SECS} +skip_compat_probe=${SKIP_COMPAT_PROBE} EOF } @@ -774,13 +1020,31 @@ start_server() { export RUSTFS_GET_CODEC_STREAMING_ROLLOUT="$rollout_target" export RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED="$body_compat_confirmed" export RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED="$header_compat_confirmed" - export RUSTFS_GET_METADATA_EARLY_STOP_ENABLE="$(bool_from_on_off "$METADATA_EARLY_STOP")" - export RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE="$(bool_from_on_off "$SHARD_LOCALITY_PREFERENCE")" + RUSTFS_GET_METADATA_EARLY_STOP_ENABLE="$(bool_from_on_off "$METADATA_EARLY_STOP")" + export RUSTFS_GET_METADATA_EARLY_STOP_ENABLE + RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE="$(bool_from_on_off "$SHARD_LOCALITY_PREFERENCE")" + export RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE export RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT="$CODEC_MAX_INFLIGHT" - export RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE="$(bool_from_on_off "$CODEC_MULTIPART")" + RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP="$(codec_reader_setup_data_blocks_first)" + export RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP + RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE="$(bool_from_on_off "$CODEC_MULTIPART")" + export RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE export RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS="$CODEC_MULTIPART_MAX_PARTS" export RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE="$OUTPUT_HANDOFF_ATTRIBUTION" export RUSTFS_GET_CODEC_STREAMING_MIN_SIZE="$CODEC_MIN_SIZE" + if [[ -n "$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES" ]]; then + export RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES="$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES" + fi + if [[ -n "$GET_SMALL_OBJECT_DIRECT_MEMORY" ]]; then + RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY="$(bool_from_on_off "$GET_SMALL_OBJECT_DIRECT_MEMORY")" + export RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY + fi + if [[ -n "$GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD" ]]; then + export RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD="$GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD" + fi + if [[ -n "$LOCAL_READ_COPY_METHOD" ]]; then + export RUSTFS_OBJECT_MMAP_READ_METHOD="$LOCAL_READ_COPY_METHOD" + fi export RUSTFS_OBS_METRICS_EXPORT_ENABLED="$DIAGNOSTIC_METRICS" if [[ "$DIAGNOSTIC_METRICS" == "true" ]]; then export RUSTFS_OBS_METER_INTERVAL="$DIAGNOSTIC_OBS_METER_INTERVAL" @@ -810,6 +1074,58 @@ start_server() { wait_for_health "$profile" "$rustfs_log" } +prepare_warp_existing_objects() { + local profile="$1" + local profile_dir="${OUT_DIR}/${profile}" + local prepare_dir="${profile_dir}/warp_prepare" + local size + size="$(single_benchmark_size)" + + if [[ "$WARP_OBJECT_LIFECYCLE" != "prepare-once" ]]; then + return 0 + fi + + mkdir -p "$prepare_dir" + + local cmd=( + "$WARP_BIN" get + --host "$ADDRESS" + --access-key "$ACCESS_KEY" + --secret-key "$SECRET_KEY" + --bucket "$BUCKET" + --obj.size "$size" + --concurrent "$CONCURRENCY" + --duration "$WARP_PREPARE_DURATION" + --region "$REGION" + --noclear + ) + if [[ -n "$WARP_OBJECTS" ]]; then + cmd+=(--objects "$WARP_OBJECTS") + fi + + { + printf 'profile=%s\n' "$profile" + printf 'mode=prepare-once\n' + printf 'size=%s\n' "$size" + printf 'duration=%s\n' "$WARP_PREPARE_DURATION" + printf 'bucket=%s\n' "$BUCKET" + printf 'command=' + printf '%q ' "${cmd[@]}" + printf '\n' + } >"${prepare_dir}/manifest.env" + + if [[ "$DRY_RUN" == "true" ]]; then + log "[DRY-RUN] prepare existing warp objects profile=${profile} size=${size}" + printf '[DRY-RUN] ' >"${prepare_dir}/prepare.log" + printf '%q ' "${cmd[@]}" >>"${prepare_dir}/prepare.log" + printf '\n' >>"${prepare_dir}/prepare.log" + return 0 + fi + + log "Preparing existing warp objects profile=${profile} size=${size} lifecycle=${WARP_OBJECT_LIFECYCLE}..." + "${cmd[@]}" >"${prepare_dir}/prepare.log" 2>&1 +} + run_bench() { local profile="$1" local baseline_csv="${2:-}" @@ -837,6 +1153,27 @@ run_bench() { if [[ -n "$baseline_csv" ]]; then cmd+=(--baseline-csv "$baseline_csv") fi + if [[ "$WARP_OBJECT_LIFECYCLE" == "per-round" && -n "$WARP_OBJECTS" ]]; then + cmd+=(--extra-args "--objects ${WARP_OBJECTS}") + fi + if [[ "$WARP_OBJECT_LIFECYCLE" != "per-round" ]]; then + local lifecycle_args="--list-existing --noclear" + if [[ -n "$WARP_OBJECTS" ]]; then + lifecycle_args="${lifecycle_args} --objects ${WARP_OBJECTS}" + fi + cmd+=(--extra-args "$lifecycle_args") + fi + if [[ "$DIAGNOSTIC_METRICS" == "true" && -z "$DIAGNOSTIC_PROMETHEUS_QUERY_URL" ]]; then + cmd+=( + --service-metrics-url "$DIAGNOSTIC_METRICS_URL" + --service-metrics-dir "${profile_dir}/service-metrics/rounds" + --service-metrics-attempts "$DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS" + --service-metrics-retry-secs "$DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS" + --service-metrics-connect-timeout-secs "$DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS" + --service-metrics-max-time-secs "$DIAGNOSTIC_METRICS_MAX_TIME_SECS" + --service-metrics-filter-regex "$DIAGNOSTIC_METRICS_FILTER_REGEX" + ) + fi if [[ "$DRY_RUN" == "true" ]]; then cmd+=(--dry-run) fi @@ -944,6 +1281,23 @@ write_status("ok") PY } +filter_diagnostic_metrics_snapshot() { + local input_file="$1" + local output_file="$2" + + if [[ -z "$DIAGNOSTIC_METRICS_FILTER_REGEX" ]]; then + mv "$input_file" "$output_file" + return 0 + fi + + awk -v pattern="$DIAGNOSTIC_METRICS_FILTER_REGEX" '$0 ~ pattern { print }' "$input_file" >"$output_file" + if [[ ! -s "$output_file" ]]; then + mv "$input_file" "$output_file" + else + rm -f "$input_file" + fi +} + capture_service_metrics_snapshot() { local profile="$1" local phase="$2" @@ -964,6 +1318,7 @@ capture_service_metrics_snapshot() { phase=${phase} status=not_run_dry_run url=${DIAGNOSTIC_METRICS_URL} +filter_regex=${DIAGNOSTIC_METRICS_FILTER_REGEX} EOF return 0 fi @@ -973,30 +1328,51 @@ EOF return 0 fi - if curl -fsS --noproxy '*' --connect-timeout 2 --max-time 5 "$DIAGNOSTIC_METRICS_URL" >"$snapshot_file"; then - if [[ ! -s "$snapshot_file" ]]; then - cat >"$status_file" <"$status_file" <"$tmp_file"; then + if [[ -s "$tmp_file" ]]; then + raw_bytes="$(wc -c <"$tmp_file" | tr -d '[:space:]')" + filter_diagnostic_metrics_snapshot "$tmp_file" "$filtered_file" + mv "$filtered_file" "$snapshot_file" + snapshot_bytes="$(wc -c <"$snapshot_file" | tr -d '[:space:]')" + cat >"$status_file" <"$snapshot_file" - cat >"$status_file" < 0 )); then + sleep "$DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS" + fi + done + + : >"$snapshot_file" + cat >"$status_file" <"$out_summary" <"$out_distribution" <"$out_percentiles" <"$out_summary" <"$out_distribution" <"$out_percentiles" <"$out_summary" <"$out_distribution" <"$out_percentiles" <= threshold: + return le + return finite_or_inf[-1][0] + + +def metric_count_sum(before, after, metric_name, path="", stage="", status=""): + count = 0.0 + total = 0.0 + for metric, labels in sorted(set(before) | set(after)): + if path and label_value(labels, "path") != path: + continue + if stage and label_value(labels, "stage") != stage: + continue + if status and label_value(labels, "status") != status: + continue + if metric == f"{metric_name}_count": + count += after.get((metric, labels), 0.0) - before.get((metric, labels), 0.0) + elif metric == f"{metric_name}_sum": + total += after.get((metric, labels), 0.0) - before.get((metric, labels), 0.0) + return count, total + + +PERCENTILE_TARGETS = [ + ("request_duration_ok", "rustfs_io_get_object_request_duration_seconds", "", "", "ok"), + ("total_duration", "rustfs_io_get_object_total_duration_seconds", "", "", ""), + ("request_context", "rustfs_io_get_object_stage_duration_seconds", "s3_handler", "request_context", ""), +] +for stage_name in TARGET_STAGES: + PERCENTILE_TARGETS.append((stage_name, "rustfs_io_get_object_stage_duration_seconds", "", stage_name, "")) + + +with summary_path.open("w", encoding="utf-8", newline="") as summary_handle, \ + distribution_path.open("w", encoding="utf-8", newline="") as distribution_handle, \ + percentiles_path.open("w", encoding="utf-8", newline="") as percentiles_handle: + summary_writer = csv.writer(summary_handle) + distribution_writer = csv.writer(distribution_handle) + percentiles_writer = csv.writer(percentiles_handle) + summary_writer.writerow([ + "profile", "size", "tool", "round", "attempt", "round_status", "stage", + "count_delta", "sum_delta", "avg_ms", "minor_fault_delta", "major_fault_delta", + "minor_faults_per_call", "major_faults_per_call", "before_status", "after_status", + ]) + distribution_writer.writerow([ + "profile", "size", "tool", "round", "attempt", "round_status", "path", "stage", + "le", "bucket_delta", "before_status", "after_status", + ]) + percentiles_writer.writerow([ + "profile", "size", "tool", "round", "attempt", "round_status", "target", + "path", "stage", "count_delta", "sum_delta", "avg_ms", + "p50_le_seconds", "p90_le_seconds", "p99_le_seconds", "bucket_resolution_note", + "before_status", "after_status", + ]) + + if not round_path.exists() or not metrics_dir.exists(): + summary_writer.writerow([ + profile, "N/A", "N/A", "N/A", "N/A", "snapshot_missing", "N/A", + "N/A", "N/A", "N/A", "N/A", "N/A", "N/A", "N/A", "missing", "missing", + ]) + distribution_writer.writerow([ + profile, "N/A", "N/A", "N/A", "N/A", "snapshot_missing", "N/A", + "N/A", "N/A", "N/A", "missing", "missing", + ]) + percentiles_writer.writerow([ + profile, "N/A", "N/A", "N/A", "N/A", "snapshot_missing", "N/A", + "N/A", "N/A", "N/A", "N/A", "N/A", "N/A", "N/A", "N/A", + "snapshot_missing", "missing", "missing", + ]) + raise SystemExit(0) + + for round_row in csv.DictReader(round_path.open(encoding="utf-8", newline="")): + size = round_row["size"] + tool = round_row["tool"] + round_id = round_row["round"] + attempt = round_row["attempt"] + round_status = round_row["status"] + token = snapshot_token(tool, size, round_id, attempt) + before_path = metrics_dir / f"{token}_before.prom" + after_path = metrics_dir / f"{token}_after.prom" + before_status = status_for(metrics_dir / f"{token}_before.status") + after_status = status_for(metrics_dir / f"{token}_after.status") + before = read_metrics(before_path) + after = read_metrics(after_path) + + minor_faults = fault_delta(before, after, "minor") + major_faults = fault_delta(before, after, "major") + for stage in TARGET_STAGES: + count_delta = metric_delta(before, after, "rustfs_io_get_object_stage_duration_seconds", stage, "count") + sum_delta = metric_delta(before, after, "rustfs_io_get_object_stage_duration_seconds", stage, "sum") + avg_ms = (sum_delta / count_delta * 1000.0) if count_delta else 0.0 + stage_minor_faults = minor_faults if stage == "reader_mmap_copy_buffer" else 0.0 + stage_major_faults = major_faults if stage == "reader_mmap_copy_buffer" else 0.0 + summary_writer.writerow([ + profile, + size, + tool, + round_id, + attempt, + round_status, + stage, + f"{count_delta:.12g}", + f"{sum_delta:.12g}", + f"{avg_ms:.6f}", + f"{stage_minor_faults:.12g}", + f"{stage_major_faults:.12g}", + f"{(stage_minor_faults / count_delta) if count_delta else 0.0:.6f}", + f"{(stage_major_faults / count_delta) if count_delta else 0.0:.6f}", + before_status, + after_status, + ]) + + for target, metric_name, path_label, stage_label, status_label in PERCENTILE_TARGETS: + buckets = histogram_bucket_deltas(before, after, metric_name, path_label, stage_label, status_label) + count_delta, sum_delta = metric_count_sum(before, after, metric_name, path_label, stage_label, status_label) + avg_ms = (sum_delta / count_delta * 1000.0) if count_delta else 0.0 + p50 = percentile_bucket_upper_bound(buckets, 0.50) + p90 = percentile_bucket_upper_bound(buckets, 0.90) + p99 = percentile_bucket_upper_bound(buckets, 0.99) + if not buckets: + note = "no_buckets" + elif p50 == p90 == p99 and p99 not in {"N/A", "+Inf"} and bucket_sort_key(p99) >= 1.0 and avg_ms < 1000.0: + note = "coarse_seconds_buckets" + else: + note = "bucket_upper_bound" + percentiles_writer.writerow([ + profile, + size, + tool, + round_id, + attempt, + round_status, + target, + path_label or "N/A", + stage_label or "N/A", + f"{count_delta:.12g}", + f"{sum_delta:.12g}", + f"{avg_ms:.6f}", + p50, + p90, + p99, + note, + before_status, + after_status, + ]) + + for metric, labels in sorted(set(before) | set(after)): + if metric != "rustfs_io_get_object_stage_duration_seconds_bucket": + continue + stage = label_value(labels, "stage") + if stage not in DISTRIBUTION_STAGES: + continue + bucket_delta = after.get((metric, labels), 0.0) - before.get((metric, labels), 0.0) + if bucket_delta == 0: + continue + distribution_writer.writerow([ + profile, + size, + tool, + round_id, + attempt, + round_status, + label_value(labels, "path"), + stage, + label_value(labels, "le"), + f"{bucket_delta:.12g}", + before_status, + after_status, + ]) +PY +} + write_root_service_metrics_summary() { local out_csv="${OUT_DIR}/service_metrics_summary.csv" local profile summary wrote_header=false @@ -1138,6 +1908,32 @@ EOF fi } +write_root_profile_csv() { + local basename="$1" + shift + local out_csv="${OUT_DIR}/${basename}" + local profile summary wrote_header=false + : >"$out_csv" + + for profile in "$@"; do + summary="${OUT_DIR}/${profile}/${basename}" + [[ -f "$summary" ]] || continue + if [[ "$wrote_header" == "false" ]]; then + cat "$summary" >>"$out_csv" + wrote_header=true + else + tail -n +2 "$summary" >>"$out_csv" + fi + done + + if [[ "$wrote_header" == "false" ]]; then + cat >"$out_csv" <<'EOF' +profile,status,note +N/A,missing,no_profile_csv +EOF + fi +} + write_service_metrics_acceptance() { local out_csv="${OUT_DIR}/service_metrics_acceptance.csv" local service_csv="${OUT_DIR}/service_metrics_summary.csv" @@ -1475,22 +2271,75 @@ start_server_sampler() { if [[ "$DRY_RUN" == "true" ]]; then cat >"$SERVER_SAMPLER_LOG" <<'EOF' -timestamp_utc,rss_kib,cpu_pct -DRY_RUN,N/A,N/A +timestamp_utc,rss_kib,cpu_pct,threads,fd_count,vm_hwm_kib,vm_data_kib,voluntary_ctxt_switches,nonvoluntary_ctxt_switches,sched_runtime_ns,sched_wait_ns,sched_timeslices +DRY_RUN,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A EOF return fi - echo "timestamp_utc,rss_kib,cpu_pct" >"$SERVER_SAMPLER_LOG" + echo "timestamp_utc,rss_kib,cpu_pct,threads,fd_count,vm_hwm_kib,vm_data_kib,voluntary_ctxt_switches,nonvoluntary_ctxt_switches,sched_runtime_ns,sched_wait_ns,sched_timeslices" >"$SERVER_SAMPLER_LOG" ( while kill -0 "$pid" >/dev/null 2>&1; do - local timestamp sample + local timestamp sample rss_cpu threads fd_count vm_hwm vm_data voluntary_ctxt nonvoluntary_ctxt sched_runtime sched_wait sched_slices timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - sample="$(ps -o rss= -o %cpu= -p "$pid" 2>/dev/null | awk 'NF >= 2 { gsub(/^[ \t]+|[ \t]+$/, "", $1); gsub(/^[ \t]+|[ \t]+$/, "", $2); print $1 "," $2; exit }')" - if [[ -n "$sample" ]]; then - echo "${timestamp},${sample}" >>"$SERVER_SAMPLER_LOG" + rss_cpu="$(ps -o rss= -o %cpu= -p "$pid" 2>/dev/null | awk 'NF >= 2 { gsub(/^[ \t]+|[ \t]+$/, "", $1); gsub(/^[ \t]+|[ \t]+$/, "", $2); print $1 "," $2; exit }')" + if [[ -z "$rss_cpu" ]]; then + sleep "$RESOURCE_SAMPLE_INTERVAL_SECS" + continue fi - sleep 5 + + threads="N/A" + fd_count="N/A" + vm_hwm="N/A" + vm_data="N/A" + voluntary_ctxt="N/A" + nonvoluntary_ctxt="N/A" + sched_runtime="N/A" + sched_wait="N/A" + sched_slices="N/A" + + if [[ -r "/proc/${pid}/status" ]]; then + sample="$(awk ' + /^Threads:/ { threads=$2 } + /^VmHWM:/ { vm_hwm=$2 } + /^VmData:/ { vm_data=$2 } + /^voluntary_ctxt_switches:/ { voluntary=$2 } + /^nonvoluntary_ctxt_switches:/ { nonvoluntary=$2 } + END { + printf "%s,%s,%s,%s,%s", + threads ? threads : "N/A", + vm_hwm ? vm_hwm : "N/A", + vm_data ? vm_data : "N/A", + voluntary ? voluntary : "N/A", + nonvoluntary ? nonvoluntary : "N/A" + } + ' "/proc/${pid}/status")" + IFS=',' read -r threads vm_hwm vm_data voluntary_ctxt nonvoluntary_ctxt <<<"$sample" + fi + if [[ -d "/proc/${pid}/fd" ]]; then + fd_count="$(find "/proc/${pid}/fd" -maxdepth 1 -type l 2>/dev/null | awk 'END { print NR + 0 }')" + fi + if [[ -d "/proc/${pid}/task" ]]; then + sample="$(awk ' + { + runtime += $1 + wait += $2 + slices += $3 + seen = 1 + } + END { + if (seen) { + printf "%.0f %.0f %.0f", runtime, wait, slices + } + } + ' /proc/"${pid}"/task/*/schedstat 2>/dev/null || true)" + if [[ -n "$sample" ]]; then + read -r sched_runtime sched_wait sched_slices <<<"$sample" + fi + fi + + echo "${timestamp},${rss_cpu},${threads},${fd_count},${vm_hwm},${vm_data},${voluntary_ctxt},${nonvoluntary_ctxt},${sched_runtime},${sched_wait},${sched_slices}" >>"$SERVER_SAMPLER_LOG" + sleep "$RESOURCE_SAMPLE_INTERVAL_SECS" done ) & SERVER_SAMPLER_PID="$!" @@ -1527,8 +2376,26 @@ EOF $2 != "N/A" && $2 != "" { rss = $2 + 0 cpu = $3 + 0 + threads = ($4 != "N/A" && $4 != "") ? $4 + 0 : 0 + fd_count = ($5 != "N/A" && $5 != "") ? $5 + 0 : 0 + vm_hwm = ($6 != "N/A" && $6 != "") ? $6 + 0 : 0 + vm_data = ($7 != "N/A" && $7 != "") ? $7 + 0 : 0 + voluntary = ($8 != "N/A" && $8 != "") ? $8 + 0 : -1 + nonvoluntary = ($9 != "N/A" && $9 != "") ? $9 + 0 : -1 if (count == 0 || rss > max_rss) max_rss = rss if (count == 0 || cpu > max_cpu) max_cpu = cpu + if (threads > max_threads) max_threads = threads + if (fd_count > max_fd_count) max_fd_count = fd_count + if (vm_hwm > max_vm_hwm) max_vm_hwm = vm_hwm + if (vm_data > max_vm_data) max_vm_data = vm_data + if (voluntary >= 0) { + if (!seen_voluntary) { first_voluntary = voluntary; seen_voluntary = 1 } + last_voluntary = voluntary + } + if (nonvoluntary >= 0) { + if (!seen_nonvoluntary) { first_nonvoluntary = nonvoluntary; seen_nonvoluntary = 1 } + last_nonvoluntary = nonvoluntary + } cpu_sum += cpu count++ } @@ -1545,13 +2412,43 @@ EOF printf "max_rss_kib=%.0f\n", max_rss printf "max_cpu_pct=%.2f\n", max_cpu printf "avg_cpu_pct=%.2f\n", cpu_sum / count + printf "max_threads=%.0f\n", max_threads + printf "max_fd_count=%.0f\n", max_fd_count + printf "max_vm_hwm_kib=%.0f\n", max_vm_hwm + printf "max_vm_data_kib=%.0f\n", max_vm_data + if (seen_voluntary) { + printf "voluntary_ctxt_switches_delta=%.0f\n", last_voluntary - first_voluntary + } else { + print "voluntary_ctxt_switches_delta=unknown" + } + if (seen_nonvoluntary) { + printf "nonvoluntary_ctxt_switches_delta=%.0f\n", last_nonvoluntary - first_nonvoluntary + } else { + print "nonvoluntary_ctxt_switches_delta=unknown" + } print "cpu_rss_acceptability=manual_review_required" - print "note=Harness captured runtime samples but does not impose a hard pass/fail threshold." + print "note=Harness captured runtime/resource samples but does not impose a hard pass/fail threshold." } } ' "$sample_csv" >"$notes_file" } +write_skipped_compat_probe() { + local profile="$1" + local profile_dir="${OUT_DIR}/${profile}" + local compat_dir="${profile_dir}/compat" + + mkdir -p "$compat_dir" + cat >"${compat_dir}/compat_summary.csv" <"${compat_dir}/fallback_probe_summary.csv" <"$out_file" + { + find "$OUT_DIR" -maxdepth 1 -type f \ + \( -name 'metrics_summary.csv' \ + -o -name 'service_metrics_summary.csv' \ + -o -name 'service_metrics_round_summary.csv' \ + -o -name 'service_metrics_stage_distribution.csv' \ + -o -name 'service_metrics_round_percentiles.csv' \ + -o -name 'service_metrics_acceptance.csv' \) | sort + for profile_dir in "${OUT_DIR}"/*; do + [[ -d "$profile_dir" ]] || continue + find "$profile_dir" -maxdepth 1 -type f \ + \( -name 'metrics_summary.csv' \ + -o -name 'service_metrics_summary.csv' \ + -o -name 'service_metrics_round_summary.csv' \ + -o -name 'service_metrics_stage_distribution.csv' \ + -o -name 'service_metrics_round_percentiles.csv' \) | sort + if [[ -d "${profile_dir}/service-metrics" ]]; then + find "${profile_dir}/service-metrics" -type f | sort + fi + if [[ -d "${profile_dir}/warp/logs" ]]; then + find "${profile_dir}/warp/logs" -type f | sort + fi + if [[ -d "${profile_dir}/warp_prepare" ]]; then + find "${profile_dir}/warp_prepare" -type f | sort + fi + done + } >"$out_file" } write_default_switch_readiness_report() { @@ -2748,7 +3672,7 @@ write_default_switch_readiness_report() { local stable not_worse improved_over_five two_sizes_gt_five local headers_compatible p95_p99_ok cpu_rss_ok runtime_metrics_ok fallback_proven kill_switch_verified correctness_matrix_ok - local all_pass decision + local decision if [[ -f "$baseline_compare_path" ]]; then read -r stable not_worse improved_over_five < <( @@ -2851,10 +3775,8 @@ PY && "$headers_compatible" == "pass" \ && "$fallback_proven" == "pass" \ && "$kill_switch_verified" == "pass" ]]; then - all_pass="pass" decision="Default enablement prerequisites passed; scoped default enablement may be considered." else - all_pass="fail" decision="Default enablement is not ready; keep opt-in only." fi @@ -2908,6 +3830,7 @@ run_profile() { stop_server start_server "$profile" + prepare_warp_existing_objects "$profile" capture_service_metrics_snapshot "$profile" before if run_bench "$profile" "$baseline_csv"; then : @@ -2915,18 +3838,27 @@ run_profile() { bench_rc=$? fi write_metrics_summary "$profile" - run_compat_probe "$profile" + if [[ "$SKIP_COMPAT_PROBE" == "true" ]]; then + log "Skipping compatibility probe for profile=${profile}" + write_skipped_compat_probe "$profile" + else + run_compat_probe "$profile" + fi if [[ "$DIAGNOSTIC_METRICS" == "true" && "$DIAGNOSTIC_METRICS_SETTLE_SECS" -gt 0 ]]; then sleep "$DIAGNOSTIC_METRICS_SETTLE_SECS" fi capture_service_metrics_snapshot "$profile" after write_profile_service_metrics_summary "$profile" + write_profile_service_metrics_round_breakdown "$profile" stop_server write_profile_cpu_rss_notes "$profile" log "Median summary: ${OUT_DIR}/${profile}/warp/median_summary.csv" log "Metrics summary: ${OUT_DIR}/${profile}/metrics_summary.csv" log "Service metrics summary: ${OUT_DIR}/${profile}/service_metrics_summary.csv" + log "Service metrics round summary: ${OUT_DIR}/${profile}/service_metrics_round_summary.csv" + log "Service metrics stage distribution: ${OUT_DIR}/${profile}/service_metrics_stage_distribution.csv" + log "Service metrics round percentiles: ${OUT_DIR}/${profile}/service_metrics_round_percentiles.csv" log "Compatibility summary: ${OUT_DIR}/${profile}/compat/compat_summary.csv" if [[ -f "${OUT_DIR}/${profile}/warp/baseline_compare.csv" ]]; then log "Baseline compare: ${OUT_DIR}/${profile}/warp/baseline_compare.csv" @@ -2940,6 +3872,7 @@ main() { parse_args "$@" validate_args setup_output + run_detached_if_requested local codec_profiles=() local profiles=() local profiles_csv="" @@ -2963,6 +3896,14 @@ main() { profiles=("legacy" "${codec_profiles[@]}") ;; esac + if [[ "$PROFILE_ORDER" == "reverse" ]]; then + local reversed_profiles=() + local profile_index + for ((profile_index=${#profiles[@]} - 1; profile_index >= 0; profile_index--)); do + reversed_profiles+=("${profiles[$profile_index]}") + done + profiles=("${reversed_profiles[@]}") + fi profiles_csv="$(IFS=,; echo "${profiles[*]}")" write_root_environment @@ -2992,6 +3933,9 @@ main() { write_root_metrics_summary "${profiles[@]}" write_root_service_metrics_summary "${profiles[@]}" + write_root_profile_csv service_metrics_round_summary.csv "${profiles[@]}" + write_root_profile_csv service_metrics_stage_distribution.csv "${profiles[@]}" + write_root_profile_csv service_metrics_round_percentiles.csv "${profiles[@]}" write_service_metrics_acceptance write_root_compat_summary "${profiles[@]}" write_root_fallback_probe_summary "${profiles[@]}" diff --git a/scripts/run_object_batch_bench_enhanced.sh b/scripts/run_object_batch_bench_enhanced.sh index 38d06bedb..230cbe45d 100755 --- a/scripts/run_object_batch_bench_enhanced.sh +++ b/scripts/run_object_batch_bench_enhanced.sh @@ -38,6 +38,13 @@ COOLDOWN_SECS=0 BASELINE_CSV="" EXTRA_ARGS=() FAILED_FINAL_ROUNDS=0 +SERVICE_METRICS_URL="" +SERVICE_METRICS_DIR="" +SERVICE_METRICS_CAPTURE_ATTEMPTS=3 +SERVICE_METRICS_CAPTURE_RETRY_SECS=1 +SERVICE_METRICS_CONNECT_TIMEOUT_SECS=2 +SERVICE_METRICS_MAX_TIME_SECS=15 +SERVICE_METRICS_FILTER_REGEX="rustfs_io_get_object_" usage() { cat <<'USAGE' @@ -77,11 +84,23 @@ Enhanced options: --round-cooldown-secs Compatibility alias for --cooldown-secs --baseline-csv Baseline median CSV to compare --extra-args Extra args appended to tool command, quoted as one string + --service-metrics-url Optional Prometheus scrape URL captured before/after each round attempt + --service-metrics-dir Output directory for per-round service metric snapshots + --service-metrics-attempts Capture attempts for each snapshot (default: 3) + --service-metrics-retry-secs Sleep seconds between failed capture attempts (default: 1) + --service-metrics-connect-timeout-secs + Curl connect timeout for each metrics capture (default: 2) + --service-metrics-max-time-secs + Curl max time for each metrics capture (default: 15) + --service-metrics-filter-regex + Regex for retained metrics lines after scrape + (default: rustfs_io_get_object_) Output files: round_results.csv One row per round attempt (with retry trace) median_summary.csv Median metrics per object size baseline_compare.csv Delta vs baseline (if --baseline-csv is set) + /*.prom Optional per-round service metric snapshots Example: scripts/run_object_batch_bench_enhanced.sh \ @@ -134,6 +153,13 @@ parse_args() { --cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;; --round-cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;; --baseline-csv) BASELINE_CSV="$2"; shift 2 ;; + --service-metrics-url) SERVICE_METRICS_URL="$2"; shift 2 ;; + --service-metrics-dir) SERVICE_METRICS_DIR="$2"; shift 2 ;; + --service-metrics-attempts) SERVICE_METRICS_CAPTURE_ATTEMPTS="$2"; shift 2 ;; + --service-metrics-retry-secs) SERVICE_METRICS_CAPTURE_RETRY_SECS="$2"; shift 2 ;; + --service-metrics-connect-timeout-secs) SERVICE_METRICS_CONNECT_TIMEOUT_SECS="$2"; shift 2 ;; + --service-metrics-max-time-secs) SERVICE_METRICS_MAX_TIME_SECS="$2"; shift 2 ;; + --service-metrics-filter-regex) SERVICE_METRICS_FILTER_REGEX="$2"; shift 2 ;; --extra-args) # shellcheck disable=SC2206 EXTRA_ARGS=($2) @@ -193,6 +219,17 @@ validate_args() { validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round" validate_positive_int "$RETRY_SLEEP_SECS" "--retry-sleep-secs" validate_nonnegative_int "$COOLDOWN_SECS" "--cooldown-secs" + validate_positive_int "$SERVICE_METRICS_CAPTURE_ATTEMPTS" "--service-metrics-attempts" + validate_nonnegative_int "$SERVICE_METRICS_CAPTURE_RETRY_SECS" "--service-metrics-retry-secs" + validate_positive_int "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" "--service-metrics-connect-timeout-secs" + validate_positive_int "$SERVICE_METRICS_MAX_TIME_SECS" "--service-metrics-max-time-secs" + if [[ -n "$SERVICE_METRICS_URL" && -z "$SERVICE_METRICS_DIR" ]]; then + echo "ERROR: --service-metrics-dir is required when --service-metrics-url is set" >&2 + exit 1 + fi + if [[ -n "$SERVICE_METRICS_URL" ]]; then + require_cmd curl + fi if [[ "$TOOL" == "s3bench" ]]; then validate_positive_int "$SAMPLES" "--samples" fi @@ -215,6 +252,9 @@ setup_output() { OUT_DIR="target/bench/object-batch-enhanced-$(date +%Y%m%d-%H%M%S)" fi mkdir -p "$OUT_DIR/logs" + if [[ -n "$SERVICE_METRICS_URL" ]]; then + mkdir -p "$SERVICE_METRICS_DIR" + fi ROUND_CSV="$OUT_DIR/round_results.csv" MEDIAN_CSV="$OUT_DIR/median_summary.csv" @@ -334,6 +374,113 @@ extract_metrics() { echo "$throughput,${reqps_num:-N/A},$latency,$req_p90,$req_p99" } +metric_snapshot_token() { + local size="$1" + local round="$2" + local attempt="$3" + local safe_size + safe_size="$(printf '%s' "$size" | tr -c '[:alnum:]_.-' '_')" + printf '%s_%s_r%s_a%s' "$TOOL" "$safe_size" "$round" "$attempt" +} + +filter_service_metrics_snapshot() { + local input_file="$1" + local output_file="$2" + + if [[ -z "$SERVICE_METRICS_FILTER_REGEX" ]]; then + mv "$input_file" "$output_file" + return 0 + fi + + awk -v pattern="$SERVICE_METRICS_FILTER_REGEX" '$0 ~ pattern { print }' "$input_file" >"$output_file" + if [[ ! -s "$output_file" ]]; then + mv "$input_file" "$output_file" + else + rm -f "$input_file" + fi +} + +capture_round_service_metrics() { + local size="$1" + local round="$2" + local attempt="$3" + local phase="$4" + + if [[ -z "$SERVICE_METRICS_URL" ]]; then + return 0 + fi + + local token snapshot_file status_file tmp_file filtered_file capture_attempt raw_bytes snapshot_bytes + token="$(metric_snapshot_token "$size" "$round" "$attempt")" + snapshot_file="${SERVICE_METRICS_DIR}/${token}_${phase}.prom" + status_file="${SERVICE_METRICS_DIR}/${token}_${phase}.status" + tmp_file="${snapshot_file}.tmp" + filtered_file="${snapshot_file}.filtered.tmp" + + if [[ "$DRY_RUN" == "true" ]]; then + : >"$snapshot_file" + cat >"$status_file" <"$tmp_file"; then + if [[ -s "$tmp_file" ]]; then + raw_bytes="$(wc -c <"$tmp_file" | tr -d '[:space:]')" + filter_service_metrics_snapshot "$tmp_file" "$filtered_file" + mv "$filtered_file" "$snapshot_file" + snapshot_bytes="$(wc -c <"$snapshot_file" | tr -d '[:space:]')" + cat >"$status_file" < 0 )); then + sleep "$SERVICE_METRICS_CAPTURE_RETRY_SECS" + fi + done + + : >"$snapshot_file" + cat >"$status_file" <&2 +} + median_from_numbers() { local values="$1" local count @@ -364,6 +511,7 @@ run_one_attempt() { local status="ok" local exit_code=0 local started_at_utc finished_at_utc + capture_round_service_metrics "$size" "$round" "$attempt" before started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)" if [[ "$TOOL" == "warp" ]]; then @@ -383,7 +531,7 @@ run_one_attempt() { if [[ "$INSECURE" == "true" ]]; then cmd+=("--insecure") fi - if [[ ${EXTRA_ARGS[@]+_} ]]; then + if ((${#EXTRA_ARGS[@]} > 0)); then cmd+=("${EXTRA_ARGS[@]}") fi @@ -414,7 +562,7 @@ run_one_attempt() { if [[ "$INSECURE" == "true" ]]; then cmd+=("-insecure") fi - if [[ ${EXTRA_ARGS[@]+_} ]]; then + if ((${#EXTRA_ARGS[@]} > 0)); then cmd+=("${EXTRA_ARGS[@]}") fi @@ -432,6 +580,7 @@ run_one_attempt() { fi fi finished_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + capture_round_service_metrics "$size" "$round" "$attempt" after local metrics throughput_human reqps latency_human throughput_bps latency_ms req_p90_human req_p90_ms req_p99_human req_p99_ms if [[ "$DRY_RUN" == "true" ]]; then