gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: optimize small GET read paths (#4022)

Browse Source
This commit is contained in:
houseme
2026-07-01 15:40:00 +08:00
committed by GitHub
parent b0f491549a
commit b16120dbcc
25 changed files with 5318 additions and 502 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+132 -46
View File
@@ -893,9 +893,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
version = "1.17.0"
version = "1.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00"
checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad"
dependencies = [
"aws-lc-sys",
"untrusted 0.7.1",
@@ -904,14 +904,15 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
version = "0.41.0"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4"
checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444"
dependencies = [
"cc",
"cmake",
"dunce",
"fs_extra",
"pkg-config",
]
[[package]]
@@ -1715,9 +1716,9 @@ dependencies = [
[[package]]
name = "camino"
version = "1.2.3"
version = "1.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ce8d3bd5823c7504d3f579f13e7b2f3da252fcb938c594d5680ee508bf846f"
checksum = "5f2d30e4173c4026932d51d31d6b0613b1fd3014bf3f9f8943d4ba139c437ba0"
dependencies = [
"serde_core",
]
@@ -5001,9 +5002,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
[[package]]
name = "hybrid-array"
version = "0.4.12"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da"
checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c"
dependencies = [
"ctutils",
"subtle",
@@ -5332,9 +5333,9 @@ dependencies = [
[[package]]
name = "io-uring"
version = "0.7.12"
version = "0.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62"
checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0"
dependencies = [
"bitflags 2.13.0",
"cfg-if",
@@ -5447,9 +5448,9 @@ dependencies = [
[[package]]
name = "jiff"
version = "0.2.29"
version = "0.2.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46"
checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634"
dependencies = [
"defmt",
"jiff-static",
@@ -5463,9 +5464,9 @@ dependencies = [
[[package]]
name = "jiff-static"
version = "0.2.29"
version = "0.2.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f"
checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2"
dependencies = [
"proc-macro2",
"quote",
@@ -5789,18 +5790,18 @@ dependencies = [
[[package]]
name = "liblzma"
version = "0.4.6"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899"
checksum = "45aec2360b3933207e27908049d8e4df4e476b58180afb1e56b2a4fb72efe4ba"
dependencies = [
"liblzma-sys",
]
[[package]]
name = "liblzma-sys"
version = "0.4.6"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6"
checksum = "a046c7f353ba30f810545151e04f63545833803f5b86ee3ddf1517247fe560a5"
dependencies = [
"cc",
"libc",
@@ -7567,11 +7568,12 @@ dependencies = [
[[package]]
name = "pkcs5"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "279a91971a1d8eb1260a30938eae3be9cb67b472dffecb222fbbbe2fd2dc1453"
checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca"
dependencies = [
"aes 0.9.1",
"aes-gcm",
"cbc 0.2.1",
"der 0.8.0",
"pbkdf2 0.13.0",
@@ -7598,7 +7600,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
dependencies = [
"der 0.8.0",
"pkcs5 0.8.0",
"pkcs5 0.8.1",
"rand_core 0.10.1",
"spki 0.8.0",
]
@@ -8227,6 +8229,16 @@ name = "quick-xml"
version = "0.40.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f"
dependencies = [
"memchr",
"serde",
]
[[package]]
name = "quick-xml"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
dependencies = [
"encoding_rs",
"memchr",
@@ -8287,7 +8299,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -8418,9 +8430,9 @@ dependencies = [
[[package]]
name = "rapidhash"
version = "4.4.1"
version = "4.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59"
checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37"
dependencies = [
"rustversion",
]
@@ -8901,7 +8913,7 @@ dependencies = [
"pageant",
"pbkdf2 0.13.0",
"pkcs1 0.8.0-rc.4",
"pkcs5 0.8.0",
"pkcs5 0.8.1",
"pkcs8 0.11.0",
"polyval",
"rand 0.10.1",
@@ -9330,7 +9342,7 @@ dependencies = [
"path-absolutize",
"pin-project-lite",
"proptest",
"quick-xml 0.40.1",
"quick-xml 0.41.0",
"rand 0.10.1",
"ratelimit",
"rcgen",
@@ -9682,7 +9694,7 @@ dependencies = [
"hashbrown 0.17.1",
"metrics",
"percent-encoding",
"quick-xml 0.40.1",
"quick-xml 0.41.0",
"rayon",
"rustc-hash",
"rustfs-config",
@@ -9821,7 +9833,7 @@ dependencies = [
"md5",
"percent-encoding",
"proptest",
"quick-xml 0.40.1",
"quick-xml 0.41.0",
"regex",
"russh",
"russh-sftp",
@@ -10318,9 +10330,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
version = "1.14.1"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
dependencies = [
"web-time",
"zeroize",
@@ -11353,9 +11365,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "suppaftp"
version = "9.0.0"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b367d8e84825dc6e858d2044f8cfff5e80f63b2aa0634c6382fc9a94b6926b22"
checksum = "81b8286a5639930b8fc0400f8a5111439ffb57e5fba6e8276a8bce857ec47b18"
dependencies = [
"async-trait",
"chrono",
@@ -11655,9 +11667,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.51"
version = "0.3.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327"
checksum = "0e48db7b415311b615f910b3dcaa4557bcd4bf1982379c95c223fd8c2a20e210"
dependencies = [
"deranged",
"js-sys",
@@ -11678,9 +11690,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
[[package]]
name = "time-macros"
version = "0.2.30"
version = "0.2.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935"
checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f"
dependencies = [
"num-conv",
"time-core",
@@ -12781,7 +12793,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
@@ -12790,7 +12802,16 @@ version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.5",
]
[[package]]
@@ -12808,14 +12829,31 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
]
[[package]]
@@ -12833,48 +12871,96 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "1.0.3"
Cargo.toml
+7 -7
View File
@@ -168,7 +168,7 @@ byteorder = "1.5.0"
flatbuffers = "25.12.19"
form_urlencoded = "1.2.2"
prost = "0.14.4"
quick-xml = "0.40.1"
quick-xml = "0.41.0"
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
serde = { version = "1.0.228", features = ["derive"] }
@@ -192,7 +192,7 @@ pbkdf2 = "0.13.0"
rsa = { version = "=0.10.0-rc.18" }
rustls = { version = "0.23.41", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
rustls-native-certs = "0.8"
rustls-pki-types = "1.14.1"
rustls-pki-types = "1.15.0"
sha1 = "0.11.0"
sha2 = "0.11.0"
subtle = "2.6"
@@ -201,8 +201,8 @@ zeroize = { version = "1.9.0", features = ["derive"] }
# Time and Date
chrono = { version = "0.4.45", features = ["serde"] }
humantime = "2.3.0"
jiff = { version = "0.2.29", features = ["serde"] }
time = { version = "0.3.51", features = ["std", "parsing", "formatting", "macros", "serde"] }
jiff = { version = "0.2.31", features = ["serde"] }
time = { version = "0.3.52", features = ["std", "parsing", "formatting", "macros", "serde"] }
# Database
deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] }
@@ -211,7 +211,7 @@ tokio-postgres-rustls = "0.14.0"
# Utilities and Tools
anyhow = "1.0.103"
arc-swap = "1.9.1"
arc-swap = "1.9.2"
astral-tokio-tar = "0.6.3"
atoi = "3.1.0"
atomic_enum = "0.3.0"
@@ -323,7 +323,7 @@ pyroscope = { version = "=2.0.5", features = ["backend-pprof-rs", "backend-jemal
# FTP and SFTP
libunftp = { version = "0.23.0", features = ["experimental"] }
unftp-core = "0.1.0"
suppaftp = { version = "9.0.0", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
suppaftp = { version = "10.0.0", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
rcgen = "0.14.8"
russh = { version = "0.61.2", features = ["serde"] }
russh-sftp = "2.3.0"
@@ -338,7 +338,7 @@ tikv-jemallocator = { version = "0.6", features = ["profiling", "stats", "unpref
# Used to control and obtain statistics for jemalloc at runtime
tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats", "profiling"] }
# Snapshot testing for output format regression detection
insta = { version = "1.41", features = ["yaml", "json"] }
insta = { version = "1.48", features = ["yaml", "json"] }
# Used to generate pprof-compatible memory profiling data and support symbolization and flame graphs
jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
# Used to generate CPU performance analysis data and flame diagrams
crates/ecstore/src/client/object_api_utils.rs
+1
View File
@@ -149,6 +149,7 @@ pub fn new_getobjectreader<'a>(
let r = GetObjectReader {
object_info: oi.clone(),
stream: Box::new(input_reader),
buffered_body: None,
};
r
//})
crates/ecstore/src/config/com.rs
+1
View File
@@ -1536,6 +1536,7 @@ mod tests {
_guard: guard,
}),
object_info: self.object_info(bucket, object),
buffered_body: None,
})
}
crates/ecstore/src/data_movement/mod.rs
+1
View File
@@ -1149,6 +1149,7 @@ mod tests {
let rd = GetObjectReader {
stream: Box::new(Cursor::new(raw_payload.clone())),
object_info: object_info.clone(),
buffered_body: None,
};
let mut data = data_movement_put_object_reader("bucket-a", &object_info, rd, "test_migration")
crates/ecstore/src/diagnostics/get.rs
+75
View File
@@ -21,9 +21,13 @@ pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING: &str = "codec_streaming";
pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE: &str = "codec_streaming_legacy_engine";
pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE: &str = "codec_streaming_rustfs_engine";
pub(crate) const GET_OBJECT_PATH_EMPTY: &str = "empty";
pub(crate) const GET_OBJECT_PATH_DIRECT_MEMORY: &str = "direct_memory";
pub(crate) const GET_OBJECT_PATH_INLINE_DIRECT: &str = "inline_direct";
pub(crate) const GET_OBJECT_PATH_LEGACY_DUPLEX: &str = "legacy_duplex";
pub(crate) const GET_OBJECT_PATH_REMOTE_TRANSITION: &str = "remote_transition";
pub(crate) const GET_OBJECT_PATH_SET_DISK: &str = "set_disk";
pub(crate) const GET_DIRECT_MEMORY_SUBPATH_DISK_DATA_BLOCKS: &str = "disk_data_blocks";
pub(crate) const GET_DIRECT_MEMORY_SUBPATH_INLINE_BUFFERED: &str = "inline_buffered";
pub(crate) const GET_CODEC_STREAMING_DECISION_USE: &str = "use";
pub(crate) const GET_CODEC_STREAMING_DECISION_FALLBACK: &str = "fallback";
pub(crate) const GET_CODEC_STREAMING_REASON_NONE: &str = "none";
@@ -42,13 +46,39 @@ pub(crate) const GET_STAGE_FIRST_METADATA_RESPONSE: &str = "first_metadata_respo
pub(crate) const GET_STAGE_FIRST_VALID_METADATA_RESPONSE: &str = "first_valid_metadata_response";
pub(crate) const GET_STAGE_FIRST_SHARD_READ: &str = "first_shard_read";
pub(crate) const GET_STAGE_FULL_BODY: &str = "full_body";
pub(crate) const GET_STAGE_INLINE_PREPARE: &str = "inline_prepare";
pub(crate) const GET_STAGE_LOCK_ACQUIRE: &str = "lock_acquire";
pub(crate) const GET_STAGE_METADATA: &str = "metadata";
pub(crate) const GET_STAGE_METADATA_CACHE_LOOKUP: &str = "metadata_cache_lookup";
pub(crate) const GET_STAGE_METADATA_FANOUT: &str = "metadata_fanout";
pub(crate) const GET_STAGE_METADATA_RESOLVE: &str = "metadata_resolve";
pub(crate) const GET_STAGE_OBJECT_INFO: &str = "object_info";
pub(crate) const GET_STAGE_OUTPUT_LOCK_WAIT: &str = "output_lock_wait";
pub(crate) const GET_STAGE_OUTPUT_POLL: &str = "output_poll";
pub(crate) const GET_STAGE_PATH_DECISION: &str = "path_decision";
pub(crate) const GET_STAGE_QUORUM_REACHED: &str = "quorum_reached";
pub(crate) const GET_STAGE_RANGE: &str = "range";
pub(crate) const GET_STAGE_READER_SETUP: &str = "reader_setup";
pub(crate) const GET_STAGE_READER_SETUP_DROP_PENDING: &str = "reader_setup_drop_pending";
pub(crate) const GET_STAGE_READER_SETUP_SCHEDULE: &str = "reader_setup_schedule";
pub(crate) const GET_STAGE_READER_SETUP_WAIT_QUORUM: &str = "reader_setup_wait_quorum";
pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK: &str = "reader_open_mmap_copy_fallback";
pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS: &str = "reader_open_mmap_copy_success";
pub(crate) const GET_STAGE_READER_OPEN_STREAM: &str = "reader_open_stream";
pub(crate) const GET_STAGE_READER_MMAP_ACCESS_CHECK: &str = "reader_mmap_access_check";
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_TASK: &str = "reader_mmap_blocking_task";
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_WAIT: &str = "reader_mmap_blocking_wait";
pub(crate) const GET_STAGE_READER_MMAP_COPY_BUFFER: &str = "reader_mmap_copy_buffer";
pub(crate) const GET_STAGE_READER_MMAP_DIRECT_READ_COPY: &str = "reader_mmap_direct_read_copy";
pub(crate) const GET_STAGE_READER_MMAP_FILE_OPEN: &str = "reader_mmap_file_open";
pub(crate) const GET_STAGE_READER_MMAP_MAP: &str = "reader_mmap_map";
pub(crate) const GET_STAGE_READER_MMAP_METADATA_LOOKUP: &str = "reader_mmap_metadata_lookup";
pub(crate) const GET_STAGE_READER_MMAP_METADATA_VALIDATE: &str = "reader_mmap_metadata_validate";
pub(crate) const GET_STAGE_READER_MMAP_PATH_RESOLVE: &str = "reader_mmap_path_resolve";
pub(crate) const GET_STAGE_READER_STREAM_FIRST_READ: &str = "reader_stream_first_read";
pub(crate) const GET_STAGE_READER_TASK_BITROT_READER_INIT: &str = "reader_task_bitrot_reader_init";
pub(crate) const GET_STAGE_READER_TASK_FILE_OPEN: &str = "reader_task_file_open";
pub(crate) const GET_STAGE_READER_TASK_READER_CONSTRUCTION: &str = "reader_task_reader_construction";
pub(crate) const GET_STAGE_RECONSTRUCT: &str = "reconstruct";
pub(crate) const GET_STAGE_RESPONSE_HANDOFF: &str = "response_handoff";
pub(crate) const GET_STAGE_SLOWEST_METADATA_RESPONSE: &str = "slowest_metadata_response";
@@ -88,6 +118,25 @@ pub(crate) const GET_METADATA_RESPONSE_NOT_FOUND: &str = "not_found";
pub(crate) const GET_METADATA_RESPONSE_TIMEOUT: &str = "timeout";
pub(crate) const GET_METADATA_RESPONSE_VALID: &str = "valid";
pub(crate) const GET_METADATA_RESPONSE_VERSION_NOT_FOUND: &str = "version_not_found";
pub(crate) const GET_METADATA_CACHE_DECISION_HIT: &str = "hit";
pub(crate) const GET_METADATA_CACHE_DECISION_MISS: &str = "miss";
pub(crate) const GET_METADATA_CACHE_DECISION_REJECT: &str = "reject";
pub(crate) const GET_METADATA_CACHE_DECISION_SKIP: &str = "skip";
pub(crate) const GET_METADATA_CACHE_REASON_DATA_MOVEMENT: &str = "data_movement";
pub(crate) const GET_METADATA_CACHE_REASON_DELETE_MARKER: &str = "delete_marker";
pub(crate) const GET_METADATA_CACHE_REASON_DIST_ERASURE: &str = "dist_erasure";
pub(crate) const GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS: &str = "incl_free_versions";
pub(crate) const GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM: &str = "insufficient_cached_quorum";
pub(crate) const GET_METADATA_CACHE_REASON_META_BUCKET: &str = "meta_bucket";
pub(crate) const GET_METADATA_CACHE_REASON_NO_LOCK: &str = "no_lock";
pub(crate) const GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED: &str = "not_found_or_expired";
pub(crate) const GET_METADATA_CACHE_REASON_NOT_READ_DATA: &str = "not_read_data";
pub(crate) const GET_METADATA_CACHE_REASON_PART_NUMBER: &str = "part_number";
pub(crate) const GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ: &str = "raw_data_movement_read";
pub(crate) const GET_METADATA_CACHE_REASON_USABLE: &str = "usable";
pub(crate) const GET_METADATA_CACHE_REASON_VERSION_ID: &str = "version_id";
pub(crate) const GET_METADATA_CACHE_REASON_VERSION_SUSPENDED: &str = "version_suspended";
pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error";
@@ -258,6 +307,7 @@ mod tests {
assert_eq!(GET_CODEC_STREAMING_OBJECT_CLASS_COMPRESSED, "compressed");
assert_eq!(GET_CODEC_STREAMING_OBJECT_CLASS_REMOTE, "remote");
assert_eq!(GET_CODEC_STREAMING_OBJECT_CLASS_MULTIPART, "multipart");
assert_eq!(GET_OBJECT_PATH_SET_DISK, "set_disk");
assert_eq!(GET_READER_PREFETCH_DIRECT, "direct");
assert_eq!(GET_READER_PREFETCH_STORED, "stored");
assert_eq!(GET_READER_PREFETCH_EOF, "eof");
@@ -275,10 +325,16 @@ mod tests {
assert_eq!(GET_STAGE_FIRST_VALID_METADATA_RESPONSE, "first_valid_metadata_response");
assert_eq!(GET_STAGE_FIRST_SHARD_READ, "first_shard_read");
assert_eq!(GET_STAGE_FULL_BODY, "full_body");
assert_eq!(GET_STAGE_INLINE_PREPARE, "inline_prepare");
assert_eq!(GET_STAGE_LOCK_ACQUIRE, "lock_acquire");
assert_eq!(GET_STAGE_METADATA, "metadata");
assert_eq!(GET_STAGE_METADATA_CACHE_LOOKUP, "metadata_cache_lookup");
assert_eq!(GET_STAGE_METADATA_FANOUT, "metadata_fanout");
assert_eq!(GET_STAGE_METADATA_RESOLVE, "metadata_resolve");
assert_eq!(GET_STAGE_OBJECT_INFO, "object_info");
assert_eq!(GET_STAGE_OUTPUT_LOCK_WAIT, "output_lock_wait");
assert_eq!(GET_STAGE_OUTPUT_POLL, "output_poll");
assert_eq!(GET_STAGE_PATH_DECISION, "path_decision");
assert_eq!(GET_STAGE_QUORUM_REACHED, "quorum_reached");
assert_eq!(GET_STAGE_RANGE, "range");
assert_eq!(GET_STAGE_READER_SETUP, "reader_setup");
@@ -308,6 +364,25 @@ mod tests {
assert_eq!(GET_METADATA_RESPONSE_TIMEOUT, "timeout");
assert_eq!(GET_METADATA_RESPONSE_VALID, "valid");
assert_eq!(GET_METADATA_RESPONSE_VERSION_NOT_FOUND, "version_not_found");
assert_eq!(GET_METADATA_CACHE_DECISION_HIT, "hit");
assert_eq!(GET_METADATA_CACHE_DECISION_MISS, "miss");
assert_eq!(GET_METADATA_CACHE_DECISION_REJECT, "reject");
assert_eq!(GET_METADATA_CACHE_DECISION_SKIP, "skip");
assert_eq!(GET_METADATA_CACHE_REASON_DATA_MOVEMENT, "data_movement");
assert_eq!(GET_METADATA_CACHE_REASON_DELETE_MARKER, "delete_marker");
assert_eq!(GET_METADATA_CACHE_REASON_DIST_ERASURE, "dist_erasure");
assert_eq!(GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, "incl_free_versions");
assert_eq!(GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM, "insufficient_cached_quorum");
assert_eq!(GET_METADATA_CACHE_REASON_META_BUCKET, "meta_bucket");
assert_eq!(GET_METADATA_CACHE_REASON_NO_LOCK, "no_lock");
assert_eq!(GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, "not_found_or_expired");
assert_eq!(GET_METADATA_CACHE_REASON_NOT_READ_DATA, "not_read_data");
assert_eq!(GET_METADATA_CACHE_REASON_PART_NUMBER, "part_number");
assert_eq!(GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, "raw_data_movement_read");
assert_eq!(GET_METADATA_CACHE_REASON_USABLE, "usable");
assert_eq!(GET_METADATA_CACHE_REASON_VERSION_ID, "version_id");
assert_eq!(GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, "version_suspended");
assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error");
crates/ecstore/src/disk/disk_store.rs
+21 -2
View File
@@ -14,8 +14,8 @@
use crate::disk::{
CheckPartsResp, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint, Error,
FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result, UpdateMetadataOpts, VolumeInfo,
WalkDirOptions,
FileInfoVersions, MmapCopyStageMetrics, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result,
UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
health_state::{
RuntimeDriveHealthState, classify_drive_recovery, get_drive_returning_probe_interval,
get_drive_returning_success_threshold, get_drive_suspect_failure_threshold, record_drive_offline_duration,
@@ -1369,6 +1369,25 @@ impl DiskAPI for LocalDiskWrapper {
.await
}
async fn read_file_mmap_copy_with_metrics(
&self,
volume: &str,
path: &str,
offset: usize,
length: usize,
metrics: Option<MmapCopyStageMetrics>,
) -> Result<bytes::Bytes> {
self.track_disk_health(
|| async {
self.disk
.read_file_mmap_copy_with_metrics(volume, path, offset, length, metrics)
.await
},
get_max_timeout_duration(),
)
.await
}
async fn append_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileWriter> {
self.track_disk_health(|| async { self.disk.append_file(volume, path).await }, Duration::ZERO)
.await
crates/ecstore/src/disk/local.rs
+639 -118
View File
@@ -18,14 +18,17 @@ use crate::disk::disk_store::get_object_disk_read_timeout;
use crate::disk::{
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP,
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int,
FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET,
RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE,
STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int,
endpoint::Endpoint,
error::{DiskError, Error, FileAccessDeniedWithContext, Result},
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
format::FormatV3,
fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename},
fs::{
O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, access_std, lstat, lstat_std, remove, remove_all_std,
remove_std, rename,
},
os,
os::{check_path_length, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source},
};
@@ -83,6 +86,130 @@ const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts";
const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed";
const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed";
const EVENT_DISK_LOCAL_FORMAT_DECODE_FAILED: &str = "disk_local_format_decode_failed";
const METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_mmap_page_faults_total";
const METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_direct_read_page_faults_total";
#[inline(always)]
fn record_mmap_copy_stage(metrics: MmapCopyStageMetrics, stage: &'static str, started_at: Option<std::time::Instant>) {
if let Some(started_at) = started_at {
rustfs_io_metrics::record_get_object_stage_duration(metrics.path, stage, started_at.elapsed().as_secs_f64());
}
}
#[cfg(unix)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct MmapPageFaultCounts {
minor: libc::c_long,
major: libc::c_long,
}
#[cfg(unix)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct MmapPageFaultDelta {
minor: u64,
major: u64,
}
#[cfg(all(unix, any(target_os = "linux", target_os = "android")))]
fn mmap_rusage_who() -> libc::c_int {
libc::RUSAGE_THREAD
}
#[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))]
fn mmap_rusage_who() -> libc::c_int {
libc::RUSAGE_SELF
}
#[cfg(unix)]
// SAFETY: this allowance is limited to reading kernel-provided rusage data via
// libc; each unsafe operation below documents pointer validity and initialization.
#[allow(unsafe_code)]
fn read_mmap_page_fault_counts(enabled: bool) -> Option<MmapPageFaultCounts> {
if !enabled {
return None;
}
let mut usage = std::mem::MaybeUninit::<libc::rusage>::uninit();
// SAFETY: `getrusage` writes to the provided `rusage` pointer when it
// returns 0. The pointer is valid for writes and initialized only on success.
let rc = unsafe { libc::getrusage(mmap_rusage_who(), usage.as_mut_ptr()) };
if rc != 0 {
return None;
}
// SAFETY: `getrusage` returned success, so `usage` has been initialized.
let usage = unsafe { usage.assume_init() };
Some(MmapPageFaultCounts {
minor: usage.ru_minflt,
major: usage.ru_majflt,
})
}
#[cfg(unix)]
fn non_negative_fault_delta(before: libc::c_long, after: libc::c_long) -> u64 {
if after <= before {
return 0;
}
u64::try_from(after - before).unwrap_or(u64::MAX)
}
#[cfg(unix)]
fn mmap_page_fault_delta(before: Option<MmapPageFaultCounts>, after: Option<MmapPageFaultCounts>) -> MmapPageFaultDelta {
match (before, after) {
(Some(before), Some(after)) => MmapPageFaultDelta {
minor: non_negative_fault_delta(before.minor, after.minor),
major: non_negative_fault_delta(before.major, after.major),
},
_ => MmapPageFaultDelta::default(),
}
}
#[cfg(unix)]
fn record_mmap_page_fault_delta(path: &'static str, stage: &'static str, delta: MmapPageFaultDelta) {
if delta.minor > 0 {
counter!(
METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL,
"path" => path,
"stage" => stage,
"kind" => "minor",
)
.increment(delta.minor);
}
if delta.major > 0 {
counter!(
METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL,
"path" => path,
"stage" => stage,
"kind" => "major",
)
.increment(delta.major);
}
}
#[cfg(unix)]
fn record_direct_read_page_fault_delta(path: &'static str, stage: &'static str, delta: MmapPageFaultDelta) {
if delta.minor > 0 {
counter!(
METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL,
"path" => path,
"stage" => stage,
"kind" => "minor",
)
.increment(delta.minor);
}
if delta.major > 0 {
counter!(
METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL,
"path" => path,
"stage" => stage,
"kind" => "major",
)
.increment(delta.major);
}
}
/// Enable O_DIRECT for large sequential reads.
/// When enabled, shard reads bypass the page cache using O_DIRECT flag.
@@ -96,6 +223,17 @@ const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false;
/// Default: 4MB.
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD";
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
const ENV_RUSTFS_OBJECT_MMAP_READ_METHOD: &str = "RUSTFS_OBJECT_MMAP_READ_METHOD";
const RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY: &str = "mmap_copy";
const RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY: &str = "direct_read_copy";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum LocalReadCopyMethod {
MmapCopy,
DirectReadCopy,
}
/// Check if O_DIRECT reads are enabled.
fn is_direct_io_read_enabled() -> bool {
@@ -107,6 +245,36 @@ fn get_direct_io_read_threshold() -> usize {
rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD)
}
fn should_populate_mmap_read(length: usize) -> bool {
length > 0 && rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE)
}
fn local_read_copy_method() -> LocalReadCopyMethod {
let method = rustfs_utils::get_env_str(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY);
match method.as_str() {
RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY => LocalReadCopyMethod::DirectReadCopy,
_ => LocalReadCopyMethod::MmapCopy,
}
}
#[cfg(unix)]
#[allow(unsafe_code)]
fn mmap_page_size() -> Result<u64> {
static PAGE_SIZE: OnceLock<Option<u64>> = OnceLock::new();
PAGE_SIZE
.get_or_init(|| {
// SAFETY: `sysconf(_SC_PAGESIZE)` has no pointer arguments and only
// queries process-global OS configuration.
let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
if page_size <= 0 {
return None;
}
u64::try_from(page_size).ok()
})
.ok_or_else(|| DiskError::other("failed to determine system page size"))
}
#[cfg(test)]
static RENAME_DATA_FAIL_BEFORE_OLD_METADATA_BACKUP: std::sync::Mutex<Option<String>> = std::sync::Mutex::new(None);
@@ -1070,62 +1238,21 @@ impl LocalDisk {
// Get the absolute path of an object
pub fn get_object_path(&self, bucket: &str, key: &str) -> Result<PathBuf> {
// For high-frequency paths, use faster string concatenation
let cache_key = if key.is_empty() {
bucket.to_string()
} else {
path_join_buf(&[bucket, key])
};
#[cfg(windows)]
let path = self.root.join(cache_key.replace('/', "\\"));
#[cfg(not(windows))]
let path = self.root.join(cache_key);
self.check_valid_path(&path)?;
Ok(path)
local_disk_object_path(&self.root, bucket, key)
}
// Get the absolute path of a bucket
pub fn get_bucket_path(&self, bucket: &str) -> Result<PathBuf> {
#[cfg(windows)]
let bucket_path = self.root.join(bucket.replace('/', "\\"));
#[cfg(not(windows))]
let bucket_path = self.root.join(bucket);
self.check_valid_path(&bucket_path)?;
Ok(bucket_path)
local_disk_bucket_path(&self.root, bucket)
}
// Check if a path is valid
fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> {
let path = normalize_path_components(path);
if !path.starts_with(&self.root) {
return Err(DiskError::InvalidPath);
}
self.reject_symlink_components(&path)
check_local_disk_valid_path(&self.root, path)
}
fn reject_symlink_components(&self, path: &Path) -> Result<()> {
let relative = path.strip_prefix(&self.root).map_err(|_| DiskError::InvalidPath)?;
let mut current = self.root.clone();
for component in relative.components() {
current.push(component.as_os_str());
match lstat_std(&current) {
Ok(metadata) => {
if metadata.file_type().is_symlink() {
return Err(DiskError::InvalidPath);
}
}
Err(err) if err.kind() == ErrorKind::NotFound => break,
Err(err) => return Err(to_file_error(err).into()),
}
}
Ok(())
reject_local_disk_symlink_components(&self.root, path)
}
// Batch path generation with single lock acquisition
@@ -2085,6 +2212,62 @@ fn skip_access_checks(p: impl AsRef<str>) -> bool {
false
}
fn local_disk_object_path(root: &Path, bucket: &str, key: &str) -> Result<PathBuf> {
let cache_key = if key.is_empty() {
bucket.to_string()
} else {
path_join_buf(&[bucket, key])
};
#[cfg(windows)]
let path = root.join(cache_key.replace('/', "\\"));
#[cfg(not(windows))]
let path = root.join(cache_key);
check_local_disk_valid_path(root, &path)?;
Ok(path)
}
fn local_disk_bucket_path(root: &Path, bucket: &str) -> Result<PathBuf> {
#[cfg(windows)]
let bucket_path = root.join(bucket.replace('/', "\\"));
#[cfg(not(windows))]
let bucket_path = root.join(bucket);
check_local_disk_valid_path(root, &bucket_path)?;
Ok(bucket_path)
}
fn check_local_disk_valid_path(root: &Path, path: impl AsRef<Path>) -> Result<()> {
let path = normalize_path_components(path);
if !path.starts_with(root) {
return Err(DiskError::InvalidPath);
}
reject_local_disk_symlink_components(root, &path)
}
fn reject_local_disk_symlink_components(root: &Path, path: &Path) -> Result<()> {
let relative = path.strip_prefix(root).map_err(|_| DiskError::InvalidPath)?;
let mut current = root.to_path_buf();
for component in relative.components() {
current.push(component.as_os_str());
match lstat_std(&current) {
Ok(metadata) => {
if metadata.file_type().is_symlink() {
return Err(DiskError::InvalidPath);
}
}
Err(err) if err.kind() == ErrorKind::NotFound => break,
Err(err) => return Err(to_file_error(err).into()),
}
}
Ok(())
}
// Lightweight path normalization without filesystem calls
fn normalize_path_components(path: impl AsRef<Path>) -> PathBuf {
let path = path.as_ref();
@@ -2784,105 +2967,303 @@ impl DiskAPI for LocalDisk {
#[allow(unsafe_code)]
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Bytes> {
let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) {
access(&volume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
let file_path = self.get_object_path(volume, path)?;
check_path_length(file_path.to_string_lossy().as_ref())?;
// Verify file exists and get metadata
let file_path_clone = file_path.clone();
let meta = tokio::task::spawn_blocking(move || std::fs::metadata(&file_path_clone).map_err(DiskError::from))
self.read_file_mmap_copy_with_metrics(volume, path, offset, length, None)
.await
.map_err(DiskError::from)??;
}
let end_offset = offset.checked_add(length).ok_or(DiskError::FileCorrupt)?;
if meta.len() < end_offset as u64 {
error!(
event = EVENT_DISK_LOCAL_READ_VERSION_FALLBACK,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
volume,
path,
offset,
length,
actual_size = meta.len(),
reason = "read_file_mmap_copy_out_of_bounds",
"Disk local read fallback failed"
);
/// File read using mmap-then-copy on Unix or efficient read on non-Unix.
// SAFETY: Unix unsafe calls in this function only query page size and mmap
// a read-only file region after bounds and alignment are validated.
#[allow(unsafe_code)]
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file_mmap_copy_with_metrics(
&self,
volume: &str,
path: &str,
offset: usize,
length: usize,
metrics: Option<MmapCopyStageMetrics>,
) -> Result<Bytes> {
let metrics = metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
let metrics_enabled = metrics.is_some();
let metadata_validate_start = metrics_enabled.then(std::time::Instant::now);
let Some(end_offset) = offset.checked_add(length) else {
if let Some(metrics) = metrics {
record_mmap_copy_stage(metrics, metrics.metadata_validate_stage, metadata_validate_start);
}
return Err(DiskError::FileCorrupt);
}
};
// Unix: use mmap to read the data (copies into Bytes for safe ownership)
// Non-Unix: fall back to efficient read
#[cfg(unix)]
{
use memmap2::MmapOptions;
use std::time::Instant;
use std::time::{Duration as StdDuration, Instant as StdInstant};
let start = Instant::now();
let file_path_clone = file_path.clone();
struct MmapCopyReadResult {
bytes: Bytes,
access_check_duration: StdDuration,
path_resolve_duration: StdDuration,
metadata_lookup_duration: StdDuration,
metadata_validate_duration: StdDuration,
file_open_duration: StdDuration,
mmap_map_duration: StdDuration,
mmap_copy_duration: StdDuration,
direct_read_copy_duration: StdDuration,
mmap_map_fault_delta: MmapPageFaultDelta,
mmap_copy_fault_delta: MmapPageFaultDelta,
direct_read_copy_fault_delta: MmapPageFaultDelta,
blocking_task_duration: StdDuration,
}
enum MmapCopyReadError {
Disk(DiskError),
OutOfBounds { actual_size: u64 },
}
impl From<DiskError> for MmapCopyReadError {
fn from(err: DiskError) -> Self {
Self::Disk(err)
}
}
let start = StdInstant::now();
let root = self.root.clone();
let volume_owned = volume.to_owned();
let path_owned = path.to_owned();
let should_reclaim_after_read = should_reclaim_file_cache_after_read(length);
let bytes = tokio::task::spawn_blocking(move || {
let file = std::fs::File::open(&file_path_clone).map_err(DiskError::from)?;
let should_populate_mmap_read = should_populate_mmap_read(length);
let read_copy_method = local_read_copy_method();
let offset_u64 = u64::try_from(offset).map_err(|_| DiskError::FileCorrupt)?;
let end_offset_u64 = u64::try_from(end_offset).map_err(|_| DiskError::FileCorrupt)?;
let blocking_wait_start = metrics_enabled.then(std::time::Instant::now);
let read_result = tokio::task::spawn_blocking(move || {
let blocking_task_start = metrics_enabled.then(StdInstant::now);
let access_check_start = metrics_enabled.then(StdInstant::now);
let volume_dir = local_disk_bucket_path(&root, &volume_owned)?;
if !skip_access_checks(&volume_owned) {
access_std(&volume_dir).map_err(|e| DiskError::from(to_access_error(e, DiskError::VolumeAccessDenied)))?;
}
let access_check_duration = access_check_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
let path_resolve_start = metrics_enabled.then(StdInstant::now);
let file_path = local_disk_object_path(&root, &volume_owned, &path_owned)?;
check_path_length(file_path.to_string_lossy().as_ref())?;
let path_resolve_duration = path_resolve_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
let file_open_start = metrics_enabled.then(StdInstant::now);
let mut file = std::fs::File::open(&file_path).map_err(DiskError::from)?;
let file_open_duration = file_open_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
let metadata_lookup_start = metrics_enabled.then(StdInstant::now);
let meta = file.metadata().map_err(DiskError::from)?;
let metadata_lookup_duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
let metadata_validate_start = metrics_enabled.then(StdInstant::now);
if meta.len() < end_offset_u64 {
return Err(MmapCopyReadError::OutOfBounds { actual_size: meta.len() });
}
let metadata_validate_duration =
metadata_validate_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
#[cfg(target_os = "macos")]
if should_reclaim_after_read {
let _ = set_std_fd_nocache(&file);
}
// mmap offsets on Unix must be page-size aligned. Align the
// mapping down to the nearest page boundary, then slice out the
// originally requested logical range.
// SAFETY: `sysconf(_SC_PAGESIZE)` has no pointer arguments and
// only queries process-global OS configuration.
let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
if page_size <= 0 {
return Err(DiskError::other("failed to determine system page size"));
}
let page_size = page_size as u64;
let offset_u64 = offset as u64;
let aligned_offset = offset_u64 - (offset_u64 % page_size);
let logical_offset = (offset_u64 - aligned_offset) as usize;
let map_len = logical_offset
.checked_add(length)
.ok_or_else(|| DiskError::other("mmap length overflow"))?;
let mut mmap_map_duration = StdDuration::ZERO;
let mut mmap_copy_duration = StdDuration::ZERO;
let mut direct_read_copy_duration = StdDuration::ZERO;
let mut mmap_map_fault_delta = MmapPageFaultDelta::default();
let mut mmap_copy_fault_delta = MmapPageFaultDelta::default();
let mut direct_read_copy_fault_delta = MmapPageFaultDelta::default();
let mut _reclaim_offset = offset_u64;
let mut _reclaim_len = length;
// SAFETY: The file is opened as read-only, and we're mapping a region
// that we've already verified exists and is within file bounds. The
// file offset passed to mmap is page-size aligned as required on Unix.
let mmap =
unsafe { MmapOptions::new().offset(aligned_offset).len(map_len).map(&file) }.map_err(DiskError::other)?;
let bytes = match read_copy_method {
LocalReadCopyMethod::MmapCopy => {
// mmap offsets on Unix must be page-size aligned. Align the
// mapping down to the nearest page boundary, then slice out the
// originally requested logical range.
let page_size = mmap_page_size()?;
let aligned_offset = offset_u64 - (offset_u64 % page_size);
let logical_offset =
usize::try_from(offset_u64 - aligned_offset).map_err(|_| DiskError::other("mmap offset overflow"))?;
let map_len = logical_offset
.checked_add(length)
.ok_or_else(|| DiskError::other("mmap length overflow"))?;
_reclaim_offset = aligned_offset;
_reclaim_len = map_len;
// Copy only the requested logical range into a Bytes buffer. This
// avoids undefined behavior from treating OS-managed mmap memory as
// allocator-managed Vec storage, at the cost of an extra copy.
let end = logical_offset
.checked_add(length)
.ok_or_else(|| DiskError::other("mmap slice length overflow"))?;
let bytes = Bytes::copy_from_slice(&mmap[logical_offset..end]);
// SAFETY: The file is opened as read-only, and we're mapping a region
// that we've already verified exists and is within file bounds. The
// file offset passed to mmap is page-size aligned as required on Unix.
let mmap_map_start = metrics_enabled.then(StdInstant::now);
let mmap_map_faults_before = read_mmap_page_fault_counts(metrics_enabled);
let mut mmap_options = MmapOptions::new();
mmap_options.offset(aligned_offset).len(map_len);
if should_populate_mmap_read {
mmap_options.populate();
}
let mmap = unsafe { mmap_options.map(&file) }.map_err(DiskError::other)?;
let mmap_map_faults_after = read_mmap_page_fault_counts(metrics_enabled);
mmap_map_duration = mmap_map_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
mmap_map_fault_delta = mmap_page_fault_delta(mmap_map_faults_before, mmap_map_faults_after);
// Copy only the requested logical range into a Bytes buffer. This
// avoids undefined behavior from treating OS-managed mmap memory as
// allocator-managed Vec storage, at the cost of an extra copy.
let end = logical_offset
.checked_add(length)
.ok_or_else(|| DiskError::other("mmap slice length overflow"))?;
let mmap_copy_start = metrics_enabled.then(StdInstant::now);
let mmap_copy_faults_before = read_mmap_page_fault_counts(metrics_enabled);
let bytes = Bytes::copy_from_slice(&mmap[logical_offset..end]);
let mmap_copy_faults_after = read_mmap_page_fault_counts(metrics_enabled);
mmap_copy_duration = mmap_copy_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
mmap_copy_fault_delta = mmap_page_fault_delta(mmap_copy_faults_before, mmap_copy_faults_after);
bytes
}
LocalReadCopyMethod::DirectReadCopy => {
use std::io::{Read as _, Seek as _};
let direct_read_copy_start = metrics_enabled.then(StdInstant::now);
let direct_read_copy_faults_before = read_mmap_page_fault_counts(metrics_enabled);
file.seek(SeekFrom::Start(offset_u64)).map_err(DiskError::from)?;
let mut buffer = vec![0; length];
file.read_exact(&mut buffer).map_err(DiskError::from)?;
let direct_read_copy_faults_after = read_mmap_page_fault_counts(metrics_enabled);
direct_read_copy_duration =
direct_read_copy_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
direct_read_copy_fault_delta =
mmap_page_fault_delta(direct_read_copy_faults_before, direct_read_copy_faults_after);
Bytes::from(buffer)
}
};
#[cfg(target_os = "linux")]
if should_reclaim_after_read {
if should_reclaim_after_read && _reclaim_len > 0 {
use core::num::NonZeroU64;
use rustix::fs::{Advice, fadvise};
let reclaim_len =
NonZeroU64::new(map_len as u64).ok_or_else(|| DiskError::other("mmap reclaim length overflow"))?;
fadvise(&file, aligned_offset, Some(reclaim_len), Advice::DontNeed)
let reclaim_len = NonZeroU64::new(
u64::try_from(_reclaim_len).map_err(|_| DiskError::other("read reclaim length overflow"))?,
)
.ok_or_else(|| DiskError::other("read reclaim length overflow"))?;
fadvise(&file, _reclaim_offset, Some(reclaim_len), Advice::DontNeed)
.map_err(std::io::Error::from)
.map_err(DiskError::from)?;
}
Ok::<Bytes, DiskError>(bytes)
let blocking_task_duration = blocking_task_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
Ok::<MmapCopyReadResult, MmapCopyReadError>(MmapCopyReadResult {
bytes,
access_check_duration,
path_resolve_duration,
metadata_lookup_duration,
metadata_validate_duration,
file_open_duration,
mmap_map_duration,
mmap_copy_duration,
direct_read_copy_duration,
mmap_map_fault_delta,
mmap_copy_fault_delta,
direct_read_copy_fault_delta,
blocking_task_duration,
})
})
.await
.map_err(DiskError::from)??;
.map_err(DiskError::from)
.map_err(MmapCopyReadError::Disk)
.and_then(|result| result);
if let Some(metrics) = metrics {
record_mmap_copy_stage(metrics, metrics.blocking_wait_stage, blocking_wait_start);
}
let read_result = match read_result {
Ok(read_result) => read_result,
Err(MmapCopyReadError::Disk(err)) => return Err(err),
Err(MmapCopyReadError::OutOfBounds { actual_size }) => {
error!(
event = EVENT_DISK_LOCAL_READ_VERSION_FALLBACK,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
volume,
path,
offset,
length,
actual_size,
reason = "read_file_mmap_copy_out_of_bounds",
"Disk local read fallback failed"
);
return Err(DiskError::FileCorrupt);
}
};
if metrics_enabled && let Some(metrics) = metrics {
rustfs_io_metrics::record_get_object_stage_duration(
metrics.path,
metrics.blocking_task_stage,
read_result.blocking_task_duration.as_secs_f64(),
);
rustfs_io_metrics::record_get_object_stage_duration(
metrics.path,
metrics.access_check_stage,
read_result.access_check_duration.as_secs_f64(),
);
rustfs_io_metrics::record_get_object_stage_duration(
metrics.path,
metrics.path_resolve_stage,
read_result.path_resolve_duration.as_secs_f64(),
);
rustfs_io_metrics::record_get_object_stage_duration(
metrics.path,
metrics.metadata_lookup_stage,
read_result.metadata_lookup_duration.as_secs_f64(),
);
rustfs_io_metrics::record_get_object_stage_duration(
metrics.path,
metrics.metadata_validate_stage,
read_result.metadata_validate_duration.as_secs_f64(),
);
rustfs_io_metrics::record_get_object_stage_duration(
metrics.path,
metrics.file_open_stage,
read_result.file_open_duration.as_secs_f64(),
);
match read_copy_method {
LocalReadCopyMethod::MmapCopy => {
rustfs_io_metrics::record_get_object_stage_duration(
metrics.path,
metrics.mmap_map_stage,
read_result.mmap_map_duration.as_secs_f64(),
);
rustfs_io_metrics::record_get_object_stage_duration(
metrics.path,
metrics.mmap_copy_stage,
read_result.mmap_copy_duration.as_secs_f64(),
);
record_mmap_page_fault_delta(metrics.path, metrics.mmap_map_stage, read_result.mmap_map_fault_delta);
record_mmap_page_fault_delta(metrics.path, metrics.mmap_copy_stage, read_result.mmap_copy_fault_delta);
}
LocalReadCopyMethod::DirectReadCopy => {
rustfs_io_metrics::record_get_object_stage_duration(
metrics.path,
metrics.direct_read_copy_stage,
read_result.direct_read_copy_duration.as_secs_f64(),
);
record_direct_read_page_fault_delta(
metrics.path,
metrics.direct_read_copy_stage,
read_result.direct_read_copy_fault_delta,
);
}
}
}
let bytes = read_result.bytes;
// Log successful mmap read metrics
let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
@@ -2890,7 +3271,13 @@ impl DiskAPI for LocalDisk {
// Record mmap read metrics
rustfs_io_metrics::record_zero_copy_read(length, duration_ms);
debug!(size = length, duration_ms = duration_ms, "mmap_read_success");
debug!(
size = length,
duration_ms = duration_ms,
mmap_populate = should_populate_mmap_read,
read_copy_method = ?read_copy_method,
"mmap_read_success"
);
return Ok(bytes);
}
@@ -2903,6 +3290,58 @@ impl DiskAPI for LocalDisk {
debug!(reason = "non_unix_platform", "zero_copy_fallback");
let access_check_start = metrics_enabled.then(std::time::Instant::now);
let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) {
access(&volume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
if let Some(metrics) = metrics {
record_mmap_copy_stage(metrics, metrics.access_check_stage, access_check_start);
}
let path_resolve_start = metrics_enabled.then(std::time::Instant::now);
let file_path = self.get_object_path(volume, path)?;
check_path_length(file_path.to_string_lossy().as_ref())?;
if let Some(metrics) = metrics {
record_mmap_copy_stage(metrics, metrics.path_resolve_stage, path_resolve_start);
}
let file_path_clone = file_path.clone();
let metadata_lookup_start = metrics_enabled.then(std::time::Instant::now);
let meta_result = tokio::task::spawn_blocking(move || std::fs::metadata(&file_path_clone).map_err(DiskError::from))
.await
.map_err(DiskError::from)
.and_then(|result| result);
if let Some(metrics) = metrics {
record_mmap_copy_stage(metrics, metrics.metadata_lookup_stage, metadata_lookup_start);
}
let meta = meta_result?;
let metadata_validate_start = metrics_enabled.then(std::time::Instant::now);
if meta.len() < end_offset as u64 {
if let Some(metrics) = metrics {
record_mmap_copy_stage(metrics, metrics.metadata_validate_stage, metadata_validate_start);
}
error!(
event = EVENT_DISK_LOCAL_READ_VERSION_FALLBACK,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
volume,
path,
offset,
length,
actual_size = meta.len(),
reason = "read_file_mmap_copy_out_of_bounds",
"Disk local read fallback failed"
);
return Err(DiskError::FileCorrupt);
}
if let Some(metrics) = metrics {
record_mmap_copy_stage(metrics, metrics.metadata_validate_stage, metadata_validate_start);
}
let mut f = self.open_file(file_path, O_RDONLY, volume_dir).await?;
if offset > 0 {
@@ -5494,6 +5933,88 @@ mod test {
});
}
#[test]
fn should_populate_mmap_read_respects_env() {
temp_env::with_var_unset(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, || {
assert!(!should_populate_mmap_read(512 * 1024));
});
temp_env::with_var(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, Some("true"), || {
assert!(should_populate_mmap_read(512 * 1024));
assert!(!should_populate_mmap_read(0));
});
temp_env::with_var(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, Some("false"), || {
assert!(!should_populate_mmap_read(512 * 1024));
});
}
#[test]
fn local_read_copy_method_respects_env() {
temp_env::with_var_unset(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, || {
assert_eq!(local_read_copy_method(), LocalReadCopyMethod::MmapCopy);
});
temp_env::with_var(
ENV_RUSTFS_OBJECT_MMAP_READ_METHOD,
Some(RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY),
|| {
assert_eq!(local_read_copy_method(), LocalReadCopyMethod::DirectReadCopy);
},
);
temp_env::with_var(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, Some("unknown"), || {
assert_eq!(local_read_copy_method(), LocalReadCopyMethod::MmapCopy);
});
}
#[cfg(unix)]
#[test]
fn mmap_page_size_is_cached_positive() {
let first = mmap_page_size().expect("page size should be available");
let second = mmap_page_size().expect("cached page size should be available");
assert!(first > 0);
assert_eq!(first, second);
}
#[cfg(unix)]
#[tokio::test]
async fn read_file_mmap_copy_supports_direct_read_copy_method() {
use tempfile::tempdir;
temp_env::async_with_vars(
[(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, Some(RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY))],
async {
let root_dir = tempdir().expect("operation should succeed");
let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("operation should succeed");
let disk = LocalDisk::new(&endpoint, false).await.expect("operation should succeed");
disk.make_volume("test-volume").await.expect("operation should succeed");
disk.write_all("test-volume", "test-file.txt", Bytes::from_static(b"0123456789abcdef"))
.await
.expect("operation should succeed");
let data = disk
.read_file_mmap_copy("test-volume", "test-file.txt", 4, 6)
.await
.expect("operation should succeed");
assert_eq!(data, Bytes::from_static(b"456789"));
},
)
.await;
}
#[cfg(unix)]
#[test]
fn mmap_page_fault_delta_clamps_non_monotonic_counts() {
let before = Some(MmapPageFaultCounts { minor: 10, major: 4 });
let after = Some(MmapPageFaultCounts { minor: 7, major: 6 });
assert_eq!(mmap_page_fault_delta(before, after), MmapPageFaultDelta { minor: 0, major: 2 });
assert_eq!(mmap_page_fault_delta(before, None), MmapPageFaultDelta::default());
}
#[test]
fn test_is_bitrot_size_mismatch_error_only_matches_target_message() {
assert!(is_bitrot_size_mismatch_error(&std::io::Error::other("bitrot shard file size mismatch")));
crates/ecstore/src/disk/mod.rs
+44
View File
@@ -59,6 +59,21 @@ pub type DiskStore = Arc<Disk>;
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
pub type FileWriter = Box<dyn AsyncWrite + Send + Sync + Unpin>;
#[derive(Clone, Copy, Debug)]
pub struct MmapCopyStageMetrics {
pub(crate) path: &'static str,
pub(crate) access_check_stage: &'static str,
pub(crate) path_resolve_stage: &'static str,
pub(crate) metadata_lookup_stage: &'static str,
pub(crate) metadata_validate_stage: &'static str,
pub(crate) blocking_wait_stage: &'static str,
pub(crate) blocking_task_stage: &'static str,
pub(crate) file_open_stage: &'static str,
pub(crate) mmap_map_stage: &'static str,
pub(crate) mmap_copy_stage: &'static str,
pub(crate) direct_read_copy_stage: &'static str,
}
#[derive(Debug)]
pub enum Disk {
Local(Box<LocalDiskWrapper>),
@@ -301,6 +316,24 @@ impl DiskAPI for Disk {
}
}
async fn read_file_mmap_copy_with_metrics(
&self,
volume: &str,
path: &str,
offset: usize,
length: usize,
metrics: Option<MmapCopyStageMetrics>,
) -> Result<Bytes> {
match self {
Disk::Local(local_disk) => {
local_disk
.read_file_mmap_copy_with_metrics(volume, path, offset, length, metrics)
.await
}
Disk::Remote(remote_disk) => remote_disk.read_file_mmap_copy(volume, path, offset, length).await,
}
}
#[tracing::instrument(skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
match self {
@@ -564,6 +597,17 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
/// File read using mmap-then-copy on Unix or an efficient read on non-Unix.
async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Bytes>;
async fn read_file_mmap_copy_with_metrics(
&self,
volume: &str,
path: &str,
offset: usize,
length: usize,
_metrics: Option<MmapCopyStageMetrics>,
) -> Result<Bytes> {
self.read_file_mmap_copy(volume, path, offset, length).await
}
/// Historical name for `read_file_mmap_copy`.
#[deprecated(
since = "1.0.0-beta.8",
crates/ecstore/src/io_support/bitrot.rs
+289 -16
View File
@@ -12,7 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, error::DiskError};
use crate::diagnostics::get::{
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT,
GET_STAGE_READER_MMAP_COPY_BUFFER, GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN,
GET_STAGE_READER_MMAP_MAP, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
};
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
use crate::erasure::coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
use bytes::Bytes;
use rustfs_config::{DEFAULT_OBJECT_MMAP_READ_ENABLE, ENV_OBJECT_MMAP_READ_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
@@ -29,6 +36,14 @@ use tracing::debug;
type BoxedObjectReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
type OpenObjectReaderFuture = Pin<Box<dyn Future<Output = disk::error::Result<Option<BoxedObjectReader>>> + Send>>;
#[derive(Clone, Copy)]
pub(crate) struct BitrotReaderStageMetrics {
pub(crate) path: &'static str,
pub(crate) reader_construction_stage: &'static str,
pub(crate) file_open_stage: &'static str,
pub(crate) bitrot_reader_init_stage: &'static str,
}
pub(crate) fn object_mmap_read_enabled() -> bool {
rustfs_utils::get_env_bool_with_aliases(
ENV_OBJECT_MMAP_READ_ENABLE,
@@ -46,6 +61,7 @@ struct BitrotReaderSource {
offset: usize,
length: usize,
use_mmap_read: bool,
stage_metrics: Option<BitrotReaderStageMetrics>,
}
impl BitrotReaderSource {
@@ -56,15 +72,67 @@ impl BitrotReaderSource {
rd.set_position(offset);
Ok(Some(Box::new(rd)))
} else if let Some(disk) = self.disk {
open_disk_reader(&disk, &self.bucket, &self.path, self.offset, self.length, self.use_mmap_read)
.await
.map(Some)
open_disk_reader(
&disk,
&self.bucket,
&self.path,
self.offset,
self.length,
self.use_mmap_read,
self.stage_metrics.map(|metrics| metrics.path),
)
.await
.map(Some)
} else {
Ok(None)
}
}
}
struct FirstReadMetricsReader {
inner: FileReader,
metrics_path: &'static str,
stage: &'static str,
started_at: Option<Instant>,
recorded: bool,
}
impl FirstReadMetricsReader {
fn new(inner: FileReader, metrics_path: &'static str, stage: &'static str) -> Self {
Self {
inner,
metrics_path,
stage,
started_at: None,
recorded: false,
}
}
}
impl AsyncRead for FirstReadMetricsReader {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
if self.recorded {
return Pin::new(&mut self.inner).poll_read(cx, buf);
}
let filled_before = buf.filled().len();
if self.started_at.is_none() {
self.started_at = Some(Instant::now());
}
match Pin::new(&mut self.inner).poll_read(cx, buf) {
Poll::Ready(Ok(())) => {
if buf.filled().len() > filled_before {
self.recorded = true;
record_get_stage_duration_if_enabled(self.metrics_path, self.stage, self.started_at.take());
}
Poll::Ready(Ok(()))
}
other => other,
}
}
}
struct DeferredObjectReader {
state: Mutex<DeferredObjectReaderState>,
}
@@ -146,14 +214,38 @@ async fn open_disk_reader(
offset: usize,
length: usize,
use_mmap_read: bool,
metrics_path: Option<&'static str>,
) -> disk::error::Result<FileReader> {
let metrics_path = metrics_path.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
let stage_metrics_enabled = metrics_path.is_some();
if use_mmap_read && disk.is_local() {
let start = Instant::now();
match disk.read_file_mmap_copy(bucket, path, offset, length).await {
let start = stage_metrics_enabled.then(Instant::now);
let zero_copy_start = Instant::now();
let mmap_metrics = metrics_path.map(|metrics_path| MmapCopyStageMetrics {
path: metrics_path,
access_check_stage: GET_STAGE_READER_MMAP_ACCESS_CHECK,
path_resolve_stage: GET_STAGE_READER_MMAP_PATH_RESOLVE,
metadata_lookup_stage: GET_STAGE_READER_MMAP_METADATA_LOOKUP,
metadata_validate_stage: GET_STAGE_READER_MMAP_METADATA_VALIDATE,
blocking_wait_stage: GET_STAGE_READER_MMAP_BLOCKING_WAIT,
blocking_task_stage: GET_STAGE_READER_MMAP_BLOCKING_TASK,
file_open_stage: GET_STAGE_READER_MMAP_FILE_OPEN,
mmap_map_stage: GET_STAGE_READER_MMAP_MAP,
mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
});
match disk
.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
.await
{
Ok(bytes) => {
let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
let duration_ms = zero_copy_start.elapsed().as_secs_f64() * 1000.0;
rustfs_io_metrics::record_zero_copy_read(bytes.len(), duration_ms);
if let Some(metrics_path) = metrics_path {
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS, start);
}
debug!(
size = bytes.len(),
path = %path,
@@ -163,6 +255,9 @@ async fn open_disk_reader(
return Ok(Box::new(Cursor::new(bytes)));
}
Err(err) => {
if let Some(metrics_path) = metrics_path {
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, start);
}
let reason = format!("{err:?}");
rustfs_io_metrics::record_zero_copy_fallback(&reason);
debug!(
@@ -171,15 +266,36 @@ async fn open_disk_reader(
"zero_copy_fallback"
);
return match disk.read_file_stream(bucket, path, offset, length).await {
Ok(reader) => Ok(reader),
let stream_start = stage_metrics_enabled.then(Instant::now);
let stream_result = disk.read_file_stream(bucket, path, offset, length).await;
if let Some(metrics_path) = metrics_path {
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_STREAM, stream_start);
}
return match stream_result {
Ok(reader) => Ok(wrap_first_read_metrics(reader, metrics_path)),
Err(_) => Err(err),
};
}
}
}
disk.read_file_stream(bucket, path, offset, length).await
let stream_start = stage_metrics_enabled.then(Instant::now);
let reader = disk.read_file_stream(bucket, path, offset, length).await?;
if let Some(metrics_path) = metrics_path {
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_STREAM, stream_start);
}
Ok(wrap_first_read_metrics(reader, metrics_path))
}
fn wrap_first_read_metrics(reader: FileReader, metrics_path: Option<&'static str>) -> FileReader {
if let Some(metrics_path) = metrics_path
&& rustfs_io_metrics::get_stage_metrics_enabled()
{
return Box::new(FirstReadMetricsReader::new(reader, metrics_path, GET_STAGE_READER_STREAM_FIRST_READ));
}
reader
}
fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksum_algo: HashAlgorithm) -> (usize, usize) {
@@ -215,21 +331,131 @@ pub async fn create_bitrot_reader(
skip_verify: bool,
use_mmap_read: bool,
) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
create_bitrot_reader_with_stage_metrics(
inline_data,
disk,
bucket,
path,
offset,
length,
shard_size,
checksum_algo,
skip_verify,
use_mmap_read,
None,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn create_bitrot_reader_with_stage_metrics(
inline_data: Option<&[u8]>,
disk: Option<&DiskStore>,
bucket: &str,
path: &str,
offset: usize,
length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify: bool,
use_mmap_read: bool,
stage_metrics: Option<BitrotReaderStageMetrics>,
) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
create_bitrot_reader_from_bytes_with_stage_metrics(
inline_data.map(Bytes::copy_from_slice),
disk,
bucket,
path,
offset,
length,
shard_size,
checksum_algo,
skip_verify,
use_mmap_read,
stage_metrics,
)
.await
}
/// Create a BitrotReader from owned inline Bytes or a disk file stream.
///
/// Passing `Bytes` preserves the shared inline data buffer and avoids copying
/// shard payloads that are already owned by metadata.
#[allow(clippy::too_many_arguments)]
pub async fn create_bitrot_reader_from_bytes(
inline_data: Option<Bytes>,
disk: Option<&DiskStore>,
bucket: &str,
path: &str,
offset: usize,
length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify: bool,
use_mmap_read: bool,
) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
create_bitrot_reader_from_bytes_with_stage_metrics(
inline_data,
disk,
bucket,
path,
offset,
length,
shard_size,
checksum_algo,
skip_verify,
use_mmap_read,
None,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn create_bitrot_reader_from_bytes_with_stage_metrics(
inline_data: Option<Bytes>,
disk: Option<&DiskStore>,
bucket: &str,
path: &str,
offset: usize,
length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify: bool,
use_mmap_read: bool,
stage_metrics: Option<BitrotReaderStageMetrics>,
) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
let stage_metrics = stage_metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
let stage_metrics_enabled = stage_metrics.is_some();
let reader_construction_start = stage_metrics_enabled.then(Instant::now);
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
let source = BitrotReaderSource {
inline_data: inline_data.map(Bytes::copy_from_slice),
inline_data,
disk: disk.cloned(),
bucket: bucket.to_string(),
path: path.to_string(),
offset,
length,
use_mmap_read,
stage_metrics,
};
if let Some(metrics) = stage_metrics {
record_get_stage_duration_if_enabled(metrics.path, metrics.reader_construction_stage, reader_construction_start);
}
source
.open()
.await
.map(|reader| reader.map(|reader| BitrotReader::new(reader, shard_size, checksum_algo, skip_verify)))
let file_open_start = stage_metrics_enabled.then(Instant::now);
let reader = source.open().await?;
if let Some(metrics) = stage_metrics {
record_get_stage_duration_if_enabled(metrics.path, metrics.file_open_stage, file_open_start);
}
let bitrot_reader_init_start = stage_metrics_enabled.then(Instant::now);
let reader = reader.map(|reader| BitrotReader::new(reader, shard_size, checksum_algo, skip_verify));
if let Some(metrics) = stage_metrics {
record_get_stage_duration_if_enabled(metrics.path, metrics.bitrot_reader_init_stage, bitrot_reader_init_start);
}
Ok(reader)
}
#[allow(clippy::too_many_arguments)]
@@ -254,6 +480,7 @@ pub fn create_deferred_bitrot_reader(
offset,
length,
use_mmap_read,
stage_metrics: None,
};
BitrotReader::new(Box::new(DeferredObjectReader::new(source)), shard_size, checksum_algo, skip_verify)
@@ -390,7 +617,7 @@ mod tests {
None,
"test-volume",
"test-path",
payload.len() as i64,
i64::try_from(payload.len()).expect("test payload length should fit i64"),
shard_size,
checksum_algo.clone(),
)
@@ -425,6 +652,52 @@ mod tests {
assert_eq!(&out[..n], b"efgh");
}
#[tokio::test]
async fn test_create_bitrot_reader_from_bytes_preserves_inline_body() {
let shard_size = 4;
let checksum_algo = HashAlgorithm::HighwayHash256S;
let payload = b"abcdefghijkl";
let mut writer = create_bitrot_writer(
true,
None,
"test-volume",
"test-path",
payload.len() as i64,
shard_size,
checksum_algo.clone(),
)
.await
.expect("inline bitrot writer");
for chunk in payload.chunks(shard_size) {
writer.write(chunk).await.expect("write chunk");
}
let inline_data = Bytes::from(writer.into_inline_data().expect("inline buffer"));
let mut reader = create_bitrot_reader_from_bytes(
Some(inline_data),
None,
"test-bucket",
"test-path",
shard_size,
shard_size,
shard_size,
checksum_algo,
false,
false,
)
.await
.expect("create reader from bytes")
.expect("reader");
let mut out = [0u8; 4];
let n = reader.read(&mut out).await.expect("read second shard from bytes");
assert_eq!(n, shard_size);
assert_eq!(&out[..n], b"efgh");
}
#[tokio::test]
async fn test_deferred_bitrot_reader_opens_inline_source_on_read() {
let shard_size = 4;
crates/ecstore/src/object_api/readers.rs
+4
View File
@@ -277,6 +277,7 @@ impl PutObjReader {
pub struct GetObjectReader {
pub stream: Box<dyn AsyncRead + Unpin + Send + Sync>,
pub object_info: ObjectInfo,
pub buffered_body: Option<Bytes>,
}
#[derive(Debug, Clone, Copy)]
@@ -529,6 +530,7 @@ impl ReadPlan {
GetObjectReader {
stream: reader,
object_info: oi.clone(),
buffered_body: None,
},
self.storage_offset,
self.storage_length,
@@ -582,6 +584,7 @@ impl ReadPlan {
GetObjectReader {
stream: final_reader,
object_info,
buffered_body: None,
},
self.storage_offset,
self.storage_length,
@@ -682,6 +685,7 @@ impl ReadPlan {
GetObjectReader {
stream: final_reader,
object_info,
buffered_body: None,
},
self.storage_offset,
self.storage_length,
crates/ecstore/src/object_api/types.rs
+3
View File
@@ -31,6 +31,9 @@ pub struct ObjectOptions {
pub delete_prefix_object: bool,
pub version_id: Option<String>,
pub no_lock: bool,
/// True when an upper layer already holds the object read lock before
/// forwarding a no_lock read to the set layer.
pub metadata_cache_safe: bool,
pub versioned: bool,
pub version_suspended: bool,
crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs
+1
View File
@@ -148,6 +148,7 @@ impl MigrationBackendSpy {
GetObjectReader {
stream: Box::new(Cursor::new(vec![0_u8; 3])),
object_info: ObjectInfo::default(),
buffered_body: None,
}
}
}
crates/ecstore/src/set_disk/mod.rs
+967 -64
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/set_disk/read.rs
+1195 -113
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/store/init.rs
+3 -2
View File
@@ -332,8 +332,8 @@ impl ECStore {
pool_meta: RwLock::new(pool_meta),
rebalance_meta: RwLock::new(None),
decommission_cancelers,
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: tokio::sync::Mutex::new(()),
start_gate: Mutex::new(()),
pool_meta_save_gate: Mutex::new(()),
});
// Only set it when the global deployment ID is not yet configured
@@ -545,6 +545,7 @@ mod tests {
Ok(GetObjectReader {
stream: Box::new(Cursor::new(self.read_payload.clone())),
object_info: self.object_info(bucket, object, self.read_payload.len()),
buffered_body: None,
})
}
crates/ecstore/src/store/object.rs
+77
View File
@@ -428,6 +428,7 @@ impl ECStore {
diag_enabled,
);
opts.no_lock = true;
opts.metadata_cache_safe = true;
Ok(Some(ObjectLockDiagGuard::new(
guard,
@@ -1301,6 +1302,10 @@ impl ECStore {
mod tests {
use super::*;
use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
use crate::layout::{
endpoints::{Endpoints, PoolEndpoints},
format::FormatV3,
};
use bytes::Bytes;
use std::io::Cursor;
use std::sync::Arc;
@@ -1818,6 +1823,76 @@ mod tests {
assert!(lookup_opts.no_lock);
}
async fn new_read_lock_test_store() -> ECStore {
let format = FormatV3::new(1, 2);
let endpoints = vec![
Endpoint::try_from("http://127.0.0.1:9000/data0").expect("first endpoint should parse"),
Endpoint::try_from("http://127.0.0.1:9001/data1").expect("second endpoint should parse"),
];
let pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 2,
endpoints: Endpoints::from(endpoints),
cmd_line: "read-lock-metadata-cache-safe-test".to_string(),
platform: "test".to_string(),
};
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone()]);
let sets = Sets::new(vec![None, None], &pool_endpoints, &format, 0, 1)
.await
.expect("test sets should be created with empty disks");
ECStore {
id: Uuid::new_v4(),
disk_map: HashMap::new(),
pools: vec![sets],
peer_sys: S3PeerSys::new(&endpoint_pools),
pool_meta: RwLock::new(PoolMeta::default()),
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: Mutex::new(()),
pool_meta_save_gate: Mutex::new(()),
}
}
#[tokio::test]
async fn acquired_read_lock_marks_metadata_cache_safe_for_set_layer() {
let store = new_read_lock_test_store().await;
let mut opts = ObjectOptions::default();
let guard = store
.acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
.await
.expect("read lock should be acquired");
assert!(guard.is_some(), "read lock should be held by the outer store layer");
assert!(opts.no_lock, "set layer should not reacquire the object lock");
assert!(
opts.metadata_cache_safe,
"metadata cache is safe only because the outer store layer acquired the read lock"
);
}
#[tokio::test]
async fn prelocked_read_request_does_not_mark_metadata_cache_safe() {
let store = new_read_lock_test_store().await;
let mut opts = ObjectOptions {
no_lock: true,
..Default::default()
};
let guard = store
.acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
.await
.expect("prelocked read should not acquire another lock");
assert!(guard.is_none(), "prelocked caller should keep lock ownership outside ECStore");
assert!(
!opts.metadata_cache_safe,
"generic no_lock callers must stay ineligible for metadata cache unless explicitly marked safe"
);
}
#[tokio::test]
#[serial_test::serial]
async fn reader_lock_is_held_when_optimization_is_disabled() {
@@ -1841,6 +1916,7 @@ mod tests {
let reader = GetObjectReader {
stream: Box::new(Cursor::new(Vec::<u8>::new())),
object_info: ObjectInfo::default(),
buffered_body: None,
};
let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
@@ -1879,6 +1955,7 @@ mod tests {
let reader = GetObjectReader {
stream: Box::new(Cursor::new(vec![1, 2, 3])),
object_info: ObjectInfo::default(),
buffered_body: None,
};
let mut reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
crates/io-metrics/src/lib.rs
+224
View File
@@ -186,6 +186,25 @@ const SHARD_READ_COST_SAME_NODE: &str = "same_node";
const SHARD_READ_COST_UNKNOWN: &str = "unknown";
const LOW_COST_QUORUM_CANDIDATE_FALSE: &str = "false";
const LOW_COST_QUORUM_CANDIDATE_TRUE: &str = "true";
pub const GET_OBJECT_SIZE_BUCKET_LE_4_KIB: &str = "le_4kib";
pub const GET_OBJECT_SIZE_BUCKET_LE_16_KIB: &str = "le_16kib";
pub const GET_OBJECT_SIZE_BUCKET_LE_64_KIB: &str = "le_64kib";
pub const GET_OBJECT_SIZE_BUCKET_LE_128_KIB: &str = "le_128kib";
pub const GET_OBJECT_SIZE_BUCKET_LE_1_MIB: &str = "le_1mib";
pub const GET_OBJECT_SIZE_BUCKET_GT_1_MIB: &str = "gt_1mib";
/// Return the bounded size bucket used by small-object GET diagnostics.
#[inline(always)]
pub const fn get_object_size_bucket(size_bytes: i64) -> &'static str {
match size_bytes {
..=4_096 => GET_OBJECT_SIZE_BUCKET_LE_4_KIB,
4_097..=16_384 => GET_OBJECT_SIZE_BUCKET_LE_16_KIB,
16_385..=65_536 => GET_OBJECT_SIZE_BUCKET_LE_64_KIB,
65_537..=131_072 => GET_OBJECT_SIZE_BUCKET_LE_128_KIB,
131_073..=1_048_576 => GET_OBJECT_SIZE_BUCKET_LE_1_MIB,
_ => GET_OBJECT_SIZE_BUCKET_GT_1_MIB,
}
}
fn saturating_sub_atomic(counter: &AtomicU64, bytes: u64) -> u64 {
let mut current = counter.load(Ordering::Relaxed);
@@ -404,6 +423,39 @@ pub fn record_get_object_reader_stream_poll(
.record(duration_secs);
}
/// Record a poll of the single-chunk in-memory GetObject handoff stream.
#[inline(always)]
pub fn record_get_object_memory_body_stream_poll(source: &'static str, outcome: &'static str, bytes: usize, duration_secs: f64) {
if !get_stage_metrics_enabled() {
return;
}
let bytes_counter = u64::try_from(bytes).unwrap_or(u64::MAX);
counter!(
"rustfs_io_get_object_memory_body_stream_poll_total",
"source" => source,
"outcome" => outcome
)
.increment(1);
counter!(
"rustfs_io_get_object_memory_body_stream_poll_bytes_total",
"source" => source,
"outcome" => outcome
)
.increment(bytes_counter);
histogram!(
"rustfs_io_get_object_memory_body_stream_poll_bytes",
"source" => source,
"outcome" => outcome
)
.record(usize_to_f64(bytes));
histogram!(
"rustfs_io_get_object_memory_body_stream_poll_duration_seconds",
"source" => source,
"outcome" => outcome
)
.record(duration_secs);
}
/// Record I/O queue congestion observation.
#[inline(always)]
pub fn record_io_queue_congestion() {
@@ -446,6 +498,28 @@ pub fn record_get_object_stage_duration(path: &'static str, stage: &'static str,
histogram!("rustfs_io_get_object_stage_duration_seconds", "path" => path, "stage" => stage).record(duration_secs);
}
/// Record GetObject stage duration with bounded object class and size labels.
#[inline(always)]
pub fn record_get_object_stage_duration_by_size(
path: &'static str,
stage: &'static str,
object_class: &'static str,
size_bucket: &'static str,
duration_secs: f64,
) {
if !get_stage_metrics_enabled() {
return;
}
histogram!(
"rustfs_io_get_object_stage_duration_seconds_by_size",
"path" => path,
"stage" => stage,
"object_class" => object_class,
"size_bucket" => size_bucket
)
.record(duration_secs);
}
/// Record GetObject metadata fanout duration.
#[inline(always)]
pub fn record_get_object_metadata_fanout_duration(path: &'static str, duration_secs: f64) {
@@ -485,6 +559,16 @@ pub fn record_get_object_metadata_response(path: &'static str, outcome: &'static
counter!("rustfs_io_get_object_metadata_response_total", "path" => path, "outcome" => outcome).increment(1);
}
/// Record one bounded metadata cache decision.
#[inline(always)]
pub fn record_get_object_metadata_cache_decision(path: &'static str, decision: &'static str, reason: &'static str) {
if !get_stage_metrics_enabled() {
return;
}
counter!("rustfs_io_get_object_metadata_cache_total", "path" => path, "decision" => decision, "reason" => reason)
.increment(1);
}
/// Record aggregate metadata fanout shape for one GetObject metadata read.
#[inline(always)]
pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize, valid: usize, ignored: usize, errors: usize) {
@@ -603,6 +687,37 @@ pub fn record_get_object_reader_path(path: &'static str) {
counter!("rustfs_io_get_object_reader_path_total", "path" => path).increment(1);
}
/// Record the selected GetObject reader path with bounded object class and size labels.
#[inline(always)]
pub fn record_get_object_reader_path_by_size(path: &'static str, object_class: &'static str, size_bucket: &'static str) {
if !get_stage_metrics_enabled() {
return;
}
counter!(
"rustfs_io_get_object_reader_path_by_size_total",
"path" => path,
"object_class" => object_class,
"size_bucket" => size_bucket
)
.increment(1);
}
/// Record the concrete subpath used by the direct-memory GetObject reader.
#[inline(always)]
pub fn record_get_object_direct_memory_subpath(subpath: &'static str, object_class: &'static str, size_bucket: &'static str) {
if !get_stage_metrics_enabled() {
return;
}
counter!("rustfs_io_get_object_direct_memory_subpath_total", "subpath" => subpath).increment(1);
counter!(
"rustfs_io_get_object_direct_memory_subpath_by_size_total",
"subpath" => subpath,
"object_class" => object_class,
"size_bucket" => size_bucket
)
.increment(1);
}
/// Record why the codec streaming reader was not selected.
#[inline(always)]
pub fn record_get_object_codec_streaming_fallback(reason: &'static str) {
@@ -627,6 +742,27 @@ pub fn record_get_object_codec_streaming_decision(outcome: &'static str, object_
.increment(1);
}
/// Record the final codec-streaming rollout decision with bounded size attribution.
#[inline(always)]
pub fn record_get_object_codec_streaming_decision_by_size(
outcome: &'static str,
object_class: &'static str,
reason: &'static str,
size_bucket: &'static str,
) {
if !get_stage_metrics_enabled() {
return;
}
counter!(
"rustfs_io_get_object_codec_streaming_decision_by_size_total",
"outcome" => outcome,
"object_class" => object_class,
"reason" => reason,
"size_bucket" => size_bucket
)
.increment(1);
}
/// Record one decoded reader stripe processed by a GetObject read path.
#[inline(always)]
pub fn record_get_object_reader_stripe(path: &'static str) {
@@ -974,6 +1110,66 @@ pub fn record_get_object_shard_read_fanout(
histogram!("rustfs_io_get_object_shard_read_failed", "path" => path).record(shard_read_fanout_to_f64(failed));
}
/// Record the bitrot reader setup scheduling strategy selected for a GET read.
#[inline(always)]
pub fn record_get_object_reader_setup_strategy(strategy: &'static str, mode: &'static str) {
if !get_stage_metrics_enabled() {
return;
}
counter!(
"rustfs_io_get_object_reader_setup_strategy_total",
"strategy" => strategy,
"mode" => mode
)
.increment(1);
}
/// Record the final bitrot reader setup fanout shape for a GET read.
#[inline(always)]
pub fn record_get_object_reader_setup_fanout(
strategy: &'static str,
mode: &'static str,
scheduled: usize,
attempted: usize,
ready: usize,
failed: usize,
deferred: usize,
) {
if !get_stage_metrics_enabled() {
return;
}
histogram!(
"rustfs_io_get_object_reader_setup_scheduled",
"strategy" => strategy,
"mode" => mode
)
.record(shard_read_fanout_to_f64(scheduled));
histogram!(
"rustfs_io_get_object_reader_setup_attempted",
"strategy" => strategy,
"mode" => mode
)
.record(shard_read_fanout_to_f64(attempted));
histogram!(
"rustfs_io_get_object_reader_setup_ready",
"strategy" => strategy,
"mode" => mode
)
.record(shard_read_fanout_to_f64(ready));
histogram!(
"rustfs_io_get_object_reader_setup_failed",
"strategy" => strategy,
"mode" => mode
)
.record(shard_read_fanout_to_f64(failed));
histogram!(
"rustfs_io_get_object_reader_setup_deferred",
"strategy" => strategy,
"mode" => mode
)
.record(shard_read_fanout_to_f64(deferred));
}
/// Record GetObject metadata resolution duration.
#[inline(always)]
pub fn record_get_object_metadata_phase_duration(duration_secs: f64) {
@@ -1694,10 +1890,13 @@ mod tests {
#[test]
fn test_record_get_object_stage_metrics() {
record_get_object_stage_duration("s3_handler", "request_context", 0.001);
record_get_object_stage_duration_by_size("legacy_duplex", "metadata", "plain_single_part", "le_4kib", 0.001);
record_get_object_reader_path("codec_streaming");
record_get_object_reader_path_by_size("codec_streaming", "plain_single_part", "le_1mib");
record_get_object_codec_streaming_fallback("range");
record_get_object_codec_streaming_decision("fallback", "range", "range");
record_get_object_codec_streaming_decision("use", "plain_single_part", "none");
record_get_object_codec_streaming_decision_by_size("fallback", "plain_single_part", "below_min_size", "le_128kib");
record_get_object_reader_stripe("codec_streaming");
record_get_object_reader_bytes("codec_streaming", 1024);
record_get_object_reader_buffer("codec_streaming", "output", 1024);
@@ -1735,6 +1934,8 @@ mod tests {
record_get_object_pipeline_failure_for_path("codec_streaming", "decode", "read_quorum");
record_get_object_shard_read_observation("codec_streaming", 0, "data", "local", "success", "none", 1024, 0.004, 0.001);
record_get_object_shard_read_cost_summary("codec_streaming", 3, 1, 2, 0, 4, 4, 4, true);
record_get_object_reader_setup_strategy("data_blocks_first", "read_quorum");
record_get_object_reader_setup_fanout("data_blocks_first", "read_quorum", 3, 2, 2, 0, 2);
assert!(0.005_f64.is_sign_positive());
}
@@ -1750,6 +1951,7 @@ mod tests {
record_get_object_reader_prefetch_bytes("codec_streaming", "single_inflight", 4096);
record_get_object_reader_stream_buffer_size("standard", "selected", 131072);
record_get_object_reader_stream_poll("standard", "selected", "ready_data", 8192, 4096, 0.0002);
record_get_object_memory_body_stream_poll("buffered_body", "ready_data", 4096, 0.0001);
assert!(0.0003_f64.is_sign_positive());
}
@@ -1794,8 +1996,11 @@ mod tests {
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
set_get_stage_metrics_enabled(true);
record_get_object_stage_duration("s3_handler", "request_context", 0.001);
record_get_object_stage_duration_by_size("legacy_duplex", "metadata", "plain_single_part", "le_4kib", 0.001);
record_get_object_reader_path("codec_streaming");
record_get_object_reader_path_by_size("codec_streaming", "plain_single_part", "le_1mib");
record_get_object_codec_streaming_fallback("range");
record_get_object_codec_streaming_decision_by_size("fallback", "plain_single_part", "below_min_size", "le_128kib");
record_get_object_reader_stripe("codec_streaming");
record_get_object_reader_bytes("codec_streaming", 1024);
record_get_object_reader_buffer("codec_streaming", "output", 1024);
@@ -1838,8 +2043,11 @@ mod tests {
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
set_get_stage_metrics_enabled(false);
record_get_object_stage_duration("s3_handler", "request_context", 0.001);
record_get_object_stage_duration_by_size("legacy_duplex", "metadata", "plain_single_part", "le_4kib", 0.001);
record_get_object_reader_path("codec_streaming");
record_get_object_reader_path_by_size("codec_streaming", "plain_single_part", "le_1mib");
record_get_object_codec_streaming_fallback("range");
record_get_object_codec_streaming_decision_by_size("fallback", "plain_single_part", "below_min_size", "le_128kib");
record_get_object_reader_stripe("codec_streaming");
record_get_object_reader_bytes("codec_streaming", 1024);
record_get_object_reader_buffer("codec_streaming", "output", 1024);
@@ -1879,6 +2087,22 @@ mod tests {
assert!(!get_stage_metrics_enabled());
}
#[test]
fn test_get_object_size_buckets_match_issue714_matrix() {
assert_eq!(get_object_size_bucket(0), GET_OBJECT_SIZE_BUCKET_LE_4_KIB);
assert_eq!(get_object_size_bucket(1024), GET_OBJECT_SIZE_BUCKET_LE_4_KIB);
assert_eq!(get_object_size_bucket(4096), GET_OBJECT_SIZE_BUCKET_LE_4_KIB);
assert_eq!(get_object_size_bucket(4097), GET_OBJECT_SIZE_BUCKET_LE_16_KIB);
assert_eq!(get_object_size_bucket(10 * 1024), GET_OBJECT_SIZE_BUCKET_LE_16_KIB);
assert_eq!(get_object_size_bucket(16 * 1024), GET_OBJECT_SIZE_BUCKET_LE_16_KIB);
assert_eq!(get_object_size_bucket((16 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_64_KIB);
assert_eq!(get_object_size_bucket(100 * 1024), GET_OBJECT_SIZE_BUCKET_LE_128_KIB);
assert_eq!(get_object_size_bucket(128 * 1024), GET_OBJECT_SIZE_BUCKET_LE_128_KIB);
assert_eq!(get_object_size_bucket((128 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_1_MIB);
assert_eq!(get_object_size_bucket(1024 * 1024), GET_OBJECT_SIZE_BUCKET_LE_1_MIB);
assert_eq!(get_object_size_bucket((1024 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_GT_1_MIB);
}
#[test]
fn test_record_stage_duration_generic() {
// Generic stage duration should always record (no gating flag)
crates/obs/src/telemetry/otel.rs
+55 -6
View File
@@ -54,7 +54,7 @@ use opentelemetry_otlp::{Compression, Protocol, WithExportConfig, WithHttpConfig
use opentelemetry_sdk::propagation::{BaggagePropagator, TraceContextPropagator};
use opentelemetry_sdk::{
logs::SdkLoggerProvider,
metrics::{PeriodicReader, SdkMeterProvider},
metrics::{Aggregation, Instrument, PeriodicReader, SdkMeterProvider, Stream},
trace::{RandomIdGenerator, Sampler, SdkTracerProvider},
};
use percent_encoding::percent_decode_str;
@@ -70,6 +70,19 @@ use tracing_error::ErrorLayer;
use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
use tracing_subscriber::{Layer, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
const GET_OBJECT_DURATION_HISTOGRAM_METRICS: &[&str] = &[
"rustfs_io_get_object_request_duration_seconds",
"rustfs_io_get_object_total_duration_seconds",
"rustfs_io_get_object_total_duration_seconds_with_path",
"rustfs_io_get_object_stage_duration_seconds",
"rustfs_io_get_object_stage_duration_seconds_by_size",
];
const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002, 0.003, 0.004, 0.005, 0.0075, 0.01, 0.015, 0.02, 0.03, 0.05, 0.075,
0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
];
/// Initialize the full OpenTelemetry HTTP pipeline (traces + metrics + logs).
///
/// This function is invoked when at least one OTLP endpoint has been
@@ -418,11 +431,14 @@ fn build_meter_provider(
let (provider, recorder) = Recorder::builder(service_name.to_string())
.with_meter_provider(|b: opentelemetry_sdk::metrics::MeterProviderBuilder| {
let b = b.with_resource(res).with_reader(
PeriodicReader::builder(exporter)
.with_interval(Duration::from_secs(meter_interval))
.build(),
);
let b = b
.with_resource(res)
.with_reader(
PeriodicReader::builder(exporter)
.with_interval(Duration::from_secs(meter_interval))
.build(),
)
.with_view(get_object_duration_histogram_view);
if use_stdout {
b.with_reader(create_periodic_reader(meter_interval))
} else {
@@ -437,6 +453,24 @@ fn build_meter_provider(
Ok(Some(provider))
}
fn get_object_duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
if !is_get_object_duration_histogram_metric(instrument.name()) {
return None;
}
Stream::builder()
.with_aggregation(Aggregation::ExplicitBucketHistogram {
boundaries: GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.to_vec(),
record_min_max: true,
})
.build()
.ok()
}
fn is_get_object_duration_histogram_metric(name: &str) -> bool {
GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name)
}
/// Build an optional [`SdkLoggerProvider`] for the given log endpoint.
///
/// Returns `None` when the endpoint is empty or log export is disabled.
@@ -687,4 +721,19 @@ mod tests {
assert_eq!(resolve_signal_timeout(Some(0), None), None);
assert_eq!(resolve_signal_timeout(None, Some(0)), None);
}
#[test]
fn test_get_object_duration_histogram_metric_match_is_scoped() {
assert!(is_get_object_duration_histogram_metric("rustfs_io_get_object_stage_duration_seconds"));
assert!(is_get_object_duration_histogram_metric("rustfs_io_get_object_request_duration_seconds"));
assert!(!is_get_object_duration_histogram_metric("rustfs_io_put_object_request_duration_seconds"));
assert!(!is_get_object_duration_histogram_metric("rustfs_io_get_object_response_size_bytes"));
}
#[test]
fn test_get_object_duration_histogram_buckets_are_sorted() {
assert!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.windows(2).all(|pair| pair[0] < pair[1]));
assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.first(), Some(&0.0001));
assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.last(), Some(&10.0));
}
}
crates/scanner/src/scanner.rs
+1
View File
@@ -1215,6 +1215,7 @@ mod tests {
Ok(GetObjectReader {
stream: Box::new(Cursor::new(data)),
object_info: ObjectInfo::default(),
buffered_body: None,
})
}
rustfs/src/app/object_usecase.rs
+336 -46
View File
@@ -191,10 +191,32 @@ const LOG_SUBSYSTEM_OBJECT: &str = "object";
const EVENT_PUT_OBJECT_STORE_INFLIGHT_SLOW: &str = "put_object_store_inflight_slow";
const EVENT_PUT_OBJECT_STORE_RETURNED: &str = "put_object_store_returned";
const EVENT_GET_OBJECT_STREAM_BODY: &str = "get_object_stream_body";
const GET_OBJECT_STAGE_PATH_S3_HANDLER: &str = "s3_handler";
const GET_OBJECT_STAGE_REQUEST_INGRESS_TO_CONTEXT: &str = "request_ingress_to_context";
const GET_OBJECT_STAGE_OUTPUT_STRATEGY: &str = "output_strategy";
const GET_OBJECT_STAGE_BODY_BUILD: &str = "body_build";
const GET_OBJECT_STAGE_BODY_ENCRYPTED_BUFFER_READ: &str = "body_encrypted_buffer_read";
const GET_OBJECT_STAGE_BODY_MEMORY_BLOB: &str = "body_memory_blob";
const GET_OBJECT_STAGE_BODY_SEEK_BUFFER_READ: &str = "body_seek_buffer_read";
const GET_OBJECT_STAGE_BODY_STREAM_STRATEGY: &str = "body_stream_strategy";
const GET_OBJECT_STAGE_BODY_STREAMING_BLOB: &str = "body_streaming_blob";
const GET_OBJECT_STAGE_CHECKSUM_HEADERS: &str = "checksum_headers";
const GET_OBJECT_STAGE_LIFECYCLE_EXPIRATION: &str = "lifecycle_expiration";
const GET_OBJECT_STAGE_METADATA_FILTER: &str = "metadata_filter";
const PUT_OBJECT_STORE_WARN_THRESHOLD: Duration = Duration::from_secs(5);
const GET_OBJECT_STREAM_WARN_THRESHOLD: Duration = Duration::from_secs(5);
static GET_OBJECT_BUFFER_THRESHOLD_WARNED: AtomicBool = AtomicBool::new(false);
fn record_get_object_s3_handler_stage_duration(stage: &'static str, start: Option<std::time::Instant>) {
if let Some(start) = start {
rustfs_io_metrics::record_get_object_stage_duration(
GET_OBJECT_STAGE_PATH_S3_HANDLER,
stage,
start.elapsed().as_secs_f64(),
);
}
}
fn decoded_content_length_from_headers(headers: &HeaderMap) -> S3Result<Option<i64>> {
let Some(val) = headers.get(AMZ_DECODED_CONTENT_LENGTH) else {
return Ok(None);
@@ -235,6 +257,23 @@ impl DeadlockRequestGuard {
request_id,
}
}
fn register_if_enabled<F>(
deadlock_detector: Arc<deadlock_detector::DeadlockDetector>,
request_id: &str,
description: F,
) -> Option<Self>
where
F: FnOnce() -> String,
{
if !deadlock_detector.is_enabled() {
return None;
}
let request_id = request_id.to_string();
deadlock_detector.register_request(&request_id, description());
Some(Self::new(deadlock_detector, request_id))
}
}
impl Drop for DeadlockRequestGuard {
@@ -248,7 +287,7 @@ struct GetObjectBootstrap {
wrapper: RequestTimeoutWrapper,
request_start: std::time::Instant,
request_guard: GetObjectGuard,
_deadlock_request_guard: DeadlockRequestGuard,
_deadlock_request_guard: Option<DeadlockRequestGuard>,
concurrent_requests: usize,
}
@@ -273,6 +312,7 @@ struct GetObjectReadSetup {
info: ObjectInfo,
event_info: ObjectInfo,
final_stream: DynReader,
buffered_body: Option<Bytes>,
rs: Option<HTTPRangeSpec>,
content_type: Option<ContentType>,
last_modified: Option<Timestamp>,
@@ -332,6 +372,7 @@ const LARGE_SEQUENTIAL_GET_STREAM_BUFFER_CAP_BYTES: usize = 4 * MI_B;
const LARGE_SEQUENTIAL_GET_READAHEAD_MULTIPLIER: usize = 2;
const LARGE_BODY_READER_STREAM_BUFFER_FLOOR_BYTES: usize = MI_B;
const LARGE_BODY_READER_STREAM_BUFFER_THRESHOLD_BYTES: i64 = 4 * MI_B as i64;
const ENV_RUSTFS_GET_SEEK_BUFFER_ENABLE: &str = "RUSTFS_GET_SEEK_BUFFER_ENABLE";
const ENV_RUSTFS_GET_READER_STREAM_BUFFER_SIZE: &str = "RUSTFS_GET_READER_STREAM_BUFFER_SIZE";
const ENV_RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE: &str = "RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE";
const GET_READER_STREAM_BUFFER_SOURCE_SELECTED: &str = "selected";
@@ -340,6 +381,9 @@ const GET_READER_STREAM_POLL_PENDING: &str = "pending";
const GET_READER_STREAM_POLL_READY_DATA: &str = "ready_data";
const GET_READER_STREAM_POLL_READY_EMPTY: &str = "ready_empty";
const GET_READER_STREAM_POLL_READY_ERROR: &str = "ready_error";
const GET_MEMORY_BODY_SOURCE_BUFFERED_BODY: &str = "buffered_body";
const GET_MEMORY_BODY_SOURCE_SEEK_BUFFER: &str = "seek_buffer";
const GET_MEMORY_BODY_SOURCE_ENCRYPTED_BUFFER: &str = "encrypted_buffer";
fn get_reader_stream_buffer_size_override() -> Option<usize> {
static GET_READER_STREAM_BUFFER_SIZE_OVERRIDE: OnceLock<Option<usize>> = OnceLock::new();
@@ -356,6 +400,11 @@ fn is_get_output_handoff_attribution_enabled() -> bool {
*ENABLED.get_or_init(|| rustfs_utils::get_env_bool(ENV_RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE, false))
}
fn is_get_seek_buffer_enabled() -> bool {
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(|| rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SEEK_BUFFER_ENABLE, false))
}
fn resolve_reader_stream_buffer_size(selected_size: usize, override_size: Option<usize>) -> (usize, &'static str) {
if let Some(override_size) = override_size.filter(|value| *value > 0) {
return (override_size, GET_READER_STREAM_BUFFER_SOURCE_ENV_OVERRIDE);
@@ -437,6 +486,8 @@ pin_project! {
struct MemoryTrackedBytesStream {
bytes: Bytes,
emitted: bool,
started: std::time::Instant,
source: &'static str,
_guard: Option<rustfs_io_metrics::MemoryGaugeGuard>,
}
}
@@ -483,10 +534,12 @@ pin_project! {
}
impl MemoryTrackedBytesStream {
fn new(bytes: Bytes, guard: Option<rustfs_io_metrics::MemoryGaugeGuard>) -> Self {
fn new(bytes: Bytes, source: &'static str, guard: Option<rustfs_io_metrics::MemoryGaugeGuard>) -> Self {
Self {
bytes,
emitted: false,
started: std::time::Instant::now(),
source,
_guard: guard,
}
}
@@ -516,11 +569,32 @@ impl futures::Stream for MemoryTrackedBytesStream {
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.project();
let poll_start = is_get_output_handoff_attribution_enabled().then(std::time::Instant::now);
if *this.emitted {
if let Some(poll_start) = poll_start {
rustfs_io_metrics::record_get_object_memory_body_stream_poll(
this.source,
GET_READER_STREAM_POLL_READY_EMPTY,
0,
poll_start.elapsed().as_secs_f64(),
);
}
return Poll::Ready(None);
}
let first_byte_elapsed = (!this.bytes.is_empty()).then(|| this.started.elapsed());
*this.emitted = true;
if let Some(elapsed) = first_byte_elapsed {
rustfs_io_metrics::record_get_object_first_byte_latency(GET_OBJECT_STAGE_PATH_S3_HANDLER, elapsed.as_secs_f64());
}
if let Some(poll_start) = poll_start {
rustfs_io_metrics::record_get_object_memory_body_stream_poll(
this.source,
GET_READER_STREAM_POLL_READY_DATA,
this.bytes.len(),
poll_start.elapsed().as_secs_f64(),
);
}
Poll::Ready(Some(Ok(this.bytes.clone())))
}
}
@@ -662,6 +736,10 @@ impl<R: AsyncRead + Unpin> AsyncRead for GetObjectStreamingReader<R> {
if !self.first_byte_reported {
self.first_byte_reported = true;
let elapsed = self.elapsed();
rustfs_io_metrics::record_get_object_first_byte_latency(
GET_OBJECT_STAGE_PATH_S3_HANDLER,
elapsed.as_secs_f64(),
);
if elapsed >= GET_OBJECT_STREAM_WARN_THRESHOLD {
warn!(
event = EVENT_GET_OBJECT_STREAM_BODY,
@@ -1149,6 +1227,7 @@ fn should_buffer_get_object_in_memory(
has_range,
configured_threshold,
concurrent_requests,
is_get_seek_buffer_enabled(),
)
}
@@ -1159,8 +1238,12 @@ fn should_buffer_get_object_in_memory_with_threshold(
has_range: bool,
configured_threshold: i64,
concurrent_requests: usize,
seek_buffer_enabled: bool,
) -> bool {
if part_number.is_some() || has_range || response_content_length <= 0 || configured_threshold <= 0 {
if !seek_buffer_enabled || part_number.is_some() || has_range || response_content_length <= 0 || configured_threshold <= 0 {
return false;
}
if usize::try_from(response_content_length).is_err() {
return false;
}
@@ -1188,6 +1271,8 @@ fn should_buffer_get_object_in_memory_with_threshold(
mod deadlock_request_guard_tests {
use super::DeadlockRequestGuard;
use crate::app::storage_api::object_usecase::deadlock_detector::{DeadlockDetector, RequestHangDetectionPolicy};
use std::cell::Cell;
use std::rc::Rc;
use std::sync::Arc;
#[test]
@@ -1208,6 +1293,24 @@ mod deadlock_request_guard_tests {
assert_eq!(detector.tracked_count(), 0);
}
#[test]
fn deadlock_request_guard_skips_disabled_detector() {
let detector = Arc::new(DeadlockDetector::new(RequestHangDetectionPolicy {
enabled: false,
..RequestHangDetectionPolicy::default()
}));
let description_built = Rc::new(Cell::new(false));
let description_built_for_closure = Rc::clone(&description_built);
let guard = DeadlockRequestGuard::register_if_enabled(detector, "test-request-id", || {
description_built_for_closure.set(true);
"test request".to_string()
});
assert!(guard.is_none());
assert!(!description_built.get());
}
}
async fn maybe_enqueue_transition_immediate(obj_info: &ObjectInfo, src: LcEventSrc) {
@@ -1970,13 +2073,39 @@ impl DefaultObjectUsecase {
}
}
fn build_memory_blob(buf: Vec<u8>, response_content_length: i64, _optimal_buffer_size: usize) -> Option<StreamingBlob> {
let guard = rustfs_io_metrics::track_get_object_buffered_bytes(buf.len());
let bytes = Bytes::from(buf);
Some(StreamingBlob::wrap(bytes_stream(
MemoryTrackedBytesStream::new(bytes, guard),
response_content_length as usize,
)))
fn build_memory_bytes_blob(
bytes: Bytes,
response_content_length: i64,
_optimal_buffer_size: usize,
source: &'static str,
) -> Option<StreamingBlob> {
let get_stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
let memory_blob_start = get_stage_metrics_enabled.then(std::time::Instant::now);
let handoff_start = get_stage_metrics_enabled.then(std::time::Instant::now);
let bytes_len = bytes.len();
let guard = rustfs_io_metrics::track_get_object_buffered_bytes(bytes_len);
let remaining = usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX);
let blob = StreamingBlob::wrap(bytes_stream(MemoryTrackedBytesStream::new(bytes, source, guard), remaining));
if let Some(handoff_start) = handoff_start {
rustfs_io_metrics::record_get_object_response_handoff(
"single_chunk",
source,
bytes_len,
response_content_length,
handoff_start.elapsed().as_secs_f64(),
);
}
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_MEMORY_BLOB, memory_blob_start);
Some(blob)
}
fn build_memory_blob(
buf: Vec<u8>,
response_content_length: i64,
optimal_buffer_size: usize,
source: &'static str,
) -> Option<StreamingBlob> {
Self::build_memory_bytes_blob(Bytes::from(buf), response_content_length, optimal_buffer_size, source)
}
fn select_stream_buffer_strategy(
@@ -2007,6 +2136,7 @@ impl DefaultObjectUsecase {
where
R: AsyncRead + Send + Sync + Unpin + 'static,
{
let streaming_blob_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let expected = usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX);
let tuned_stream_buffer_size =
tune_reader_stream_buffer_size(stream_buffer_size, response_content_length, stream_strategy);
@@ -2033,20 +2163,21 @@ impl DefaultObjectUsecase {
handoff_start.elapsed().as_secs_f64(),
);
}
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAMING_BLOB, streaming_blob_start);
Some(blob)
}
fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result<GetObjectBootstrap> {
let timeout_config = GetObjectTimeoutPolicy::from_env();
let timeout_config = GetObjectTimeoutPolicy::cached_from_env();
let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.to_string());
let request_start = std::time::Instant::now();
let request_guard = ConcurrencyManager::track_request();
let concurrent_requests = GetObjectGuard::concurrent_requests();
let deadlock_detector = deadlock_detector::get_deadlock_detector();
let request_id = wrapper.request_id().to_string();
deadlock_detector.register_request(&request_id, format!("GetObject {bucket}/{key}"));
let deadlock_request_guard = DeadlockRequestGuard::new(deadlock_detector, request_id);
let deadlock_request_guard = DeadlockRequestGuard::register_if_enabled(deadlock_detector, wrapper.request_id(), || {
format!("GetObject {bucket}/{key}")
});
Self::ensure_get_object_not_timed_out(&wrapper, &timeout_config, bucket, key, GetObjectTimeoutStage::BeforeProcessing)?;
@@ -2255,6 +2386,8 @@ impl DefaultObjectUsecase {
}
let info = reader.object_info;
let stream = reader.stream;
let buffered_body = reader.buffered_body;
let read_duration = read_start.elapsed();
@@ -2342,6 +2475,7 @@ impl DefaultObjectUsecase {
ssekms_key_id,
encryption_applied,
final_stream,
buffered_body,
) = match sse_decryption(decryption_request).await? {
Some(material) => {
let server_side_encryption = Some(material.server_side_encryption.clone());
@@ -2353,10 +2487,11 @@ impl DefaultObjectUsecase {
sse_customer_key_md5,
material.kms_key_id,
true,
wrap_reader(reader.stream),
wrap_reader(stream),
None,
)
}
None => (None, None, None, None, false, wrap_reader(reader.stream)),
None => (None, None, None, None, false, wrap_reader(stream), buffered_body),
};
// Detect inline fast path: data is in memory, no disk I/O semaphore needed.
@@ -2367,6 +2502,7 @@ impl DefaultObjectUsecase {
info,
event_info,
final_stream,
buffered_body,
rs,
content_type,
last_modified,
@@ -2536,6 +2672,7 @@ impl DefaultObjectUsecase {
part_number: Option<usize>,
has_range: bool,
encryption_applied: bool,
buffered_body: Option<Bytes>,
bucket: &str,
key: &str,
) -> S3Result<Option<StreamingBlob>>
@@ -2548,7 +2685,10 @@ impl DefaultObjectUsecase {
if should_buffer_encrypted_object {
let mut buf = Vec::with_capacity(response_content_length as usize);
if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await {
let buffer_read_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let read_result = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await;
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_ENCRYPTED_BUFFER_READ, buffer_read_start);
if let Err(e) = read_result {
error!(error = %e, "GetObject decrypted object buffering failed");
return Err(ApiError::from(StorageError::other(format!("Failed to read decrypted object: {e}"))).into());
}
@@ -2561,12 +2701,19 @@ impl DefaultObjectUsecase {
);
}
return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size));
return Ok(Self::build_memory_blob(
buf,
response_content_length,
optimal_buffer_size,
GET_MEMORY_BODY_SOURCE_ENCRYPTED_BUFFER,
));
}
debug!(buffer_size = optimal_buffer_size, "Encrypted object uses streaming decrypt path");
let stream_strategy_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let (stream_buffer_size, stream_strategy) =
Self::select_stream_buffer_strategy(response_content_length, optimal_buffer_size, enable_readahead, has_range);
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAM_STRATEGY, stream_strategy_start);
return Ok(Self::build_reader_blob(
final_stream,
response_content_length,
@@ -2577,12 +2724,32 @@ impl DefaultObjectUsecase {
));
}
if let Some(buffered_body) = buffered_body {
if buffered_body.len() != usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX) {
warn!(
expected = response_content_length,
actual = buffered_body.len(),
"Buffered GetObject body size mismatch"
);
}
return Ok(Self::build_memory_bytes_blob(
buffered_body,
response_content_length,
optimal_buffer_size,
GET_MEMORY_BODY_SOURCE_BUFFERED_BODY,
));
}
let should_provide_seek_support =
should_buffer_get_object_in_memory(info, response_content_length, part_number, has_range, concurrent_requests);
if should_provide_seek_support {
let mut buf = Vec::with_capacity(response_content_length as usize);
match tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await {
let buffer_read_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let read_result = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await;
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_SEEK_BUFFER_READ, buffer_read_start);
match read_result {
Ok(_) => {
if buf.len() != response_content_length as usize {
warn!(
@@ -2592,7 +2759,12 @@ impl DefaultObjectUsecase {
);
}
return Ok(Self::build_memory_blob(buf, response_content_length, optimal_buffer_size));
return Ok(Self::build_memory_blob(
buf,
response_content_length,
optimal_buffer_size,
GET_MEMORY_BODY_SOURCE_SEEK_BUFFER,
));
}
Err(e) => {
error!(error = %e, "GetObject seek-support buffering failed");
@@ -2600,8 +2772,10 @@ impl DefaultObjectUsecase {
}
}
let stream_strategy_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let (stream_buffer_size, stream_strategy) =
Self::select_stream_buffer_strategy(response_content_length, optimal_buffer_size, enable_readahead, has_range);
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAM_STRATEGY, stream_strategy_start);
Ok(Self::build_reader_blob(
final_stream,
response_content_length,
@@ -3297,6 +3471,7 @@ impl DefaultObjectUsecase {
info: ObjectInfo,
event_info: ObjectInfo,
final_stream: DynReader,
buffered_body: Option<Bytes>,
rs: Option<HTTPRangeSpec>,
content_type: Option<ContentType>,
last_modified: Option<Timestamp>,
@@ -3314,6 +3489,7 @@ impl DefaultObjectUsecase {
part_number: Option<usize>,
versioned: bool,
) -> S3Result<GetObjectOutputContext> {
let strategy_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let strategy = self.finalize_get_object_strategy(
manager,
bucket,
@@ -3326,12 +3502,14 @@ impl DefaultObjectUsecase {
queue_status,
concurrent_requests,
);
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_OUTPUT_STRATEGY, strategy_start);
let GetObjectStrategyContext {
io_strategy: _,
optimal_buffer_size,
enable_readahead,
} = strategy;
let body_build_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let body = Self::build_get_object_body(
final_stream,
&info,
@@ -3342,12 +3520,16 @@ impl DefaultObjectUsecase {
part_number,
rs.is_some(),
encryption_applied,
buffered_body,
bucket,
key,
)
.await?;
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_BUILD, body_build_start);
let checksum_headers_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let checksums = Self::build_get_object_checksums(&info, &req.headers, part_number, rs.as_ref())?;
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_CHECKSUM_HEADERS, checksum_headers_start);
let output_version_id = if versioned {
info.version_id.map(|vid| {
@@ -3368,10 +3550,16 @@ impl DefaultObjectUsecase {
});
// x-amz-expiration: predict from lifecycle configuration
let lifecycle_expiration_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let expiration = resolve_put_object_expiration(bucket, &info).await;
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_LIFECYCLE_EXPIRATION, lifecycle_expiration_start);
let storage_class = response_storage_class(&info, &info.user_defined);
let content_disposition = info.user_defined.get("content-disposition").cloned();
let metadata_filter_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let metadata = filter_object_metadata(&info.user_defined);
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_METADATA_FILTER, metadata_filter_start);
let output = GetObjectOutput {
body,
content_length: Some(response_content_length),
@@ -3382,7 +3570,7 @@ impl DefaultObjectUsecase {
accept_ranges: Some(ACCEPT_RANGES_BYTES.to_string()),
content_range,
e_tag: info.etag.map(|etag| to_s3s_etag(&etag)),
metadata: filter_object_metadata(&info.user_defined),
metadata,
server_side_encryption,
sse_customer_algorithm,
sse_customer_key_md5,
@@ -3418,11 +3606,19 @@ impl DefaultObjectUsecase {
let _ = context.object_store();
}
let request_id = req
.extensions
.get::<request_context::RequestContext>()
let inbound_request_context = req.extensions.get::<request_context::RequestContext>();
let request_id = inbound_request_context
.map(|ctx| ctx.request_id.clone())
.unwrap_or_else(|| request_context::RequestContext::fallback().request_id);
if rustfs_io_metrics::get_stage_metrics_enabled()
&& let Some(context) = inbound_request_context
{
rustfs_io_metrics::record_get_object_stage_duration(
GET_OBJECT_STAGE_PATH_S3_HANDLER,
GET_OBJECT_STAGE_REQUEST_INGRESS_TO_CONTEXT,
context.start_time.elapsed().as_secs_f64(),
);
}
let bootstrap = Self::init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?;
let timeout_config = bootstrap.timeout_config;
let wrapper = bootstrap.wrapper;
@@ -3477,6 +3673,7 @@ impl DefaultObjectUsecase {
info,
event_info,
final_stream,
buffered_body,
rs,
content_type,
last_modified,
@@ -3495,15 +3692,6 @@ impl DefaultObjectUsecase {
final_stream
};
let versioning_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
if let Some(versioning_start) = versioning_start {
rustfs_io_metrics::record_get_object_stage_duration(
"s3_handler",
"versioning_lookup",
versioning_start.elapsed().as_secs_f64(),
);
}
let output_build_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
let output_context = self
.build_get_object_output_context(
@@ -3514,6 +3702,7 @@ impl DefaultObjectUsecase {
info,
event_info,
final_stream,
buffered_body,
rs,
content_type,
last_modified,
@@ -3529,7 +3718,7 @@ impl DefaultObjectUsecase {
&queue_status,
concurrent_requests,
part_number,
versioned,
opts.versioned,
)
.await?;
if let Some(output_build_start) = output_build_start {
@@ -6007,7 +6196,7 @@ mod tests {
let configured_threshold = 20_i64 * 1024 * 1024 * 1024;
let response_len = 80_i64 * 1024 * 1024;
let should_buffer =
should_buffer_get_object_in_memory_with_threshold(&info, response_len, None, false, configured_threshold, 1);
should_buffer_get_object_in_memory_with_threshold(&info, response_len, None, false, configured_threshold, 1, true);
assert!(
!should_buffer,
@@ -6026,7 +6215,8 @@ mod tests {
None,
false,
configured_threshold,
1
1,
true
));
assert!(!should_buffer_get_object_in_memory_with_threshold(
&info,
@@ -6034,7 +6224,8 @@ mod tests {
Some(1),
false,
configured_threshold,
1
1,
true
));
assert!(!should_buffer_get_object_in_memory_with_threshold(
&info,
@@ -6042,7 +6233,24 @@ mod tests {
None,
true,
configured_threshold,
1
1,
true
));
}
#[test]
fn should_buffer_get_object_in_memory_requires_seek_buffer_opt_in() {
let info = ObjectInfo::default();
let configured_threshold = 10_i64 * 1024 * 1024;
assert!(!should_buffer_get_object_in_memory_with_threshold(
&info,
1024,
None,
false,
configured_threshold,
1,
false
));
}
@@ -6057,7 +6265,8 @@ mod tests {
None,
false,
configured_threshold,
1
1,
true
));
assert!(!should_buffer_get_object_in_memory_with_threshold(
&info,
@@ -6065,7 +6274,8 @@ mod tests {
None,
false,
configured_threshold,
1
1,
true
));
}
@@ -6080,7 +6290,8 @@ mod tests {
None,
false,
configured_threshold,
1
1,
true
));
assert!(!should_buffer_get_object_in_memory_with_threshold(
&info,
@@ -6088,9 +6299,10 @@ mod tests {
None,
false,
configured_threshold,
1
1,
true
));
assert!(!should_buffer_get_object_in_memory_with_threshold(&info, 1024, None, false, 0, 1));
assert!(!should_buffer_get_object_in_memory_with_threshold(&info, 1024, None, false, 0, 1, true));
}
#[test]
@@ -6104,7 +6316,8 @@ mod tests {
None,
false,
configured_threshold,
1
1,
true
));
assert!(!should_buffer_get_object_in_memory_with_threshold(
&info,
@@ -6112,7 +6325,8 @@ mod tests {
None,
false,
configured_threshold,
32
32,
true
));
assert!(should_buffer_get_object_in_memory_with_threshold(
&info,
@@ -6120,7 +6334,8 @@ mod tests {
None,
false,
configured_threshold,
rustfs_config::DEFAULT_OBJECT_HIGH_CONCURRENCY_THRESHOLD
rustfs_config::DEFAULT_OBJECT_HIGH_CONCURRENCY_THRESHOLD,
true
));
}
@@ -6194,6 +6409,7 @@ mod tests {
None,
false,
false,
None,
"test-bucket",
"large-object",
)
@@ -6229,6 +6445,7 @@ mod tests {
None,
false,
true,
None,
"test-bucket",
"large-encrypted-object",
)
@@ -6243,6 +6460,78 @@ mod tests {
);
}
#[tokio::test]
async fn build_get_object_body_uses_buffered_body_without_reader_preread() {
let reads = Arc::new(AtomicUsize::new(0));
let reader = ReadProbeReader {
reads: Arc::clone(&reads),
};
let info = ObjectInfo {
size: 4,
..Default::default()
};
let body = DefaultObjectUsecase::build_get_object_body(
reader,
&info,
4,
128 * 1024,
false,
1,
None,
false,
false,
Some(Bytes::from_static(b"test")),
"test-bucket",
"direct-memory-object",
)
.await
.expect("build_get_object_body should consume buffered body");
assert!(body.is_some());
assert_eq!(
reads.load(AtomicOrdering::Relaxed),
0,
"buffered GetObject body must not be read from the fallback reader"
);
}
#[tokio::test]
async fn build_get_object_body_keeps_small_plain_objects_on_streaming_path_by_default() {
let reads = Arc::new(AtomicUsize::new(0));
let reader = ReadProbeReader {
reads: Arc::clone(&reads),
};
let info = ObjectInfo {
size: 4,
..Default::default()
};
let body = DefaultObjectUsecase::build_get_object_body(
reader,
&info,
4,
128 * 1024,
false,
1,
None,
false,
false,
None,
"test-bucket",
"small-plain-object",
)
.await
.expect("build_get_object_body should keep small plain object on streaming path");
assert!(body.is_some());
assert_eq!(
reads.load(AtomicOrdering::Relaxed),
0,
"default GetObject response construction should not pre-read small plain object data"
);
}
#[test]
fn select_stream_buffer_strategy_expands_large_sequential_gets() {
let (buffer_size, strategy) =
@@ -6887,6 +7176,7 @@ mod tests {
None,
None,
None,
None,
0,
None,
None,
rustfs/src/storage/options.rs
+67 -37
View File
@@ -20,7 +20,8 @@ use rustfs_filemeta::ReplicationStatusType;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_FORCE_DELETE, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, get_header,
insert_header_map, is_encryption_metadata_key, is_internal_key,
insert_header_map,
metadata_compat::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX},
};
use rustfs_utils::http::{
AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER,
@@ -493,6 +494,49 @@ pub fn extract_metadata_from_mime_with_object_name(
}
}
fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool {
value
.get(..prefix.len())
.is_some_and(|head| head.eq_ignore_ascii_case(prefix))
}
fn should_skip_object_metadata_key(key: &str, value: &str, excluded_headers: &[&str]) -> bool {
const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
const X_AMZ_PREFIX: &str = "x-amz-";
// Skip internal/reserved metadata (x-rustfs-internal-* or x-minio-internal-*)
if starts_with_ignore_ascii_case(key, RUSTFS_INTERNAL_PREFIX) || starts_with_ignore_ascii_case(key, MINIO_INTERNAL_PREFIX) {
return true;
}
// Skip internal encryption metadata (x-rustfs-encryption-* or x-minio-encryption-*)
if starts_with_ignore_ascii_case(key, RUSTFS_ENCRYPTION_PREFIX) || starts_with_ignore_ascii_case(key, MINIO_ENCRYPTION_PREFIX)
{
return true;
}
// Skip empty object lock values
if value.is_empty()
&& (key.eq_ignore_ascii_case(X_AMZ_OBJECT_LOCK_MODE.as_str())
|| key.eq_ignore_ascii_case(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()))
{
return true;
}
if key.eq_ignore_ascii_case(AMZ_META_UNENCRYPTED_CONTENT_MD5) || key.eq_ignore_ascii_case(AMZ_META_UNENCRYPTED_CONTENT_LENGTH)
{
return true;
}
if excluded_headers.iter().any(|excluded| key.eq_ignore_ascii_case(excluded)) {
return true;
}
// User metadata is stored without the x-amz-meta- prefix by extract_metadata_from_mime.
starts_with_ignore_ascii_case(key, X_AMZ_PREFIX)
}
pub(crate) fn filter_object_metadata(metadata: &HashMap<String, String>) -> Option<HashMap<String, String>> {
// HTTP headers that should NOT be returned in the Metadata field.
// These headers are returned as separate response headers, not user metadata.
@@ -513,48 +557,19 @@ pub(crate) fn filter_object_metadata(metadata: &HashMap<String, String>) -> Opti
"x-amz-server-side-encryption-aws-kms-key-id",
];
let mut filtered_metadata = HashMap::new();
let mut filtered_metadata = None;
for (k, v) in metadata {
let lower_key = k.to_ascii_lowercase();
// Skip internal/reserved metadata (x-rustfs-internal-* or x-minio-internal-*)
if is_internal_key(&lower_key) {
continue;
}
// Skip internal encryption metadata (x-rustfs-encryption-* or x-minio-encryption-*)
if is_encryption_metadata_key(&lower_key) {
continue;
}
// Skip empty object lock values
if v.is_empty() && (k == &X_AMZ_OBJECT_LOCK_MODE.to_string() || k == &X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string()) {
continue;
}
// Skip UNENCRYPTED metadata placeholders
if k == AMZ_META_UNENCRYPTED_CONTENT_MD5 || k == AMZ_META_UNENCRYPTED_CONTENT_LENGTH {
continue;
}
// Skip excluded HTTP headers (they are returned as separate headers, not metadata)
if EXCLUDED_HEADERS.contains(&lower_key.as_str()) {
continue;
}
// Skip any x-amz-* headers that are not user metadata
// User metadata was stored WITHOUT the x-amz-meta- prefix by extract_metadata_from_mime
if lower_key.starts_with("x-amz-") {
if should_skip_object_metadata_key(k, v, EXCLUDED_HEADERS) {
continue;
}
// Include user-defined metadata (keys like "meta1", "custom-key", etc.)
filtered_metadata.insert(k.clone(), v.clone());
}
if filtered_metadata.is_empty() {
None
} else {
Some(filtered_metadata)
filtered_metadata
.get_or_insert_with(HashMap::new)
.insert(k.clone(), v.clone());
}
filtered_metadata
}
/// Detects content type from object name based on file extension.
@@ -1504,6 +1519,21 @@ mod tests {
assert!(filtered.is_none(), "content-type must not be exposed as user metadata");
}
#[test]
fn test_filter_object_metadata_excludes_case_insensitive_system_headers() {
let mut metadata = HashMap::new();
metadata.insert("Content-Type".to_string(), "application/octet-stream".to_string());
metadata.insert("X-Amz-Storage-Class".to_string(), "STANDARD".to_string());
metadata.insert("X-RustFS-Internal-Healing".to_string(), "true".to_string());
metadata.insert("X-Minio-Encryption-Iv".to_string(), "secret".to_string());
metadata.insert("custom-key".to_string(), "custom-value".to_string());
let filtered = filter_object_metadata(&metadata).expect("user metadata should remain");
assert_eq!(filtered.len(), 1);
assert_eq!(filtered.get("custom-key"), Some(&"custom-value".to_string()));
}
#[test]
fn test_detect_content_type_from_object_name() {
// Test Parquet files (our custom handling)
rustfs/src/storage/timeout_wrapper.rs
+37
View File
@@ -53,6 +53,8 @@
//! }
//! ```
#[cfg(not(test))]
use std::sync::OnceLock;
use std::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;
use tracing::{debug, warn};
@@ -118,6 +120,19 @@ impl GetObjectTimeoutPolicy {
}
}
/// Load the process-wide GetObject timeout policy once for hot request paths.
#[cfg(not(test))]
pub fn cached_from_env() -> Self {
static POLICY: OnceLock<GetObjectTimeoutPolicy> = OnceLock::new();
POLICY.get_or_init(Self::from_env).clone()
}
/// Load the GetObject timeout policy from the current test environment.
#[cfg(test)]
pub fn cached_from_env() -> Self {
Self::from_env()
}
/// Check if timeout is enabled (timeout > 0).
pub fn is_timeout_enabled(&self) -> bool {
self.get_object_timeout > Duration::ZERO
@@ -475,6 +490,28 @@ mod tests {
assert_eq!(config.get_object_timeout, Duration::from_secs(30));
}
#[test]
fn test_cached_timeout_config_is_stable() {
let first = GetObjectTimeoutPolicy::cached_from_env();
let second = GetObjectTimeoutPolicy::cached_from_env();
assert_eq!(first.get_object_timeout, second.get_object_timeout);
assert_eq!(first.enable_dynamic_timeout, second.enable_dynamic_timeout);
assert_eq!(first.bytes_per_second, second.bytes_per_second);
assert_eq!(first.min_timeout, second.min_timeout);
assert_eq!(first.max_timeout, second.max_timeout);
}
#[test]
fn test_cached_timeout_config_honors_test_env_overrides() {
temp_env::with_var(rustfs_config::ENV_OBJECT_GET_TIMEOUT, Some("7"), || {
assert_eq!(GetObjectTimeoutPolicy::cached_from_env().get_object_timeout, Duration::from_secs(7));
});
temp_env::with_var(rustfs_config::ENV_OBJECT_GET_TIMEOUT, Some("11"), || {
assert_eq!(GetObjectTimeoutPolicy::cached_from_env().get_object_timeout, Duration::from_secs(11));
});
}
#[test]
fn test_timeout_config_is_enabled() {
let config = GetObjectTimeoutPolicy::default();
scripts/run_get_codec_streaming_smoke.sh
+987 -43
View File
File diff suppressed because it is too large Load Diff
scripts/run_object_batch_bench_enhanced.sh
+151 -2
View File
@@ -38,6 +38,13 @@ COOLDOWN_SECS=0
BASELINE_CSV=""
EXTRA_ARGS=()
FAILED_FINAL_ROUNDS=0
SERVICE_METRICS_URL=""
SERVICE_METRICS_DIR=""
SERVICE_METRICS_CAPTURE_ATTEMPTS=3
SERVICE_METRICS_CAPTURE_RETRY_SECS=1
SERVICE_METRICS_CONNECT_TIMEOUT_SECS=2
SERVICE_METRICS_MAX_TIME_SECS=15
SERVICE_METRICS_FILTER_REGEX="rustfs_io_get_object_"
usage() {
cat <<'USAGE'
@@ -77,11 +84,23 @@ Enhanced options:
--round-cooldown-secs Compatibility alias for --cooldown-secs
--baseline-csv Baseline median CSV to compare
--extra-args Extra args appended to tool command, quoted as one string
--service-metrics-url Optional Prometheus scrape URL captured before/after each round attempt
--service-metrics-dir Output directory for per-round service metric snapshots
--service-metrics-attempts Capture attempts for each snapshot (default: 3)
--service-metrics-retry-secs Sleep seconds between failed capture attempts (default: 1)
--service-metrics-connect-timeout-secs
Curl connect timeout for each metrics capture (default: 2)
--service-metrics-max-time-secs
Curl max time for each metrics capture (default: 15)
--service-metrics-filter-regex
Regex for retained metrics lines after scrape
(default: rustfs_io_get_object_)
Output files:
round_results.csv One row per round attempt (with retry trace)
median_summary.csv Median metrics per object size
baseline_compare.csv Delta vs baseline (if --baseline-csv is set)
<service-metrics-dir>/*.prom Optional per-round service metric snapshots
Example:
scripts/run_object_batch_bench_enhanced.sh \
@@ -134,6 +153,13 @@ parse_args() {
--cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;;
--round-cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;;
--baseline-csv) BASELINE_CSV="$2"; shift 2 ;;
--service-metrics-url) SERVICE_METRICS_URL="$2"; shift 2 ;;
--service-metrics-dir) SERVICE_METRICS_DIR="$2"; shift 2 ;;
--service-metrics-attempts) SERVICE_METRICS_CAPTURE_ATTEMPTS="$2"; shift 2 ;;
--service-metrics-retry-secs) SERVICE_METRICS_CAPTURE_RETRY_SECS="$2"; shift 2 ;;
--service-metrics-connect-timeout-secs) SERVICE_METRICS_CONNECT_TIMEOUT_SECS="$2"; shift 2 ;;
--service-metrics-max-time-secs) SERVICE_METRICS_MAX_TIME_SECS="$2"; shift 2 ;;
--service-metrics-filter-regex) SERVICE_METRICS_FILTER_REGEX="$2"; shift 2 ;;
--extra-args)
# shellcheck disable=SC2206
EXTRA_ARGS=($2)
@@ -193,6 +219,17 @@ validate_args() {
validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round"
validate_positive_int "$RETRY_SLEEP_SECS" "--retry-sleep-secs"
validate_nonnegative_int "$COOLDOWN_SECS" "--cooldown-secs"
validate_positive_int "$SERVICE_METRICS_CAPTURE_ATTEMPTS" "--service-metrics-attempts"
validate_nonnegative_int "$SERVICE_METRICS_CAPTURE_RETRY_SECS" "--service-metrics-retry-secs"
validate_positive_int "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" "--service-metrics-connect-timeout-secs"
validate_positive_int "$SERVICE_METRICS_MAX_TIME_SECS" "--service-metrics-max-time-secs"
if [[ -n "$SERVICE_METRICS_URL" && -z "$SERVICE_METRICS_DIR" ]]; then
echo "ERROR: --service-metrics-dir is required when --service-metrics-url is set" >&2
exit 1
fi
if [[ -n "$SERVICE_METRICS_URL" ]]; then
require_cmd curl
fi
if [[ "$TOOL" == "s3bench" ]]; then
validate_positive_int "$SAMPLES" "--samples"
fi
@@ -215,6 +252,9 @@ setup_output() {
OUT_DIR="target/bench/object-batch-enhanced-$(date +%Y%m%d-%H%M%S)"
fi
mkdir -p "$OUT_DIR/logs"
if [[ -n "$SERVICE_METRICS_URL" ]]; then
mkdir -p "$SERVICE_METRICS_DIR"
fi
ROUND_CSV="$OUT_DIR/round_results.csv"
MEDIAN_CSV="$OUT_DIR/median_summary.csv"
@@ -334,6 +374,113 @@ extract_metrics() {
echo "$throughput,${reqps_num:-N/A},$latency,$req_p90,$req_p99"
}
metric_snapshot_token() {
local size="$1"
local round="$2"
local attempt="$3"
local safe_size
safe_size="$(printf '%s' "$size" | tr -c '[:alnum:]_.-' '_')"
printf '%s_%s_r%s_a%s' "$TOOL" "$safe_size" "$round" "$attempt"
}
filter_service_metrics_snapshot() {
local input_file="$1"
local output_file="$2"
if [[ -z "$SERVICE_METRICS_FILTER_REGEX" ]]; then
mv "$input_file" "$output_file"
return 0
fi
awk -v pattern="$SERVICE_METRICS_FILTER_REGEX" '$0 ~ pattern { print }' "$input_file" >"$output_file"
if [[ ! -s "$output_file" ]]; then
mv "$input_file" "$output_file"
else
rm -f "$input_file"
fi
}
capture_round_service_metrics() {
local size="$1"
local round="$2"
local attempt="$3"
local phase="$4"
if [[ -z "$SERVICE_METRICS_URL" ]]; then
return 0
fi
local token snapshot_file status_file tmp_file filtered_file capture_attempt raw_bytes snapshot_bytes
token="$(metric_snapshot_token "$size" "$round" "$attempt")"
snapshot_file="${SERVICE_METRICS_DIR}/${token}_${phase}.prom"
status_file="${SERVICE_METRICS_DIR}/${token}_${phase}.status"
tmp_file="${snapshot_file}.tmp"
filtered_file="${snapshot_file}.filtered.tmp"
if [[ "$DRY_RUN" == "true" ]]; then
: >"$snapshot_file"
cat >"$status_file" <<EOF
size=${size}
round=${round}
attempt=${attempt}
phase=${phase}
status=not_run_dry_run
url=${SERVICE_METRICS_URL}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
EOF
return 0
fi
for ((capture_attempt=1; capture_attempt<=SERVICE_METRICS_CAPTURE_ATTEMPTS; capture_attempt++)); do
if curl -fsS --noproxy '*' \
--connect-timeout "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" \
--max-time "$SERVICE_METRICS_MAX_TIME_SECS" \
"$SERVICE_METRICS_URL" >"$tmp_file"; then
if [[ -s "$tmp_file" ]]; then
raw_bytes="$(wc -c <"$tmp_file" | tr -d '[:space:]')"
filter_service_metrics_snapshot "$tmp_file" "$filtered_file"
mv "$filtered_file" "$snapshot_file"
snapshot_bytes="$(wc -c <"$snapshot_file" | tr -d '[:space:]')"
cat >"$status_file" <<EOF
size=${size}
round=${round}
attempt=${attempt}
phase=${phase}
status=ok
capture_attempt=${capture_attempt}
url=${SERVICE_METRICS_URL}
connect_timeout_secs=${SERVICE_METRICS_CONNECT_TIMEOUT_SECS}
max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
raw_bytes=${raw_bytes}
snapshot_bytes=${snapshot_bytes}
EOF
return 0
fi
fi
rm -f "$tmp_file" "$filtered_file"
if (( capture_attempt < SERVICE_METRICS_CAPTURE_ATTEMPTS && SERVICE_METRICS_CAPTURE_RETRY_SECS > 0 )); then
sleep "$SERVICE_METRICS_CAPTURE_RETRY_SECS"
fi
done
: >"$snapshot_file"
cat >"$status_file" <<EOF
size=${size}
round=${round}
attempt=${attempt}
phase=${phase}
status=capture_failed
capture_attempts=${SERVICE_METRICS_CAPTURE_ATTEMPTS}
url=${SERVICE_METRICS_URL}
connect_timeout_secs=${SERVICE_METRICS_CONNECT_TIMEOUT_SECS}
max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
EOF
echo "WARN: failed to capture service metrics size=${size} round=${round} attempt=${attempt} phase=${phase}" >&2
}
median_from_numbers() {
local values="$1"
local count
@@ -364,6 +511,7 @@ run_one_attempt() {
local status="ok"
local exit_code=0
local started_at_utc finished_at_utc
capture_round_service_metrics "$size" "$round" "$attempt" before
started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
if [[ "$TOOL" == "warp" ]]; then
@@ -383,7 +531,7 @@ run_one_attempt() {
if [[ "$INSECURE" == "true" ]]; then
cmd+=("--insecure")
fi
if [[ ${EXTRA_ARGS[@]+_} ]]; then
if ((${#EXTRA_ARGS[@]} > 0)); then
cmd+=("${EXTRA_ARGS[@]}")
fi
@@ -414,7 +562,7 @@ run_one_attempt() {
if [[ "$INSECURE" == "true" ]]; then
cmd+=("-insecure")
fi
if [[ ${EXTRA_ARGS[@]+_} ]]; then
if ((${#EXTRA_ARGS[@]} > 0)); then
cmd+=("${EXTRA_ARGS[@]}")
fi
@@ -432,6 +580,7 @@ run_one_attempt() {
fi
fi
finished_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
capture_round_service_metrics "$size" "$round" "$attempt" after
local metrics throughput_human reqps latency_human throughput_bps latency_ms req_p90_human req_p90_ms req_p99_human req_p99_ms
if [[ "$DRY_RUN" == "true" ]]; then